"""Preserve, then isolate, a virtual machine tagged incident=. Order of work: 1. record the VM exactly as it is, before anything changes, with a hash 2. keep it: a lock that refuses deletion 3. snapshot every managed disk, into the evidence group 4. cut it off: every interface into the quarantine group for its region 5. mark it done, so the tag change this makes does not start it again Each step publishes a status event, so the incident channel sees progress. """ import datetime import hashlib import json import logging import os import re import azure.functions as func from azure.core.messaging import CloudEvent from azure.eventgrid import EventGridPublisherClient from azure.identity import DefaultAzureCredential from azure.mgmt.compute import ComputeManagementClient from azure.mgmt.compute.models import CreationData, Snapshot from azure.mgmt.network import NetworkManagementClient from azure.mgmt.network.models import NetworkSecurityGroup from azure.mgmt.resource.locks import ManagementLockClient from azure.mgmt.resource.locks.models import ManagementLockObject from azure.mgmt.resource.resources import ResourceManagementClient from azure.mgmt.resource.resources.models import Tags, TagsPatchResource from azure.storage.blob import BlobServiceClient NSGS = json.loads(os.environ["QUARANTINE_NSG_IDS"]) # region -> security group id SNAPSHOT_RG = os.environ["SNAPSHOT_RESOURCE_GROUP"] EVIDENCE_URL = os.environ["EVIDENCE_ACCOUNT_URL"] CONTAINER = os.environ["EVIDENCE_CONTAINER"] STATUS = os.environ["STATUS_TOPIC_ENDPOINT"] DONE = "preserved-and-isolated" # A write on the VM names the VM. A write through the tags API names the VM's # tags resource, which sits beneath it. VM = re.compile(r"^/subscriptions/([^/]+)/resourceGroups/([^/]+)/providers/Microsoft\.Compute/virtualMachines/([^/]+)(/providers/Microsoft\.Resources/tags/default)?$", re.I) app = func.FunctionApp() credential = DefaultAzureCredential() status = EventGridPublisherClient(STATUS, credential) def say(incident, vm_id, step, detail=""): body = {"incident": incident, "vm": vm_id, "step": step, "detail": detail, "at": datetime.datetime.now(datetime.timezone.utc).isoformat()} logging.info("incident %s: %s %s", incident, step, detail) status.send([CloudEvent(source="incident-responder", type=f"incident.responder.{step}", subject=vm_id, data=body)]) def safe(value, limit=40): return re.sub(r"[^A-Za-z0-9_-]", "-", value)[:limit].strip("-") or "incident" @app.function_name(name="preserve") @app.event_grid_trigger(arg_name="event") def preserve(event: func.EventGridEvent): m = VM.match(event.get_json().get("resourceUri", "")) if not m: return sub, rg, name = m.group(1), m.group(2), m.group(3) compute = ComputeManagementClient(credential, sub) vm = compute.virtual_machines.get(rg, name, expand="instanceView") tags = vm.tags or {} incident = tags.get("incident") # Marking the VM done changes its tags, which fires this again. Stop there. if not incident or tags.get("incident-status") == DONE: return incident = safe(incident) stamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ") prefix = f"incidents/{incident}/{name}/{stamp}" # 1. The VM as it was: size, disks, interfaces, identity, tags, and its # running state. Hashed, then stored where nothing can change it. record = json.dumps(vm.as_dict(), indent=2, sort_keys=True, default=str).encode() digest = hashlib.sha256(record).hexdigest() blobs = BlobServiceClient(EVIDENCE_URL, credential).get_container_client(CONTAINER) blobs.upload_blob(f"{prefix}/instance.json", record) blobs.upload_blob(f"{prefix}/instance.json.sha256", f"{digest} instance.json\n".encode()) say(incident, vm.id, "recorded", f"{prefix}/instance.json sha256 {digest}") # 2. Keep it. Nothing can delete the VM while the lock is on. ManagementLockClient(credential, sub).management_locks.create_or_update_by_scope( vm.id, f"incident-{incident}", ManagementLockObject(level="CanNotDelete", notes=f"Incident {incident}: evidence")) say(incident, vm.id, "locked", "CanNotDelete") # 3. Snapshot every managed disk. An ephemeral OS disk lives on the host # and cannot be snapshotted, so it is skipped and said so. sp = vm.storage_profile disks = [d.managed_disk.id for d in (sp.data_disks or []) if d.managed_disk] if sp.os_disk.diff_disk_settings: say(incident, vm.id, "snapshot-skipped", "the OS disk is ephemeral and cannot be snapshotted") elif sp.os_disk.managed_disk: disks.insert(0, sp.os_disk.managed_disk.id) for i, disk in enumerate(disks, 1): snap = compute.snapshots.begin_create_or_update(SNAPSHOT_RG, safe(f"ir-{incident}-{name}-{stamp}-{i}", 80), Snapshot( location=vm.location, incremental=True, creation_data=CreationData(create_option="Copy", source_resource_id=disk), tags={"incident": incident, "source-vm": vm.id, "source-disk": disk.rsplit("/", 1)[-1]})).result() say(incident, vm.id, "snapshot", snap.id) # 4. Cut it off. Connections already open may survive the change; a # restart ends them, at the cost of the machine's memory. network = NetworkManagementClient(credential, sub) nsg = NSGS.get(vm.location.lower()) if not nsg: say(incident, vm.id, "isolation-failed", f"no quarantine group in {vm.location}; add it to locations") else: for ref in vm.network_profile.network_interfaces: nic_rg, nic_name = ref.id.split("/")[4], ref.id.split("/")[-1] nic = network.network_interfaces.get(nic_rg, nic_name) nic.network_security_group = NetworkSecurityGroup(id=nsg) network.network_interfaces.begin_create_or_update(nic_rg, nic_name, nic).result() say(incident, vm.id, "isolated", nic_name) ResourceManagementClient(credential, sub).tags.begin_update_at_scope( vm.id, TagsPatchResource(operation="Merge", properties=Tags(tags={"incident-status": DONE}))).result() say(incident, vm.id, "done", prefix)