"""Preserve and isolate a Compute Engine VM labelled incident=. Order of work: 1. record the VM exactly as it is, before anything changes, with a hash 2. keep it alive: deletion protection on, out of its managed instance group 3. snapshot every disk 4. cut it off: a network tag that deny-all firewall rules match 5. mark it done, so the label change this makes does not start it again Each step publishes a status message, so the incident channel sees progress. """ import datetime import hashlib import json import os import re import functions_framework from google.cloud import compute_v1, pubsub_v1, storage from google.protobuf.json_format import MessageToDict TAG = os.environ.get("QUARANTINE_TAG", "incident-quarantine") BUCKET = os.environ["EVIDENCE_BUCKET"] TOPIC = os.environ["STATUS_TOPIC"] DONE = "preserved-and-isolated" publisher = pubsub_v1.PublisherClient() def say(incident, vm, step, detail=""): msg = {"incident": incident, "instance": vm, "step": step, "detail": detail, "at": datetime.datetime.now(datetime.timezone.utc).isoformat()} publisher.publish(TOPIC, json.dumps(msg).encode()).result() def safe(value, limit=62): """Label values and resource names allow lower-case letters, digits, and dashes.""" n = re.sub(r"[^a-z0-9-]", "-", value.lower())[:limit].strip("-") return n if n[:1].isalpha() else ("ir-" + n)[:limit] @functions_framework.cloud_event def handler(event): parts = event.data.get("protoPayload", {}).get("resourceName", "").split("/") if len(parts) < 6 or parts[4] != "instances": return project, zone, name = parts[1], parts[3], parts[5] vms = compute_v1.InstancesClient() vm = vms.get(project=project, zone=zone, instance=name) incident = vm.labels.get("incident") if not incident or vm.labels.get("incident-status") == DONE: return incident = safe(incident, 40) stamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dt%H%M%Sz") prefix = f"incidents/{incident}/{name}/{stamp}" # 1. The VM as it was: metadata, disks, network, service account, labels. record = json.dumps(MessageToDict(vm._pb), indent=2, sort_keys=True).encode() digest = hashlib.sha256(record).hexdigest() bucket = storage.Client().bucket(BUCKET) bucket.blob(f"{prefix}/instance.json").upload_from_string(record, content_type="application/json") bucket.blob(f"{prefix}/instance.json.sha256").upload_from_string(f"{digest} instance.json\n") say(incident, name, "recorded", f"gs://{BUCKET}/{prefix}/instance.json sha256 {digest}") # 2. Keep it alive. vms.set_deletion_protection(project=project, zone=zone, resource=name, deletion_protection=True).result() for item in vm.metadata.items: if item.key == "created-by" and "/instanceGroupManagers/" in item.value: mig = item.value.split("/")[-1] if "/regions/" in item.value: region = item.value.split("/regions/")[1].split("/")[0] compute_v1.RegionInstanceGroupManagersClient().abandon_instances( project=project, region=region, instance_group_manager=mig, region_instance_group_managers_abandon_instances_request_resource= compute_v1.RegionInstanceGroupManagersAbandonInstancesRequest(instances=[vm.self_link])).result() else: compute_v1.InstanceGroupManagersClient().abandon_instances( project=project, zone=zone, instance_group_manager=mig, instance_group_managers_abandon_instances_request_resource= compute_v1.InstanceGroupManagersAbandonInstancesRequest(instances=[vm.self_link])).result() say(incident, name, "detached", mig) say(incident, name, "protected") # 3. Every disk, snapshotted. disks = compute_v1.DisksClient() for d in vm.disks: disk = d.source.split("/")[-1] snap = safe(f"ir-{incident}-{disk}-{stamp}") disks.create_snapshot(project=project, zone=zone, disk=disk, snapshot_resource=compute_v1.Snapshot( name=snap, labels={"incident": incident, "source-instance": safe(name)})).result() say(incident, name, "snapshot", snap) # 4. Cut off. tags = sorted(set(list(vm.tags.items) + [TAG])) vms.set_tags(project=project, zone=zone, instance=name, tags_resource=compute_v1.Tags(items=tags, fingerprint=vm.tags.fingerprint)).result() say(incident, name, "isolated", TAG) # 5. Done. The fingerprint changed when the tags did, so read it again. fresh = vms.get(project=project, zone=zone, instance=name) labels = dict(fresh.labels) labels["incident-status"] = DONE vms.set_labels(project=project, zone=zone, instance=name, instances_set_labels_request_resource=compute_v1.InstancesSetLabelsRequest( labels=labels, label_fingerprint=fresh.label_fingerprint)).result() say(incident, name, "done")