"""Preserve, then isolate, a Compute instance tagged incident=. Order of work: 1. record the instance exactly as it is, before anything changes, with a hash 2. keep it: out of its instance pool, so the pool cannot replace it 3. back up the boot volume and every attached block volume 4. cut it off: the quarantine security attribute, or a power off 5. mark it done, so the tag change this makes does not start it again Each step publishes a status message, so the incident channel sees progress. """ import datetime import hashlib import io import json import logging import os import oci NAMESPACE = os.environ["QUARANTINE_NAMESPACE"] ATTRIBUTE = os.environ["QUARANTINE_ATTRIBUTE"] ISOLATION = os.environ.get("ISOLATION", "zpr") OS_NAMESPACE = os.environ["EVIDENCE_NAMESPACE"] BUCKET = os.environ["EVIDENCE_BUCKET"] TOPIC = os.environ["STATUS_TOPIC"] DONE = "preserved-and-isolated" log = logging.getLogger() def handler(ctx, data: io.BytesIO = None): event = json.loads(data.getvalue()) instance_id = event.get("data", {}).get("resourceId", "") if not instance_id.startswith("ocid1.instance."): return signer = oci.auth.signers.get_resource_principals_signer() compute = oci.core.ComputeClient({}, signer=signer) storage = oci.core.BlockstorageClient({}, signer=signer) pools = oci.core.ComputeManagementClient({}, signer=signer) objects = oci.object_storage.ObjectStorageClient({}, signer=signer) notify = oci.ons.NotificationDataPlaneClient({}, signer=signer) instance = compute.get_instance(instance_id).data tags = dict(instance.freeform_tags or {}) incident = tags.get("incident") # Marking the instance done changes its tags, which fires this again. Stop there. if not incident or tags.get("incident-status") == DONE: return stamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ") prefix = f"incidents/{incident}/{instance.display_name}/{stamp}" marks = {"incident": incident, "source-instance": instance_id} def say(step, detail=""): body = {"incident": incident, "instance": instance_id, "step": step, "detail": detail, "at": datetime.datetime.now(datetime.timezone.utc).isoformat()} log.info("incident %s: %s %s", incident, step, detail) notify.publish_message(TOPIC, oci.ons.models.MessageDetails( title=f"Incident {incident}: {step}", body=json.dumps(body, indent=2))) # 1. The instance as it was: shape, image, tags, attributes, and its VNICs. record = {"instance": json.loads(str(instance)), "vnic_attachments": [json.loads(str(a)) for a in compute.list_vnic_attachments(instance.compartment_id, instance_id=instance_id).data], "volume_attachments": [json.loads(str(a)) for a in compute.list_volume_attachments(instance.compartment_id, instance_id=instance_id).data], "boot_volume_attachments": [json.loads(str(a)) for a in compute.list_boot_volume_attachments(instance.availability_domain, instance.compartment_id, instance_id=instance_id).data]} raw = json.dumps(record, indent=2, sort_keys=True, default=str).encode() digest = hashlib.sha256(raw).hexdigest() objects.put_object(OS_NAMESPACE, BUCKET, f"{prefix}/instance.json", raw) objects.put_object(OS_NAMESPACE, BUCKET, f"{prefix}/instance.json.sha256", f"{digest} instance.json\n".encode()) say("recorded", f"{prefix}/instance.json sha256 {digest}") # 2. Keep it. An instance pool replaces a member it loses, and terminates # one it scales in. Detaching, without shrinking the pool, does neither. for pool in oci.pagination.list_call_get_all_results(pools.list_instance_pools, instance.compartment_id).data: if pool.lifecycle_state in ("TERMINATED", "TERMINATING"): continue members = oci.pagination.list_call_get_all_results(pools.list_instance_pool_instances, instance.compartment_id, pool.id).data if any(m.id == instance_id for m in members): pools.detach_instance_pool_instance(pool.id, oci.core.models.DetachInstancePoolInstanceDetails( instance_id=instance_id, is_decrement_size=False, is_auto_terminate=False)) say("detached", f"from pool {pool.display_name}; the pool will launch a replacement") # 3. Back up every volume. A backup is fixed when it is requested, and a # full one stands alone. for a in record["boot_volume_attachments"]: b = storage.create_boot_volume_backup(oci.core.models.CreateBootVolumeBackupDetails( boot_volume_id=a["boot_volume_id"], type="FULL", display_name=f"ir-{incident}-boot", freeform_tags=marks)).data say("backup", b.id) for a in record["volume_attachments"]: if a.get("lifecycle_state") == "ATTACHED": b = storage.create_volume_backup(oci.core.models.CreateVolumeBackupDetails( volume_id=a["volume_id"], type="FULL", display_name=f"ir-{incident}-{a['volume_id'][-8:]}", freeform_tags=marks)).data say("backup", b.id) # 4. Cut it off. Tags and attributes are replaced whole on update, so the # ones already there are kept. tags["incident-status"] = DONE update = oci.core.models.UpdateInstanceDetails(freeform_tags=tags) if ISOLATION == "zpr": attributes = dict(instance.security_attributes or {}) attributes.setdefault(NAMESPACE, {})[ATTRIBUTE] = {"value": "true", "mode": "enforce"} update.security_attributes = attributes compute.update_instance(instance_id, update) say("isolated", f"security attribute {NAMESPACE}.{ATTRIBUTE}; no ZPR policy allows its traffic") else: compute.update_instance(instance_id, update) compute.instance_action(instance_id, "STOP") say("isolated", "powered off; memory is gone, the volumes and their backups remain") say("done", prefix)