"""Preserve, then isolate, an EC2 instance tagged incident=. Two events drive it: the incident tag being added -> preserve, then start memory capture memory capture finishing, however -> isolate Isolation waits for memory capture because the capture uploads over the network, and isolation cuts the network. If the instance cannot run the capture, isolation happens straight away. Order matters and follows the order of volatility: record what the instance looks like, stop anything destroying it, fix the disks in time, take memory, then cut it off. """ import datetime import hashlib import json import os import re import boto3 ec2 = boto3.client("ec2") asg = boto3.client("autoscaling") ssm = boto3.client("ssm") s3 = boto3.client("s3") sns = boto3.client("sns") BUCKET = os.environ["EVIDENCE_BUCKET"] DOCUMENT = os.environ["MEMORY_DOCUMENT"] TOOL_KEY = os.environ["TOOL_KEY"] TOPIC = os.environ["STATUS_TOPIC"] QUARANTINE = "incident-quarantine" def now(): return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H%M%SZ") def tell(incident, text): sns.publish(TopicArn=TOPIC, Subject=f"Incident {incident}"[:100], Message=text) def put_with_hash(key, body): """Store an evidence record and, beside it, its SHA-256 in sha256sum form. The hash is taken from the bytes sent, before they leave this function, so evidence-verify can later show the stored copy is the one written here. """ digest = hashlib.sha256(body).hexdigest() s3.put_object(Bucket=BUCKET, Key=key, Body=body) s3.put_object(Bucket=BUCKET, Key=f"{key}.sha256", Body=f"{digest} {key.rsplit('/', 1)[-1]}\n".encode()) return digest def describe(instance_id): return ec2.describe_instances(InstanceIds=[instance_id])["Reservations"][0]["Instances"][0] def preserve(instance_id, incident): inst = describe(instance_id) stamp = now() # The incident id lands in object keys and in the capture document's # parameters, which only accept these characters. incident = re.sub(r"[^A-Za-z0-9._-]", "-", incident)[:64] prefix = f"incidents/{incident}/{instance_id}/{stamp}" # 1. Record the instance as it is, before anything here changes it: its # addresses, security groups, role, volumes, tags, and launch time. put_with_hash(f"{prefix}/instance.json", json.dumps(inst, default=str, indent=2).encode()) # 2. Stop anything destroying it. The Auto Scaling group launches a # replacement, so the service keeps running while this one is examined. ec2.modify_instance_attribute(InstanceId=instance_id, DisableApiTermination={"Value": True}) groups = [g["AutoScalingGroupName"] for g in asg.describe_auto_scaling_instances(InstanceIds=[instance_id])["AutoScalingInstances"]] for g in groups: asg.detach_instances(InstanceIds=[instance_id], AutoScalingGroupName=g, ShouldDecrementDesiredCapacity=False) # 3. Fix the disks in time. Snapshots are point-in-time at the moment they # are requested, so the memory image written afterwards is not in them. snaps = ec2.create_snapshots( InstanceSpecification={"InstanceId": instance_id, "ExcludeBootVolume": False}, Description=f"incident {incident} {instance_id} {stamp}", CopyTagsFromSource="volume", TagSpecifications=[{"ResourceType": "snapshot", "Tags": [ {"Key": "incident", "Value": incident}, {"Key": "source-instance", "Value": instance_id}, {"Key": "captured-at", "Value": stamp}, ]}], )["Snapshots"] snap_ids = [s["SnapshotId"] for s in snaps] manifest = {"incident": incident, "instance": instance_id, "started": stamp, "detached_from": groups, "snapshots": snap_ids} put_with_hash(f"{prefix}/manifest.json", json.dumps(manifest, indent=2).encode()) # 4. Take memory, if Systems Manager can reach the instance. Isolation # follows when the command finishes. info = ssm.describe_instance_information( Filters=[{"Key": "InstanceIds", "Values": [instance_id]}])["InstanceInformationList"] if info and info[0].get("PingStatus") == "Online" and info[0].get("PlatformType") == "Linux": ssm.send_command( InstanceIds=[instance_id], DocumentName=DOCUMENT, TimeoutSeconds=60, Parameters={"EvidenceBucket": [BUCKET], "ToolKey": [TOOL_KEY], "ImageKey": [f"{incident}/{instance_id}/{stamp}"]}, # The instance writes this log with its own credentials, which the # bucket only allows under memory/. OutputS3BucketName=BUCKET, OutputS3KeyPrefix=f"memory/{incident}/{instance_id}/{stamp}/capture-log", Comment=f"incident {incident}"[:100], ) tell(incident, f"{instance_id}: preserved. {len(snap_ids)} snapshot(s): {', '.join(snap_ids)}. " f"Memory capture started. Isolation follows when it finishes.") return {"instance": instance_id, "snapshots": snap_ids, "memory": "started"} isolate(instance_id, incident, reason="memory capture not possible on this instance") return {"instance": instance_id, "snapshots": snap_ids, "memory": "not possible"} def isolate(instance_id, incident, reason): inst = describe(instance_id) vpc = inst["VpcId"] found = ec2.describe_security_groups(Filters=[ {"Name": "group-name", "Values": [QUARANTINE]}, {"Name": "vpc-id", "Values": [vpc]}])["SecurityGroups"] if found: sg = found[0]["GroupId"] else: sg = ec2.create_security_group(GroupName=QUARANTINE, VpcId=vpc, Description="No traffic in or out. Used to isolate instances under investigation.")["GroupId"] egress = ec2.describe_security_groups(GroupIds=[sg])["SecurityGroups"][0]["IpPermissionsEgress"] if egress: ec2.revoke_security_group_egress(GroupId=sg, IpPermissions=egress) for eni in inst["NetworkInterfaces"]: ec2.modify_network_interface_attribute(NetworkInterfaceId=eni["NetworkInterfaceId"], Groups=[sg]) ec2.create_tags(Resources=[instance_id], Tags=[{"Key": "incident-status", "Value": "preserved-and-isolated"}]) tell(incident, f"{instance_id}: isolated in {sg} ({reason}). Connections already open can outlive a " f"security group change. Add a network ACL deny if they must stop now.") def handler(event, context): detail = event.get("detail", {}) if event.get("detail-type") == "Tag Change on Resource": incident = detail.get("tags", {}).get("incident") if not incident: return {"skipped": "incident tag was removed, not added"} return preserve(event["resources"][0].split("/")[-1], incident) if event.get("detail-type") == "EC2 Command Invocation Status-change Notification": instance_id = detail["instance-id"] tags = {t["Key"]: t["Value"] for t in describe(instance_id).get("Tags", [])} incident = tags.get("incident", "unknown") isolate(instance_id, incident, reason=f"memory capture {detail.get('status', 'finished').lower()}") return {"instance": instance_id, "isolated": True} return {"skipped": event.get("detail-type")}