# container-evidence / aws / t0 "Quick start" # # What a responder needs to preserve a compromised container on EKS: a bucket # whose objects cannot be deleted for a day, a role that can protect and # snapshot the node and write evidence and nothing more, access to the # cluster for that role, and the commands to run, in order, printed with your # names filled in. # # Kubernetes rights come from the incident-responder role in the Kubernetes # evidence template (outcomes/evidence-capture/kubernetes). Apply it to the # cluster first. The access entry below puts this role in its group. # # The cluster's authentication mode must allow access entries (API or # API_AND_CONFIG_MAP). # # Change the two values below, then: # terraform init && terraform apply # terraform output capture # # Verify: run the printed commands against a test pod. The pod's record, logs, # and hashes land in the bucket, the node's volumes have snapshots tagged with # the incident, and deleting an object version from the bucket is refused. locals { cluster = "your-cluster" # change me responders = ["arn:aws:iam::111122223333:role/Responder"] # change me: who may take the role name = "container-evidence" } terraform { required_version = ">= 1.6" required_providers { aws = { source = "hashicorp/aws" version = "~> 6.0" } random = { source = "hashicorp/random" version = "~> 3.6" } } } provider "aws" { default_tags { tags = { owner = "security-operations" environment = "prod" data_classification = "restricted" managed_by = "terraform" } } } data "aws_caller_identity" "current" {} # Keeps names unique, so the quick start and Standard can sit in one account. resource "random_id" "suffix" { byte_length = 3 } data "aws_eks_cluster" "this" { name = local.cluster } # Object Lock in governance mode for one day, so you can try this and clean it # up tomorrow. The Standard version locks for a year, with compliance mode as # an option. resource "aws_s3_bucket" "evidence" { bucket = "${local.name}-${random_id.suffix.hex}-${data.aws_caller_identity.current.account_id}" object_lock_enabled = true } resource "aws_s3_bucket_versioning" "evidence" { bucket = aws_s3_bucket.evidence.id versioning_configuration { status = "Enabled" } } resource "aws_s3_bucket_object_lock_configuration" "evidence" { bucket = aws_s3_bucket.evidence.id rule { default_retention { mode = "GOVERNANCE" days = 1 } } depends_on = [aws_s3_bucket_versioning.evidence] } resource "aws_s3_bucket_public_access_block" "evidence" { bucket = aws_s3_bucket.evidence.id block_public_acls = true block_public_policy = true ignore_public_acls = true restrict_public_buckets = true } resource "aws_s3_bucket_ownership_controls" "evidence" { bucket = aws_s3_bucket.evidence.id rule { object_ownership = "BucketOwnerEnforced" } } data "aws_iam_policy_document" "trust" { statement { actions = ["sts:AssumeRole"] principals { type = "AWS" identifiers = local.responders } } } resource "aws_iam_role" "responder" { name = "${local.name}-responder-${random_id.suffix.hex}" assume_role_policy = data.aws_iam_policy_document.trust.json } # Protect, snapshot, and record. Nothing here can stop, terminate, or change # what a node runs. data "aws_iam_policy_document" "responder" { statement { sid = "FindTheCluster" actions = ["eks:DescribeCluster"] resources = [data.aws_eks_cluster.this.arn] } statement { sid = "FindTheNode" actions = [ "ec2:DescribeInstances", "ec2:DescribeVolumes", "ec2:DescribeSnapshots", "autoscaling:DescribeAutoScalingInstances", ] resources = ["*"] } statement { sid = "KeepTheNode" actions = ["ec2:ModifyInstanceAttribute", "autoscaling:SetInstanceProtection"] resources = ["*"] } statement { sid = "SnapshotTheNode" actions = ["ec2:CreateSnapshots", "ec2:CreateSnapshot", "ec2:CreateTags"] resources = ["*"] } statement { sid = "StoreEvidence" actions = ["s3:PutObject"] resources = ["${aws_s3_bucket.evidence.arn}/*"] } } resource "aws_iam_role_policy" "responder" { name = "container-evidence" role = aws_iam_role.responder.id policy = data.aws_iam_policy_document.responder.json } # The role joins the incident-responders group inside the cluster, which the # incident-responder cluster role is bound to. resource "aws_eks_access_entry" "responder" { cluster_name = local.cluster principal_arn = aws_iam_role.responder.arn kubernetes_groups = ["incident-responders"] type = "STANDARD" } output "bucket" { description = "Where evidence goes." value = aws_s3_bucket.evidence.bucket } output "capture" { description = "The commands, in order. Set ID, NS, and POD first." value = <<-EOT # Run everything as the responder role: ${aws_iam_role.responder.arn} ID=inc-001 NS= POD= aws eks update-kubeconfig --name ${local.cluster} --alias "incident-$ID" NODE=$(kubectl get pod -n "$NS" "$POD" -o jsonpath='{.spec.nodeName}') mkdir -p "$ID" && cd "$ID" # On Fargate there is no node to keep or snapshot: skip steps 1 and 3. kubectl get node "$NODE" -o jsonpath='{.metadata.labels.eks\.amazonaws\.com/compute-type}'; echo # 1. Keep the node: no termination, no scale-in, no autoscaler removal, nothing new scheduled. # Spot instances cannot take termination protection, and Karpenter's annotation does not stop expiry or interruption. INSTANCE=$(kubectl get node "$NODE" -o jsonpath='{.spec.providerID}' | awk -F/ '{print $NF}') aws ec2 modify-instance-attribute --instance-id "$INSTANCE" --disable-api-termination ASG=$(aws autoscaling describe-auto-scaling-instances --instance-ids "$INSTANCE" --query 'AutoScalingInstances[0].AutoScalingGroupName' --output text) [ "$ASG" = "None" ] || aws autoscaling set-instance-protection --instance-ids "$INSTANCE" --auto-scaling-group-name "$ASG" --protected-from-scale-in kubectl annotate node "$NODE" cluster-autoscaler.kubernetes.io/scale-down-disabled=true karpenter.sh/do-not-disrupt=true kubectl cordon "$NODE" # 2. Record the pod, its images by digest, its logs, and its node. kubectl get pod -n "$NS" "$POD" -o json > pod.json kubectl logs -n "$NS" "$POD" --all-containers --timestamps > logs.txt kubectl logs -n "$NS" "$POD" --all-containers --timestamps --previous > logs-previous.txt || true kubectl get events -n "$NS" --field-selector involvedObject.name="$POD" -o json > events.json kubectl get node "$NODE" -o json > node.json # 3. Snapshot every volume on the node at the same instant. aws ec2 create-snapshots --instance-specification InstanceId="$INSTANCE" \ --tag-specifications "ResourceType=snapshot,Tags=[{Key=incident,Value=$ID}]" > snapshots.json # 4. Cut it off. The quarantine network policy from the Kubernetes evidence template matches this label. kubectl label pod -n "$NS" "$POD" quarantine=true incident="$ID" --overwrite # 5. Hash every file, then store them with their hashes. for f in *; do sha256sum "$f" > "$f.sha256"; done aws s3 cp . s3://${aws_s3_bucket.evidence.bucket}/"$ID"/"$NS"/"$POD"/"$(date -u +%Y%m%dT%H%M%SZ)"/ --recursive EOT }