From 9a559eeaa8d6acba8be4008504f1ba593f3dff8d Mon Sep 17 00:00:00 2001 From: Ariel Rolfo Date: Thu, 3 Sep 2026 08:10:05 -0300 Subject: [PATCH 1/3] eks-ng-private: reduce shared platform node group 4 -> 2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The private node group hosts only shared platform services (ingress-nginx, coredns, cert-manager, external-secrets, EBS CSI controllers, metrics-server, New Relic + CloudWatch agents) — ~1.9Gi of requests total. It sat idle at 4x t3.large because cluster-autoscaler never scales down nodes running kube-system pods (skip-nodes-with-system-pods). Raise priv_ng_min_size 0 -> 2 (explicit HA floor, 1 per AZ). Actual reduction to 2 is triggered by setting desiredSize=2 via the EKS API, since TF ignores desired_size (CA-managed). PDBs on ingress/coredns/ebs-csi-controller keep the drain safe (one disruption at a time). --- terraform/environments/eks/terraform.tfvars | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/terraform/environments/eks/terraform.tfvars b/terraform/environments/eks/terraform.tfvars index d5e02749..fb7da914 100644 --- a/terraform/environments/eks/terraform.tfvars +++ b/terraform/environments/eks/terraform.tfvars @@ -22,7 +22,11 @@ db_username_staging = "ceregistrystaging" db_username_prod = "ceregistryprod" priv_ng_max_size = 10 -priv_ng_min_size = 0 +# Floor raised 0 -> 2: this node group holds only shared platform services (ingress, +# coredns, cert-manager, external-secrets, CSI controllers, monitoring). CA never +# scales it down (skip-nodes-with-system-pods), so it sat idle at 4x t3.large. +# 2 nodes (1 per AZ) covers HA for ingress/coredns with room to spare. +priv_ng_min_size = 2 priv_ng_des_size = 2 ## this is irrelevant since the cluster uses the autoscaler to determine the appropriate value for it priv_ng_instance_type = "t3.large" route53_hosted_zone_id = "Z1N75467P1FUL5" From 513c01aa548d52f554473089616a5b419ca9f0bd Mon Sep 17 00:00:00 2001 From: Ariel Rolfo Date: Thu, 3 Sep 2026 09:07:45 -0300 Subject: [PATCH 2/3] eks-ng-private: terraform fmt (fix = alignment after comment insertion) --- terraform/environments/eks/terraform.tfvars | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/eks/terraform.tfvars b/terraform/environments/eks/terraform.tfvars index fb7da914..cbc84374 100644 --- a/terraform/environments/eks/terraform.tfvars +++ b/terraform/environments/eks/terraform.tfvars @@ -21,7 +21,7 @@ db_username_sandbox = "ceregistrysandbox" db_username_staging = "ceregistrystaging" db_username_prod = "ceregistryprod" -priv_ng_max_size = 10 +priv_ng_max_size = 10 # Floor raised 0 -> 2: this node group holds only shared platform services (ingress, # coredns, cert-manager, external-secrets, CSI controllers, monitoring). CA never # scales it down (skip-nodes-with-system-pods), so it sat idle at 4x t3.large. From f0cff0a68b2c9d62fece31a8e3b992ab4d3b3cdb Mon Sep 17 00:00:00 2001 From: Ariel Rolfo Date: Fri, 4 Sep 2026 09:42:50 -0300 Subject: [PATCH 3/3] eks-ng-private: t3.medium via prefix delegation + max_size 3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses PR review (Rohit): the private node group runs only platform/system services, so downsize from t3.large to t3.medium (~half the cost) and cap max at 3 (was 10 — surge headroom for node upgrades, not the excessive 10). t3.medium is capped at 17 pods by the VPC CNI ENI limit and the platform layer runs ~19/node, so this requires prefix delegation: - ENABLE_PREFIX_DELEGATION=true on the aws-node DaemonSet (set on the cluster) - a launch template on the private node group with a nodeadm maxPods=110 override Attaching the LT + changing instance type forces a node-group replacement, so name_prefix + create_before_destroy roll the new t3.medium group in before the old t3.large group is destroyed. Registry app nodes (prod-v2) are untouched; only platform pods reschedule (PDB-protected). --- terraform/environments/eks/terraform.tfvars | 10 ++-- terraform/modules/eks/node-group-private.tf | 54 +++++++++++++++++-- .../modules/eks/private-node-userdata.mime | 14 +++++ 3 files changed, 70 insertions(+), 8 deletions(-) create mode 100644 terraform/modules/eks/private-node-userdata.mime diff --git a/terraform/environments/eks/terraform.tfvars b/terraform/environments/eks/terraform.tfvars index cbc84374..0eeb18be 100644 --- a/terraform/environments/eks/terraform.tfvars +++ b/terraform/environments/eks/terraform.tfvars @@ -21,14 +21,16 @@ db_username_sandbox = "ceregistrysandbox" db_username_staging = "ceregistrystaging" db_username_prod = "ceregistryprod" -priv_ng_max_size = 10 +priv_ng_max_size = 3 # Floor raised 0 -> 2: this node group holds only shared platform services (ingress, # coredns, cert-manager, external-secrets, CSI controllers, monitoring). CA never # scales it down (skip-nodes-with-system-pods), so it sat idle at 4x t3.large. # 2 nodes (1 per AZ) covers HA for ingress/coredns with room to spare. -priv_ng_min_size = 2 -priv_ng_des_size = 2 ## this is irrelevant since the cluster uses the autoscaler to determine the appropriate value for it -priv_ng_instance_type = "t3.large" +priv_ng_min_size = 2 +priv_ng_des_size = 2 ## this is irrelevant since the cluster uses the autoscaler to determine the appropriate value for it +# t3.medium (was t3.large): platform-only workload fits with prefix delegation +# (maxPods 110 via the private node group's launch template); ~half the cost. +priv_ng_instance_type = "t3.medium" route53_hosted_zone_id = "Z1N75467P1FUL5" # Env node group scaling diff --git a/terraform/modules/eks/node-group-private.tf b/terraform/modules/eks/node-group-private.tf index 948895cd..f122332c 100644 --- a/terraform/modules/eks/node-group-private.tf +++ b/terraform/modules/eks/node-group-private.tf @@ -1,17 +1,62 @@ # Create AWS EKS Node Group - Private +# Launch template for the private node group. Sole purpose: raise kubelet max-pods +# to 110 (via nodeadm) so small instances (t3.medium) aren't capped by the VPC CNI +# ENI IP limit (17). Requires prefix delegation (ENABLE_PREFIX_DELEGATION=true on the +# aws-node DaemonSet), which is set on the cluster. AMI is intentionally omitted so +# EKS keeps managing the AL2023 image and merges its bootstrap with the override. +# Disk sizing moves here because a launch template is attached. +resource "aws_launch_template" "eks_ng_private" { + name_prefix = "${var.cluster_name}-eks-ng-private-" + + vpc_security_group_ids = [aws_eks_cluster.eks_cluster.vpc_config[0].cluster_security_group_id] + + block_device_mappings { + device_name = "/dev/xvda" + ebs { + volume_size = 20 + volume_type = "gp3" + encrypted = true + } + } + + metadata_options { + http_endpoint = "enabled" + http_tokens = "required" + http_put_response_hop_limit = 2 + } + + user_data = base64encode(file("${path.module}/private-node-userdata.mime")) + + tag_specifications { + resource_type = "instance" + tags = merge(var.common_tags, { Name = "${var.cluster_name}-eks-ng-private" }) + } + + lifecycle { + create_before_destroy = true + } +} + resource "aws_eks_node_group" "eks_ng_private" { cluster_name = aws_eks_cluster.eks_cluster.name - node_group_name = "${var.cluster_name}-eks-ng-private" - node_role_arn = aws_iam_role.eks_nodegroup_role.arn - subnet_ids = var.private_subnets + # name_prefix (not a fixed name) so create_before_destroy can stand up the + # replacement node group before the old one is destroyed (see lifecycle below). + node_group_name_prefix = "${var.cluster_name}-eks-ng-private-" + node_role_arn = aws_iam_role.eks_nodegroup_role.arn + subnet_ids = var.private_subnets ami_type = "AL2023_x86_64_STANDARD" capacity_type = "ON_DEMAND" - disk_size = 20 instance_types = [var.priv_ng_instance_type] + # disk sizing lives in the launch template (required when a custom LT is attached) + launch_template { + id = aws_launch_template.eks_ng_private.id + version = aws_launch_template.eks_ng_private.latest_version + } + scaling_config { desired_size = var.priv_ng_des_size min_size = var.priv_ng_min_size @@ -26,6 +71,7 @@ resource "aws_eks_node_group" "eks_ng_private" { # us to set an initial size during first provisioning. ########################################################################### lifecycle { + create_before_destroy = true ignore_changes = [ scaling_config[0].desired_size ] diff --git a/terraform/modules/eks/private-node-userdata.mime b/terraform/modules/eks/private-node-userdata.mime new file mode 100644 index 00000000..d61d65b5 --- /dev/null +++ b/terraform/modules/eks/private-node-userdata.mime @@ -0,0 +1,14 @@ +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: application/node.eks.aws + +--- +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + kubelet: + config: + maxPods: 110 +--//--