Registry production cluster: reduce platform node group 4 -> 2

terraform/environments/eks/terraform.tfvars

@@ -21,10 +21,16 @@ db_username_sandbox = "ceregistrysandbox"
 db_username_staging = "ceregistrystaging"
 db_username_prod    = "ceregistryprod"
 
-priv_ng_max_size       = 10
-priv_ng_min_size       = 0
-priv_ng_des_size       = 2 ## this is irrelevant since the cluster uses the autoscaler to determine the appropriate value for it
-priv_ng_instance_type  = "t3.large"
+priv_ng_max_size = 3
+# Floor raised 0 -> 2: this node group holds only shared platform services (ingress,
+# coredns, cert-manager, external-secrets, CSI controllers, monitoring). CA never
+# scales it down (skip-nodes-with-system-pods), so it sat idle at 4x t3.large.
+# 2 nodes (1 per AZ) covers HA for ingress/coredns with room to spare.
+priv_ng_min_size = 2
+priv_ng_des_size = 2 ## this is irrelevant since the cluster uses the autoscaler to determine the appropriate value for it
+# t3.medium (was t3.large): platform-only workload fits with prefix delegation
+# (maxPods 110 via the private node group's launch template); ~half the cost.
+priv_ng_instance_type  = "t3.medium"
 route53_hosted_zone_id = "Z1N75467P1FUL5"
 
 # Env node group scaling

terraform/modules/eks/node-group-private.tf

@@ -1,17 +1,62 @@
 # Create AWS EKS Node Group - Private
 
+# Launch template for the private node group. Sole purpose: raise kubelet max-pods
+# to 110 (via nodeadm) so small instances (t3.medium) aren't capped by the VPC CNI
+# ENI IP limit (17). Requires prefix delegation (ENABLE_PREFIX_DELEGATION=true on the
+# aws-node DaemonSet), which is set on the cluster. AMI is intentionally omitted so
+# EKS keeps managing the AL2023 image and merges its bootstrap with the override.
+# Disk sizing moves here because a launch template is attached.
+resource "aws_launch_template" "eks_ng_private" {
+  name_prefix = "${var.cluster_name}-eks-ng-private-"
+
+  vpc_security_group_ids = [aws_eks_cluster.eks_cluster.vpc_config[0].cluster_security_group_id]
+
+  block_device_mappings {
+    device_name = "/dev/xvda"
+    ebs {
+      volume_size = 20
+      volume_type = "gp3"
+      encrypted   = true
+    }
+  }
+
+  metadata_options {
+    http_endpoint               = "enabled"
+    http_tokens                 = "required"
+    http_put_response_hop_limit = 2
+  }
+
+  user_data = base64encode(file("${path.module}/private-node-userdata.mime"))
+
+  tag_specifications {
+    resource_type = "instance"
+    tags          = merge(var.common_tags, { Name = "${var.cluster_name}-eks-ng-private" })
+  }
+
+  lifecycle {
+    create_before_destroy = true
+  }
+}
+
 resource "aws_eks_node_group" "eks_ng_private" {
   cluster_name = aws_eks_cluster.eks_cluster.name
 
-  node_group_name = "${var.cluster_name}-eks-ng-private"
-  node_role_arn   = aws_iam_role.eks_nodegroup_role.arn
-  subnet_ids      = var.private_subnets
+  # name_prefix (not a fixed name) so create_before_destroy can stand up the
+  # replacement node group before the old one is destroyed (see lifecycle below).
+  node_group_name_prefix = "${var.cluster_name}-eks-ng-private-"
+  node_role_arn          = aws_iam_role.eks_nodegroup_role.arn
+  subnet_ids             = var.private_subnets
 
   ami_type       = "AL2023_x86_64_STANDARD"
   capacity_type  = "ON_DEMAND"
-  disk_size      = 20
   instance_types = [var.priv_ng_instance_type]
 
+  # disk sizing lives in the launch template (required when a custom LT is attached)
+  launch_template {
+    id      = aws_launch_template.eks_ng_private.id
+    version = aws_launch_template.eks_ng_private.latest_version
+  }
+
   scaling_config {
     desired_size = var.priv_ng_des_size
     min_size     = var.priv_ng_min_size
@@ -26,6 +71,7 @@ resource "aws_eks_node_group" "eks_ng_private" {
   # us to set an initial size during first provisioning.
   ###########################################################################
   lifecycle {
+    create_before_destroy = true
     ignore_changes = [
       scaling_config[0].desired_size
     ]

terraform/modules/eks/private-node-userdata.mime

@@ -0,0 +1,14 @@
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="//"
+
+--//
+Content-Type: application/node.eks.aws
+
+---
+apiVersion: node.eks.aws/v1alpha1
+kind: NodeConfig
+spec:
+  kubelet:
+    config:
+      maxPods: 110
+--//--