Remove support for Calico CNI and add support for IPv6 (#10)

cluster.tf

@@ -1,3 +1,10 @@
 data "aws_eks_cluster" "cluster" {
   name = var.cluster_name
+
+  lifecycle {
+    postcondition {
+      condition     = self.kubernetes_network_config[0].ip_family == "ipv6" || !var.enable_high_pod_density
+      error_message = "High pod density can only be enabled with IPv6-based clusters"
+    }
+  }
 }

data/userdata.tpl

@@ -13,31 +13,9 @@ Content-Type: text/x-shellscript; charset="us-ascii"
 set -ex
 
 /etc/eks/bootstrap.sh \
-    --kubelet-extra-args '--node-labels=eks.amazonaws.com/nodegroup-image=${ami_id},eks.amazonaws.com/capacityType=${capacity_type},eks.amazonaws.com/nodegroup=${node_group_name}' \
     --b64-cluster-ca '${certificate_authority_data}' \
     --apiserver-endpoint '${cluster_endpoint}' \
     ${bootstrap_extra_args} \
     '${cluster_name}'
 
-if ${disable_source_dest_checks}
-then
-    # Disable source/dest checks
-    # (see https://docs.projectcalico.org/reference/public-cloud/aws#routing-traffic-within-a-single-vpc-subnet)
-
-    export TOKEN=$(
-        curl -s -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 21600" \
-        "http://169.254.169.254/latest/api/token"
-    )
-    export INSTANCE_ID=$(
-        curl -s -H "X-aws-ec2-metadata-token: $TOKEN" \
-        http://169.254.169.254/latest/meta-data/instance-id
-    )
-    export REGION=$(
-        curl -s -H "X-aws-ec2-metadata-token: $TOKEN" \
-        http://169.254.169.254/latest/dynamic/instance-identity/document | grep region | awk -F\" '{print $4}'
-    )
-
-    aws ec2 modify-instance-attribute --instance-id $INSTANCE_ID --no-source-dest-check --region $REGION
-fi
-
 --//--

examples/basic/cluster.tf

@@ -1,11 +1,11 @@
 locals {
   cluster_name    = "simple-eks-integration-test-for-eks-node-group"
-  cluster_version = "1.20"
+  cluster_version = "1.23"
 }
 
 module "cluster" {
   source  = "vistaprint/simple-eks/aws"
-  version = "0.3.3"
+  version = "0.4.0-rc1"
 
   cluster_name    = local.cluster_name
   cluster_version = local.cluster_version
@@ -33,34 +33,6 @@ module "node_group" {
   worker_role_arn = module.cluster.worker_role_arn
   subnet_ids      = module.cluster.private_subnet_ids
 
-  use_calico_cni = false
-
-  region  = var.aws_region
-  profile = var.aws_profile
-
-  depends_on = [module.cluster]
-}
-
-module "calico_enabled_node_group" {
-  source = "../.."
-
-  cluster_name       = local.cluster_name
-  node_group_version = local.cluster_version
-  node_group_name    = "calico"
-
-  instance_types = ["t3a.small"]
-
-  scaling_config = {
-    desired_size = 1
-    max_size     = 1
-    min_size     = 1
-  }
-
-  worker_role_arn = module.cluster.worker_role_arn
-  subnet_ids      = module.cluster.private_subnet_ids
-
-  use_calico_cni = true
-
   region  = var.aws_region
   profile = var.aws_profile
 
@@ -85,8 +57,7 @@ module "ebs_encrypted_node_group" {
   worker_role_arn = module.cluster.worker_role_arn
   subnet_ids      = module.cluster.private_subnet_ids
 
-  use_calico_cni = false
-  encrypt_ebs    = true
+  encrypt_ebs = true
 
   region  = var.aws_region
   profile = var.aws_profile

examples/basic/main.tf

@@ -1,5 +1,5 @@
 terraform {
-  required_version = ">= 0.13"
+  required_version = ">= 1"
 }
 
 provider "aws" {}

kms-key.tf

@@ -51,7 +51,9 @@ resource "aws_kms_key" "ebs_encryption_key" {
           "kms:Get*",
           "kms:Delete*",
           "kms:ScheduleKeyDeletion",
-          "kms:CancelKeyDeletion"
+          "kms:CancelKeyDeletion",
+          "kms:TagResource",
+          "kms:UntagResource"
         ],
         "Resource" : "*"
       }

launch-template.tf

@@ -1,8 +1,31 @@
 locals {
-  is_launch_template_needed = var.use_calico_cni || var.encrypt_ebs || var.volume_type != null
+  is_launch_template_needed = var.encrypt_ebs || var.volume_type != null
 
   x86_64_ami = "amazon-eks-node-${var.node_group_version}-v*"
   arm64_ami  = "amazon-eks-arm64-node-${var.node_group_version}-v*"
+
+  node_labels = "--node-labels=${join(",", [
+    "eks.amazonaws.com/nodegroup-image=${data.aws_ami.ami.id}",
+    "eks.amazonaws.com/capacityType=${local.capacity_type}",
+    "eks.amazonaws.com/nodegroup=${var.node_group_name}"
+  ])}"
+
+  kubelet_extra_args = "--kubelet-extra-args '${join(" ", concat(
+    [local.node_labels],
+    # TODO: instances with more than 30 vCPUs have a larger value for max-pods.
+    #  Let's compute the value instead of hardcoding it.
+    #  (see https://aws.amazon.com/blogs/containers/amazon-vpc-cni-increases-pods-per-node-limits/)
+    var.enable_high_pod_density ? ["--max-pods=110"] : []
+  ))}'"
+
+  kubernetes_network_config = data.aws_eks_cluster.cluster.kubernetes_network_config[0]
+
+  bootstrap_extra_args = join(" ", concat(
+    var.enable_high_pod_density ? ["--use-max-pods false"] : [],
+    local.kubernetes_network_config.ip_family == "ipv6" ? ["--ip-family ipv6"] : [],
+    local.kubernetes_network_config.ip_family == "ipv6" ? ["--service-ipv6-cidr ${local.kubernetes_network_config.service_ipv6_cidr}"] : [],
+    [local.kubelet_extra_args]
+  ))
 }
 
 data "aws_ami" "ami" {
@@ -41,6 +64,7 @@ resource "aws_launch_template" "worker_nodes" {
     http_endpoint               = "enabled"
     http_tokens                 = "required"
     http_put_response_hop_limit = 2
+    http_protocol_ipv6          = local.kubernetes_network_config.ip_family == "ipv6" ? "enabled" : "disabled"
   }
 
   dynamic "tag_specifications" {
@@ -55,10 +79,6 @@ resource "aws_launch_template" "worker_nodes" {
     cluster_name               = var.cluster_name
     cluster_endpoint           = data.aws_eks_cluster.cluster.endpoint
     certificate_authority_data = data.aws_eks_cluster.cluster.certificate_authority[0].data
-    bootstrap_extra_args       = var.use_calico_cni ? "--use-max-pods false" : ""
-    disable_source_dest_checks = var.use_calico_cni
-    ami_id                     = data.aws_ami.ami.id
-    node_group_name            = var.node_group_name
-    capacity_type              = local.capacity_type
+    bootstrap_extra_args       = local.bootstrap_extra_args
   }))
 }

test/basic_test.go

@@ -35,6 +35,7 @@ func TestTerraformBasicExample(t *testing.T) {
 			"vpc_name":    vpcName,
 		},
 		NoColor: true,
+		Upgrade: true,
 	})
 
 	defer terraform.Destroy(t, terraformOptions)

variables.tf

@@ -51,11 +51,6 @@ variable "architecture" {
   }
 }
 
-variable "use_calico_cni" {
-  type    = bool
-  default = false
-}
-
 variable "worker_role_arn" {
   type = string
 }
@@ -64,6 +59,29 @@ variable "subnet_ids" {
   type = list(string)
 }
 
+variable "enable_high_pod_density" {
+  type    = bool
+  default = false
+
+  description = <<-EOT
+    Nodes in EKS clusters can only host a limited amount of pods. The number
+    of network interfaces in an EC2 instance is what introduces this limit.
+    As this limit is quite small (e.g., 29 pods for an m5.large instance), AWS
+    came up with a solution (prefix delegation) to increase the pod density in
+    EKS nodes.
+
+    By enabling this option, each node will be able to host a signficantly
+    larger amount of pods (e.g., 110 pods for an m5.large instance).
+
+    While prefix delegation works for IPv4-based clusters, this module chooses
+    to only support it for IPv6-based ones, for simplicity reasons.
+
+    For more details see:
+      - https://aws.amazon.com/blogs/containers/amazon-vpc-cni-increases-pods-per-node-limits/
+      - https://docs.aws.amazon.com/eks/latest/userguide/cni-increase-ip-addresses.html
+  EOT
+}
+
 variable "encrypt_ebs" {
   type    = bool
   default = true
@@ -72,4 +90,4 @@ variable "encrypt_ebs" {
 variable "volume_type" {
   type    = string
   default = "gp3"
-}
+}

versions.tf

@@ -2,11 +2,11 @@ terraform {
   required_providers {
     helm = {
       source  = "hashicorp/helm"
-      version = "~> 2.0"
+      version = ">= 2"
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"
-      version = "~> 1.13"
+      version = ">= 2"
     }
   }
 }