From 05944d1658cac834c891093b99b9043a176586f5 Mon Sep 17 00:00:00 2001
From: Peter Svensson <peter@sparetimecoders.com>
Date: Mon, 19 Sep 2022 16:12:39 +0200
Subject: [PATCH] feat: support external cluster autoscaler

---
 addons/{addons.yaml => addons.yaml.tpl} |  3 +++
 k8s.tf                                  | 22 ++++++++++++++++------
 locals.tf                               | 25 +++++++++++++++++--------
 vars.tf                                 | 13 ++++++++++---
 4 files changed, 46 insertions(+), 17 deletions(-)
 rename addons/{addons.yaml => addons.yaml.tpl} (68%)

diff --git a/addons/addons.yaml b/addons/addons.yaml.tpl
similarity index 68%
rename from addons/addons.yaml
rename to addons/addons.yaml.tpl
index 08902a1..e8ede4b 100644
--- a/addons/addons.yaml
+++ b/addons/addons.yaml.tpl
@@ -7,6 +7,9 @@ spec:
     - manifest: ${ addon.name }/v${ addon.version }.yaml
       name: ${ addon.name }
       version: ${ addon.version }
+      %{~ if lookup(addon, "kubernetes_version", null) != null ~}
+      kubernetesVersion: '${ addon.kubernetes_version }'
+      %{~ endif ~}
       manifestHash: ${ md5(addon.content) }
       selector:
         k8s-addon: ${ addon.name }
diff --git a/k8s.tf b/k8s.tf
index 3acc6a9..84aff97 100644
--- a/k8s.tf
+++ b/k8s.tf
@@ -1,5 +1,5 @@
 resource "aws_s3_object" "extra_addons" {
-  for_each = { for a in local.addons : a.name => a }
+  for_each = { for a in local.addons : "${a.name}-${a.version}" => a }
   bucket   = var.bucket_state_store.id
   key      = "${var.name}-addons/${each.value.name}/v${each.value.version}.yaml"
   content  = each.value.content
@@ -136,11 +136,14 @@ resource "kops_cluster" "k8s" {
     managed = true
   }
 
-  cluster_autoscaler {
-    balance_similar_node_groups   = false
-    enabled                       = true
-    skip_nodes_with_local_storage = false
-    skip_nodes_with_system_pods   = false
+  dynamic "cluster_autoscaler" {
+    for_each = var.external_cluster_autoscaler ? [] : [1]
+    content {
+      balance_similar_node_groups   = false
+      enabled                       = true
+      skip_nodes_with_local_storage = false
+      skip_nodes_with_system_pods   = false
+    }
   }
 
   container_runtime = var.container_runtime
@@ -335,3 +338,10 @@ data "aws_security_group" "nodes" {
   depends_on = [kops_cluster_updater.k8s_updater]
   name       = "nodes.${var.name}"
 }
+
+module "cluster_autoscaler" {
+  source       = "opzkit/k8s-addons-cluster-autoscaler/aws"
+  version      = "1.25.0"
+  replicas     = length(var.public_subnet_ids) * var.node_min_size > 1 ? 2 : 1
+  cluster_name = var.name
+}
diff --git a/locals.tf b/locals.tf
index 1ed9287..c623a30 100644
--- a/locals.tf
+++ b/locals.tf
@@ -26,10 +26,17 @@ locals {
     var.node_policies
     ]
   )
-  external_permissions = flatten([
-    var.service_account_external_permissions
-    ]
-  )
+
+  external_permissions = concat(var.service_account_external_permissions, var.external_cluster_autoscaler ? [
+    for v in module.cluster_autoscaler.permissions : {
+      name      = v.name
+      namespace = v.namespace
+      aws = {
+        inline_policy = lookup(v.aws, "inline_policy", null)
+        policy_ar_ns  = lookup(v.aws, "policy_ar_ns", tolist(null))
+      }
+    }
+  ] : [])
 
   iam_auth_configmap = {
     name    = "aws_iam_authenticator_config"
@@ -46,11 +53,13 @@ locals {
     content = file("${path.module}/addons/default-request-adder.yaml")
   }
 
-  addons = concat(var.extra_addons, [
-    local.iam_auth_configmap,
-    local.default_request_adder
+  addons = flatten([
+    var.extra_addons, [
+      local.iam_auth_configmap,
+      local.default_request_adder
+    ], var.external_cluster_autoscaler ? module.cluster_autoscaler.addons : []
   ])
-  addons_yaml = templatefile("${path.module}/addons/addons.yaml", {
+  addons_yaml = templatefile("${path.module}/addons/addons.yaml.tpl", {
     addons = local.addons
   })
 
diff --git a/vars.tf b/vars.tf
index 779b144..6645922 100644
--- a/vars.tf
+++ b/vars.tf
@@ -123,9 +123,10 @@ variable "kubernetes_version" {
 
 variable "extra_addons" {
   type = list(object({
-    name    = string,
-    version = string,
-    content = string,
+    name               = string,
+    version            = string,
+    kubernetes_version = optional(string),
+    content            = string,
   }))
   default     = []
   description = "Extra addons in the form [{name: \"<name>\", version:\"<version>\", content: \"<YAML content>\"}]"
@@ -208,3 +209,9 @@ variable "cloud_only" {
   default     = false
   description = "CloudOnly perform rolling update without confirming progress with k8s."
 }
+
+variable "external_cluster_autoscaler" {
+  type        = bool
+  default     = false
+  description = "Use external cluster autoscaler and not the built in kOps addon (to support clusters with only spot instances)"
+}