feat: move vcluster DNS records to their own route53 zone, add wildca…

…rd cert for vcluster apps (#559)

stages/apps/lead/vcluster.tf

@@ -11,13 +11,29 @@ module "vcluster_namespace" {
 
 // we need a dedicated instance of ingress-nginx in order to enable ssl passthrough to the k8s API server.
 // we could technically enable this on an existing instance of ingress-nginx, but there's a noticable performance hit
+// we can also reuse this nginx instance to front applications that are running on each vcluster. ingresses and
+// services are synced to the host cluster, so the host cluster needs an ingress controller that these synced ingresses
+// can use.
+module "vcluster_apps_wildcard_cert" {
+  count  = var.enable_vcluster ? 1 : 0
+  source = "../../../modules/common/certificates"
+
+  name      = "vcluster-apps-wildcard"
+  namespace = module.vcluster_namespace[0].name
+  domain    = "apps.vcluster.${var.cluster_name}.${var.root_zone_name}"
+
+  issuer_name = module.cluster_issuer.issuer_name
+  issuer_kind = module.cluster_issuer.issuer_kind
+}
+
 module "vcluster_nginx" {
   count  = var.enable_vcluster ? 1 : 0
   source = "../../../modules/tools/nginx"
 
-  name          = "vcluster"
-  namespace     = module.vcluster_namespace[0].name
-  ingress_class = local.vcluster_ingress_class
+  name                = "vcluster"
+  namespace           = module.vcluster_namespace[0].name
+  ingress_class       = local.vcluster_ingress_class
+  default_certificate = "${module.vcluster_namespace[0].name}/${module.vcluster_apps_wildcard_cert[0].cert_secret_name}"
   extra_args = {
     "enable-ssl-passthrough" : "true"
   }

stages/cloud-provider/aws/lead/iam.tf

@@ -14,9 +14,10 @@ module "external_dns_iam" {
   namespace                   = var.system_namespace
   openid_connect_provider_arn = module.eks.aws_iam_openid_connect_provider_arn
   openid_connect_provider_url = module.eks.aws_iam_openid_connect_provider_url
-  route53_zone_ids = [
-    aws_route53_zone.cluster_zone.zone_id
-  ]
+  route53_zone_ids = compact([
+    aws_route53_zone.cluster_zone.zone_id,
+    var.enable_vcluster ? aws_route53_zone.vcluster[0].zone_id : ""
+  ])
 }
 
 module "cluster_autoscaler_iam" {

stages/cloud-provider/aws/lead/route53.tf

@@ -11,12 +11,38 @@ resource "aws_route53_record" "cluster_zone" {
   zone_id = data.aws_route53_zone.root_zone.zone_id
   name    = "${var.cluster_name}.${data.aws_route53_zone.root_zone.name}"
   type    = "NS"
-  ttl     = "30"
+  ttl     = "60"
+
+  records = aws_route53_zone.cluster_zone.name_servers
+}
+
+// zone for vcluster records
+
+resource "aws_route53_zone" "vcluster" {
+  count = var.enable_vcluster ? 1 : 0
+  name  = "vcluster.${aws_route53_zone.cluster_zone.name}"
+}
+
+resource "aws_route53_record" "vcluster_ns" {
+  count   = var.enable_vcluster ? 1 : 0
+  zone_id = aws_route53_zone.cluster_zone.zone_id
+  name    = aws_route53_zone.vcluster[0].name
+  type    = "NS"
+  ttl     = "60"
+
+  records = aws_route53_zone.vcluster[0].name_servers
+}
+
+resource "aws_route53_record" "vcluster_soa" {
+  count   = var.enable_vcluster ? 1 : 0
+  zone_id = aws_route53_zone.vcluster[0].zone_id
+  name    = aws_route53_zone.vcluster[0].name
+  type    = "SOA"
+  ttl     = "60"
+
+  allow_overwrite = true
 
   records = [
-    aws_route53_zone.cluster_zone.name_servers[0],
-    aws_route53_zone.cluster_zone.name_servers[1],
-    aws_route53_zone.cluster_zone.name_servers[2],
-    aws_route53_zone.cluster_zone.name_servers[3],
+    "${aws_route53_zone.vcluster[0].name_servers[0]}. awsdns-hostmaster.amazon.com. 1 7200 900 1209600 86400"
   ]
 }

stages/cloud-provider/aws/lead/variables.tf

@@ -120,3 +120,7 @@ variable "velero_namespace" {
 variable "enable_eks_ssh_access" {
   default = false
 }
+
+variable "enable_vcluster" {
+  default = false
+}