feat: support GKE private cluster

examples/gh-runner-gke-dind-rootless/main.tf

@@ -18,16 +18,17 @@ module "runner-gke" {
   source  = "terraform-google-modules/github-actions-runners/google//modules/gh-runner-gke"
   version = "~> 4.0"
 
-  project_id             = var.project_id
-  create_network         = true
-  network_name           = "runner-network-dind-r"
-  subnet_name            = "runner-subnet-dind-r"
-  cluster_suffix         = "dind-rootless"
-  gh_app_id              = "123456"
-  gh_app_installation_id = "12345678"
-  gh_app_private_key     = "sample"
-  gh_config_url          = "https://github.com/ORGANIZATION"
-  enable_private_nodes   = true
+  project_id              = var.project_id
+  create_network          = true
+  network_name            = "runner-network-dind-r"
+  subnet_name             = "runner-subnet-dind-r"
+  cluster_suffix          = "dind-rootless"
+  gh_app_id               = "123456"
+  gh_app_installation_id  = "12345678"
+  gh_app_private_key      = "sample"
+  gh_config_url           = "https://github.com/ORGANIZATION"
+  enable_private_endpoint = true
+  enable_private_nodes    = true
 
   # pass values.yaml for dind-rootless runners configuratin
   arc_runners_values = [

examples/gh-runner-gke-dind/main.tf

@@ -18,15 +18,16 @@ module "runner-gke" {
   source  = "terraform-google-modules/github-actions-runners/google//modules/gh-runner-gke"
   version = "~> 4.0"
 
-  project_id             = var.project_id
-  create_network         = true
-  network_name           = "runner-network-dind"
-  subnet_name            = "runner-subnet-dind"
-  cluster_suffix         = "dind"
-  gh_app_id              = "123456"
-  gh_app_installation_id = "12345678"
-  gh_app_private_key     = "sample"
-  gh_config_url          = "https://github.com/ORGANIZATION"
-  arc_container_mode     = "dind"
-  enable_private_nodes   = true
+  project_id              = var.project_id
+  create_network          = true
+  network_name            = "runner-network-dind"
+  subnet_name             = "runner-subnet-dind"
+  cluster_suffix          = "dind"
+  gh_app_id               = "123456"
+  gh_app_installation_id  = "12345678"
+  gh_app_private_key      = "sample"
+  gh_config_url           = "https://github.com/ORGANIZATION"
+  arc_container_mode      = "dind"
+  enable_private_endpoint = true
+  enable_private_nodes    = true
 }

examples/gh-runner-gke-simple/main.tf

@@ -18,11 +18,13 @@ module "runner-gke" {
   source  = "terraform-google-modules/github-actions-runners/google//modules/gh-runner-gke"
   version = "~> 4.0"
 
-  project_id             = var.project_id
-  create_network         = true
-  cluster_suffix         = "k8s"
-  gh_app_id              = "123456"
-  gh_app_installation_id = "12345678"
-  gh_app_private_key     = "sample"
-  gh_config_url          = "https://github.com/ORGANIZATION"
+  project_id              = var.project_id
+  create_network          = true
+  cluster_suffix          = "k8s"
+  gh_app_id               = "123456"
+  gh_app_installation_id  = "12345678"
+  gh_app_private_key      = "sample"
+  gh_config_url           = "https://github.com/ORGANIZATION"
+  enable_private_endpoint = true
+  enable_private_nodes    = true
 }

examples/gh-runner-gke-simple/providers.tf

@@ -22,16 +22,16 @@ data "google_client_config" "default" {
 }
 
 provider "kubernetes" {
-  host                   = "https://${module.runner-gke.kubernetes_endpoint}"
+  host                   = module.runner-gke.connect_gateway_endpoint
   token                  = data.google_client_config.default.access_token
-  cluster_ca_certificate = base64decode(module.runner-gke.ca_certificate)
+  # cluster_ca_certificate = base64decode(module.runner-gke.ca_certificate)
 }
 
 provider "helm" {
   kubernetes {
-    host                   = "https://${module.runner-gke.kubernetes_endpoint}"
+    host                   = module.runner-gke.connect_gateway_endpoint
     token                  = data.google_client_config.default.access_token
-    cluster_ca_certificate = base64decode(module.runner-gke.ca_certificate)
+    # cluster_ca_certificate = base64decode(module.runner-gke.ca_certificate)
     exec {
       api_version = "client.authentication.k8s.io/v1beta1"
       command     = "gke-gcloud-auth-plugin"

modules/gh-runner-gke/README.md

@@ -83,17 +83,19 @@ This example shows how to deploy Self Hosted Runners on GKE that supports Docker
 | arc\_systems\_namespace | Namespace created for the ARC operator pods. | `string` | `"arc-systems"` | no |
 | cluster\_suffix | Name of the GitHub organization associated with this runner cluster. | `string` | `"arc"` | no |
 | create\_network | When set to true, VPC will be auto created | `bool` | `true` | no |
-| enable\_private\_nodes | Whether nodes have internal IP addresses only. | `bool` | `false` | no |
+| enable\_private\_endpoint | Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no |
+| enable\_private\_nodes | Whether nodes have internal IP addresses only | `bool` | `false` | no |
 | gh\_app\_id | After creating the GitHub App, on the GitHub App's page, note the value for "App ID". | `string` | n/a | yes |
 | gh\_app\_installation\_id | You can find the app installation ID on the app installation page, which has the following URL format: `https://github.com/organizations/ORGANIZATION/settings/installations/INSTALLATION_ID` | `string` | n/a | yes |
 | gh\_app\_pre\_defined\_secret\_name | Name for the k8s secret required to configure gh runners on GKE via GitHub App authentication | `string` | `"gh-app-pre-defined-secret"` | no |
 | gh\_app\_private\_key | Under "Private keys", click Generate a private key, and save the .pem file. Use the contents of this file for this variable. | `string` | n/a | yes |
 | gh\_config\_url | URL of GitHub App config. If installed in an organization, this is in the format "https://github.com/ORGANIZATION" | `string` | n/a | yes |
 | ip\_range\_pods\_cidr | The secondary ip range cidr to use for pods | `string` | `"192.168.0.0/18"` | no |
 | ip\_range\_pods\_name | The secondary ip range to use for pods | `string` | `"ip-range-pods"` | no |
-| ip\_range\_services\_cider | The secondary ip range cidr to use for services | `string` | `"192.168.64.0/18"` | no |
+| ip\_range\_services\_cidr | The secondary ip range cidr to use for services | `string` | `"192.168.64.0/18"` | no |
 | ip\_range\_services\_name | The secondary ip range to use for services | `string` | `"ip-range-scv"` | no |
 | machine\_type | Machine type for runner node pool | `string` | `"n1-standard-4"` | no |
+| master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network of the GKE cluster. | `string` | `"172.16.0.0/28"` | no |
 | max\_node\_count | Maximum number of nodes in the runner node pool | `number` | `4` | no |
 | min\_node\_count | Minimum number of nodes in the runner node pool | `number` | `2` | no |
 | network\_name | Name for the VPC network | `string` | `"runner-network"` | no |
@@ -112,6 +114,7 @@ This example shows how to deploy Self Hosted Runners on GKE that supports Docker
 | ca\_certificate | The cluster ca certificate (base64 encoded) |
 | client\_token | The bearer token for auth |
 | cluster\_name | Cluster name |
+| connect\_gateway\_endpoint | Cluster endpoint for connecting via Connect Gateway, required for private clusters |
 | kubernetes\_endpoint | The cluster endpoint |
 | location | Cluster location |
 | network\_name | Name of VPC |

modules/gh-runner-gke/main.tf

@@ -17,6 +17,11 @@ locals {
   network_name    = var.create_network ? google_compute_network.gh-network[0].name : var.network_name
   subnet_name     = var.create_network ? google_compute_subnetwork.gh-subnetwork[0].name : var.subnet_name
   service_account = var.service_account == "" ? "create" : var.service_account
+  connect_gateway = "https://connectgateway.googleapis.com/v1/projects/${data.google_project.project.number}/locations/${module.hub.location}/gkeMemberships/${module.hub.cluster_membership_id}"
+}
+
+data "google_project" "project" {
+  project_id = var.project_id
 }
 
 /*****************************************
@@ -43,20 +48,23 @@ resource "google_compute_subnetwork" "gh-subnetwork" {
 
   secondary_ip_range {
     range_name    = var.ip_range_services_name
-    ip_cidr_range = var.ip_range_services_cider
+    ip_cidr_range = var.ip_range_services_cidr
   }
 }
 /*****************************************
   Runner GKE
  *****************************************/
 module "runner-cluster" {
-  source                   = "terraform-google-modules/kubernetes-engine/google//modules/beta-public-cluster/"
+  source                   = "terraform-google-modules/kubernetes-engine/google//modules/private-cluster/"
   version                  = "~> 32.0"
   project_id               = var.project_id
   name                     = "gh-runner-${var.cluster_suffix}"
   regional                 = false
   region                   = var.region
   zones                    = var.zones
+  enable_private_endpoint  = var.enable_private_endpoint
+  enable_private_nodes     = var.enable_private_nodes
+  master_ipv4_cidr_block   = var.master_ipv4_cidr_block
   network                  = local.network_name
   network_project_id       = var.subnetwork_project != "" ? var.subnetwork_project : var.project_id
   subnetwork               = local.subnet_name
@@ -68,18 +76,33 @@ module "runner-cluster" {
   service_account          = local.service_account
   gce_pd_csi_driver        = true
   deletion_protection      = false
+  master_authorized_networks = [
+    {
+      cidr_block   = google_compute_subnetwork.gh-subnetwork[0].ip_cidr_range
+      display_name = "VPC"
+    }
+  ]
   node_pools = [
     {
-      name                 = "runner-pool"
-      min_count            = var.min_node_count
-      max_count            = var.max_node_count
-      auto_upgrade         = true
-      machine_type         = var.machine_type
-      enable_private_nodes = var.enable_private_nodes
+      name          = "runner-pool"
+      min_count     = var.min_node_count
+      max_count     = var.max_node_count
+      auto_upgrade  = true
+      machine_type  = var.machine_type
+      cpu_cfs_quota = false
     }
   ]
 }
 
+module "hub" {
+  source              = "terraform-google-modules/kubernetes-engine/google//modules/fleet-membership"
+  version             = "~> 32.0"
+  project_id          = var.project_id
+  cluster_name        = module.runner-cluster.name
+  location            = module.runner-cluster.location
+  membership_location = var.region
+}
+
 data "google_client_config" "default" {
 }
 

modules/gh-runner-gke/outputs.tf

@@ -20,6 +20,11 @@ output "kubernetes_endpoint" {
   value       = module.runner-cluster.endpoint
 }
 
+output "connect_gateway_endpoint" {
+  description = "Cluster endpoint for connecting via Connect Gateway, required for private clusters"
+  value       = local.connect_gateway
+}
+
 output "client_token" {
   description = "The bearer token for auth"
   sensitive   = true

modules/gh-runner-gke/variables.tf

@@ -49,7 +49,7 @@ variable "ip_range_pods_cidr" {
   default     = "192.168.0.0/18"
 }
 
-variable "ip_range_services_cider" {
+variable "ip_range_services_cidr" {
   type        = string
   description = "The secondary ip range cidr to use for services"
   default     = "192.168.64.0/18"
@@ -184,8 +184,20 @@ variable "arc_runners_values" {
   default     = []
 }
 
+variable "enable_private_endpoint" {
+  description = "Whether the master's internal IP address is used as the cluster endpoint"
+  type        = bool
+  default     = false
+}
+
 variable "enable_private_nodes" {
+  description = "Whether nodes have internal IP addresses only"
   type        = bool
-  description = "Whether nodes have internal IP addresses only."
   default     = false
 }
+
+variable "master_ipv4_cidr_block" {
+  description = "The IP range in CIDR notation to use for the hosted master network of the GKE cluster."
+  type        = string
+  default     = "172.16.0.0/28"
+}

test/setup/main.tf

@@ -29,7 +29,8 @@ module "project" {
     "storage-api.googleapis.com",
     "iam.googleapis.com",
     "container.googleapis.com",
-    "serviceusage.googleapis.com"
+    "serviceusage.googleapis.com",
+    "gkehub.googleapis.com",
   ]
 }
 
@@ -51,6 +52,7 @@ module "project-gke" {
     "storage-component.googleapis.com",
     "logging.googleapis.com",
     "monitoring.googleapis.com",
+    "gkehub.googleapis.com",
   ]
 }