diff --git a/examples/gh-runner-gke-dind-rootless/main.tf b/examples/gh-runner-gke-dind-rootless/main.tf index 99a1553..5a22060 100644 --- a/examples/gh-runner-gke-dind-rootless/main.tf +++ b/examples/gh-runner-gke-dind-rootless/main.tf @@ -18,16 +18,17 @@ module "runner-gke" { source = "terraform-google-modules/github-actions-runners/google//modules/gh-runner-gke" version = "~> 4.0" - project_id = var.project_id - create_network = true - network_name = "runner-network-dind-r" - subnet_name = "runner-subnet-dind-r" - cluster_suffix = "dind-rootless" - gh_app_id = "123456" - gh_app_installation_id = "12345678" - gh_app_private_key = "sample" - gh_config_url = "https://github.com/ORGANIZATION" - enable_private_nodes = true + project_id = var.project_id + create_network = true + network_name = "runner-network-dind-r" + subnet_name = "runner-subnet-dind-r" + cluster_suffix = "dind-rootless" + gh_app_id = "123456" + gh_app_installation_id = "12345678" + gh_app_private_key = "sample" + gh_config_url = "https://github.com/ORGANIZATION" + enable_private_endpoint = true + enable_private_nodes = true # pass values.yaml for dind-rootless runners configuratin arc_runners_values = [ diff --git a/examples/gh-runner-gke-dind/main.tf b/examples/gh-runner-gke-dind/main.tf index 2cae6e5..35c1a98 100644 --- a/examples/gh-runner-gke-dind/main.tf +++ b/examples/gh-runner-gke-dind/main.tf @@ -18,15 +18,16 @@ module "runner-gke" { source = "terraform-google-modules/github-actions-runners/google//modules/gh-runner-gke" version = "~> 4.0" - project_id = var.project_id - create_network = true - network_name = "runner-network-dind" - subnet_name = "runner-subnet-dind" - cluster_suffix = "dind" - gh_app_id = "123456" - gh_app_installation_id = "12345678" - gh_app_private_key = "sample" - gh_config_url = "https://github.com/ORGANIZATION" - arc_container_mode = "dind" - enable_private_nodes = true + project_id = var.project_id + create_network = true + network_name = "runner-network-dind" + subnet_name = "runner-subnet-dind" + cluster_suffix = "dind" + gh_app_id = "123456" + gh_app_installation_id = "12345678" + gh_app_private_key = "sample" + gh_config_url = "https://github.com/ORGANIZATION" + arc_container_mode = "dind" + enable_private_endpoint = true + enable_private_nodes = true } diff --git a/examples/gh-runner-gke-simple/main.tf b/examples/gh-runner-gke-simple/main.tf index 4c16d36..811018f 100644 --- a/examples/gh-runner-gke-simple/main.tf +++ b/examples/gh-runner-gke-simple/main.tf @@ -18,11 +18,13 @@ module "runner-gke" { source = "terraform-google-modules/github-actions-runners/google//modules/gh-runner-gke" version = "~> 4.0" - project_id = var.project_id - create_network = true - cluster_suffix = "k8s" - gh_app_id = "123456" - gh_app_installation_id = "12345678" - gh_app_private_key = "sample" - gh_config_url = "https://github.com/ORGANIZATION" + project_id = var.project_id + create_network = true + cluster_suffix = "k8s" + gh_app_id = "123456" + gh_app_installation_id = "12345678" + gh_app_private_key = "sample" + gh_config_url = "https://github.com/ORGANIZATION" + enable_private_endpoint = true + enable_private_nodes = true } diff --git a/examples/gh-runner-gke-simple/providers.tf b/examples/gh-runner-gke-simple/providers.tf index a259f8b..fd8bc73 100644 --- a/examples/gh-runner-gke-simple/providers.tf +++ b/examples/gh-runner-gke-simple/providers.tf @@ -22,16 +22,16 @@ data "google_client_config" "default" { } provider "kubernetes" { - host = "https://${module.runner-gke.kubernetes_endpoint}" + host = module.runner-gke.connect_gateway_endpoint token = data.google_client_config.default.access_token - cluster_ca_certificate = base64decode(module.runner-gke.ca_certificate) + # cluster_ca_certificate = base64decode(module.runner-gke.ca_certificate) } provider "helm" { kubernetes { - host = "https://${module.runner-gke.kubernetes_endpoint}" + host = module.runner-gke.connect_gateway_endpoint token = data.google_client_config.default.access_token - cluster_ca_certificate = base64decode(module.runner-gke.ca_certificate) + # cluster_ca_certificate = base64decode(module.runner-gke.ca_certificate) exec { api_version = "client.authentication.k8s.io/v1beta1" command = "gke-gcloud-auth-plugin" diff --git a/modules/gh-runner-gke/README.md b/modules/gh-runner-gke/README.md index ff65c34..aa826f2 100644 --- a/modules/gh-runner-gke/README.md +++ b/modules/gh-runner-gke/README.md @@ -83,7 +83,8 @@ This example shows how to deploy Self Hosted Runners on GKE that supports Docker | arc\_systems\_namespace | Namespace created for the ARC operator pods. | `string` | `"arc-systems"` | no | | cluster\_suffix | Name of the GitHub organization associated with this runner cluster. | `string` | `"arc"` | no | | create\_network | When set to true, VPC will be auto created | `bool` | `true` | no | -| enable\_private\_nodes | Whether nodes have internal IP addresses only. | `bool` | `false` | no | +| enable\_private\_endpoint | Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | +| enable\_private\_nodes | Whether nodes have internal IP addresses only | `bool` | `false` | no | | gh\_app\_id | After creating the GitHub App, on the GitHub App's page, note the value for "App ID". | `string` | n/a | yes | | gh\_app\_installation\_id | You can find the app installation ID on the app installation page, which has the following URL format: `https://github.com/organizations/ORGANIZATION/settings/installations/INSTALLATION_ID` | `string` | n/a | yes | | gh\_app\_pre\_defined\_secret\_name | Name for the k8s secret required to configure gh runners on GKE via GitHub App authentication | `string` | `"gh-app-pre-defined-secret"` | no | @@ -91,9 +92,10 @@ This example shows how to deploy Self Hosted Runners on GKE that supports Docker | gh\_config\_url | URL of GitHub App config. If installed in an organization, this is in the format "https://github.com/ORGANIZATION" | `string` | n/a | yes | | ip\_range\_pods\_cidr | The secondary ip range cidr to use for pods | `string` | `"192.168.0.0/18"` | no | | ip\_range\_pods\_name | The secondary ip range to use for pods | `string` | `"ip-range-pods"` | no | -| ip\_range\_services\_cider | The secondary ip range cidr to use for services | `string` | `"192.168.64.0/18"` | no | +| ip\_range\_services\_cidr | The secondary ip range cidr to use for services | `string` | `"192.168.64.0/18"` | no | | ip\_range\_services\_name | The secondary ip range to use for services | `string` | `"ip-range-scv"` | no | | machine\_type | Machine type for runner node pool | `string` | `"n1-standard-4"` | no | +| master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network of the GKE cluster. | `string` | `"172.16.0.0/28"` | no | | max\_node\_count | Maximum number of nodes in the runner node pool | `number` | `4` | no | | min\_node\_count | Minimum number of nodes in the runner node pool | `number` | `2` | no | | network\_name | Name for the VPC network | `string` | `"runner-network"` | no | @@ -112,6 +114,7 @@ This example shows how to deploy Self Hosted Runners on GKE that supports Docker | ca\_certificate | The cluster ca certificate (base64 encoded) | | client\_token | The bearer token for auth | | cluster\_name | Cluster name | +| connect\_gateway\_endpoint | Cluster endpoint for connecting via Connect Gateway, required for private clusters | | kubernetes\_endpoint | The cluster endpoint | | location | Cluster location | | network\_name | Name of VPC | diff --git a/modules/gh-runner-gke/main.tf b/modules/gh-runner-gke/main.tf index c4b193f..c8cd2d6 100644 --- a/modules/gh-runner-gke/main.tf +++ b/modules/gh-runner-gke/main.tf @@ -17,6 +17,11 @@ locals { network_name = var.create_network ? google_compute_network.gh-network[0].name : var.network_name subnet_name = var.create_network ? google_compute_subnetwork.gh-subnetwork[0].name : var.subnet_name service_account = var.service_account == "" ? "create" : var.service_account + connect_gateway = "https://connectgateway.googleapis.com/v1/projects/${data.google_project.project.number}/locations/${module.hub.location}/gkeMemberships/${module.hub.cluster_membership_id}" +} + +data "google_project" "project" { + project_id = var.project_id } /***************************************** @@ -43,20 +48,23 @@ resource "google_compute_subnetwork" "gh-subnetwork" { secondary_ip_range { range_name = var.ip_range_services_name - ip_cidr_range = var.ip_range_services_cider + ip_cidr_range = var.ip_range_services_cidr } } /***************************************** Runner GKE *****************************************/ module "runner-cluster" { - source = "terraform-google-modules/kubernetes-engine/google//modules/beta-public-cluster/" + source = "terraform-google-modules/kubernetes-engine/google//modules/private-cluster/" version = "~> 32.0" project_id = var.project_id name = "gh-runner-${var.cluster_suffix}" regional = false region = var.region zones = var.zones + enable_private_endpoint = var.enable_private_endpoint + enable_private_nodes = var.enable_private_nodes + master_ipv4_cidr_block = var.master_ipv4_cidr_block network = local.network_name network_project_id = var.subnetwork_project != "" ? var.subnetwork_project : var.project_id subnetwork = local.subnet_name @@ -68,18 +76,33 @@ module "runner-cluster" { service_account = local.service_account gce_pd_csi_driver = true deletion_protection = false + master_authorized_networks = [ + { + cidr_block = google_compute_subnetwork.gh-subnetwork[0].ip_cidr_range + display_name = "VPC" + } + ] node_pools = [ { - name = "runner-pool" - min_count = var.min_node_count - max_count = var.max_node_count - auto_upgrade = true - machine_type = var.machine_type - enable_private_nodes = var.enable_private_nodes + name = "runner-pool" + min_count = var.min_node_count + max_count = var.max_node_count + auto_upgrade = true + machine_type = var.machine_type + cpu_cfs_quota = false } ] } +module "hub" { + source = "terraform-google-modules/kubernetes-engine/google//modules/fleet-membership" + version = "~> 32.0" + project_id = var.project_id + cluster_name = module.runner-cluster.name + location = module.runner-cluster.location + membership_location = var.region +} + data "google_client_config" "default" { } diff --git a/modules/gh-runner-gke/outputs.tf b/modules/gh-runner-gke/outputs.tf index fe48a39..6c5ecea 100644 --- a/modules/gh-runner-gke/outputs.tf +++ b/modules/gh-runner-gke/outputs.tf @@ -20,6 +20,11 @@ output "kubernetes_endpoint" { value = module.runner-cluster.endpoint } +output "connect_gateway_endpoint" { + description = "Cluster endpoint for connecting via Connect Gateway, required for private clusters" + value = local.connect_gateway +} + output "client_token" { description = "The bearer token for auth" sensitive = true diff --git a/modules/gh-runner-gke/variables.tf b/modules/gh-runner-gke/variables.tf index 0d96ed0..3974dcd 100644 --- a/modules/gh-runner-gke/variables.tf +++ b/modules/gh-runner-gke/variables.tf @@ -49,7 +49,7 @@ variable "ip_range_pods_cidr" { default = "192.168.0.0/18" } -variable "ip_range_services_cider" { +variable "ip_range_services_cidr" { type = string description = "The secondary ip range cidr to use for services" default = "192.168.64.0/18" @@ -184,8 +184,20 @@ variable "arc_runners_values" { default = [] } +variable "enable_private_endpoint" { + description = "Whether the master's internal IP address is used as the cluster endpoint" + type = bool + default = false +} + variable "enable_private_nodes" { + description = "Whether nodes have internal IP addresses only" type = bool - description = "Whether nodes have internal IP addresses only." default = false } + +variable "master_ipv4_cidr_block" { + description = "The IP range in CIDR notation to use for the hosted master network of the GKE cluster." + type = string + default = "172.16.0.0/28" +} diff --git a/test/setup/main.tf b/test/setup/main.tf index 6f86d18..21a3faa 100644 --- a/test/setup/main.tf +++ b/test/setup/main.tf @@ -29,7 +29,8 @@ module "project" { "storage-api.googleapis.com", "iam.googleapis.com", "container.googleapis.com", - "serviceusage.googleapis.com" + "serviceusage.googleapis.com", + "gkehub.googleapis.com", ] } @@ -51,6 +52,7 @@ module "project-gke" { "storage-component.googleapis.com", "logging.googleapis.com", "monitoring.googleapis.com", + "gkehub.googleapis.com", ] }