From 8443f370d43f94abbd44ee856358dc857e265d2a Mon Sep 17 00:00:00 2001 From: Chad Swenson Date: Tue, 7 Jan 2025 02:14:28 -0600 Subject: [PATCH] Structured AuthorizationConfiguration (#11852) Adds the ability to configure the Kubernetes API server with a structured authorization configuration file. Structured AuthorizationConfiguration is a new feature in Kubernetes v1.29+ (GA in v1.32) that configures the API server's authorization modes with a structured configuration file. AuthorizationConfiguration files offer features not available with the `--authorization-mode` flag, although Kubespray supports both methods and authorization-mode remains the default for now. Note: Because the `--authorization-config` and `--authorization-mode` flags are mutually exclusive, the `authorization_modes` ansible variable is ignored when `kube_apiserver_use_authorization_config_file` is set to true. The two features cannot be used at the same time. Docs: https://kubernetes.io/docs/reference/access-authn-authz/authorization/#configuring-the-api-server-using-an-authorization-config-file Blog + Examples: https://kubernetes.io/blog/2024/04/26/multi-webhook-and-modular-authorization-made-much-easier/ KEP: https://github.com/kubernetes/enhancements/tree/master/keps/sig-auth/3221-structured-authorization-configuration I tested this all the way back to k8s v1.29 when AuthorizationConfiguration was first introduced as an alpha feature, although v1.29 required some additional workarounds with `kubeadm_patches`, which I included in example comments. I also included some example comments with CEL expressions that allowed me to configure webhook authorizers without hitting kubeadm 1.29+ issues that block cluster creation and upgrades such as this one: https://github.com/kubernetes/cloud-provider-openstack/issues/2575. My workaround configures the webhook to ignore requests from kubeadm and system components, which prevents fatal errors from webhooks that are not available yet, and should be authorized by Node or RBAC anyway. --- .../group_vars/k8s_cluster/k8s-cluster.yml | 2 +- roles/kubernetes/control-plane/tasks/main.yml | 12 ++++ .../templates/kubeadm-config.v1beta3.yaml.j2 | 11 +++- .../templates/kubeadm-config.v1beta4.yaml.j2 | 12 +++- .../kubespray-defaults/defaults/main/main.yml | 57 ++++++++++++++++++- 5 files changed, 90 insertions(+), 4 deletions(-) diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml index bfba61eb894..2be3eae140d 100644 --- a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml @@ -60,7 +60,7 @@ credentials_dir: "{{ inventory_dir }}/credentials" # kube_webhook_token_auth_url: https://... # kube_webhook_token_auth_url_skip_tls_verify: false -## For webhook authorization, authorization_modes must include Webhook +## For webhook authorization, authorization_modes must include Webhook or kube_apiserver_authorization_config_authorizers must configure a type: Webhook # kube_webhook_authorization: false # kube_webhook_authorization_url: https://... # kube_webhook_authorization_url_skip_tls_verify: false diff --git a/roles/kubernetes/control-plane/tasks/main.yml b/roles/kubernetes/control-plane/tasks/main.yml index 055e8155443..d9647150c44 100644 --- a/roles/kubernetes/control-plane/tasks/main.yml +++ b/roles/kubernetes/control-plane/tasks/main.yml @@ -18,6 +18,18 @@ mode: "0640" when: kube_webhook_authorization | default(false) +- name: Create structured AuthorizationConfiguration file + copy: + content: "{{ authz_config | to_nice_yaml(indent=2, sort_keys=false) }}" + dest: "{{ kube_config_dir }}/apiserver-authorization-config.yaml" + mode: "0640" + vars: + authz_config: + apiVersion: apiserver.config.k8s.io/{{ 'v1alpha1' if kube_version is version('v1.30.0', '<') else 'v1beta1' if kube_version is version('v1.32.0', '<') else 'v1' }} + kind: AuthorizationConfiguration + authorizers: "{{ kube_apiserver_authorization_config_authorizers }}" + when: kube_apiserver_use_authorization_config_file + - name: Create kube-scheduler config template: src: kubescheduler-config.yaml.j2 diff --git a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 index 123a68ad579..067100185b0 100644 --- a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 +++ b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 @@ -126,7 +126,11 @@ apiServer: {% if kube_api_anonymous_auth is defined %} anonymous-auth: "{{ kube_api_anonymous_auth }}" {% endif %} +{% if kube_apiserver_use_authorization_config_file %} + authorization-config: "{{ kube_config_dir }}/apiserver-authorization-config.yaml" +{% else %} authorization-mode: {{ authorization_modes | join(',') }} +{% endif %} bind-address: {{ kube_apiserver_bind_address }} {% if kube_apiserver_enable_admission_plugins | length > 0 %} enable-admission-plugins: {{ kube_apiserver_enable_admission_plugins | join(',') }} @@ -176,7 +180,7 @@ apiServer: {% if kube_webhook_token_auth | default(false) %} authentication-token-webhook-config-file: {{ kube_config_dir }}/webhook-token-auth-config.yaml {% endif %} -{% if kube_webhook_authorization | default(false) %} +{% if kube_webhook_authorization and not kube_apiserver_use_authorization_config_file %} authorization-webhook-config-file: {{ kube_config_dir }}/webhook-authorization-config.yaml {% endif %} {% if kube_encrypt_secret_data %} @@ -243,6 +247,11 @@ apiServer: hostPath: {{ kube_config_dir }}/webhook-authorization-config.yaml mountPath: {{ kube_config_dir }}/webhook-authorization-config.yaml {% endif %} +{% if kube_apiserver_use_authorization_config_file %} + - name: authorization-config + hostPath: {{ kube_config_dir }}/apiserver-authorization-config.yaml + mountPath: {{ kube_config_dir }}/apiserver-authorization-config.yaml +{% endif %} {% if kubernetes_audit or kubernetes_audit_webhook %} - name: {{ audit_policy_name }} hostPath: {{ audit_policy_hostpath }} diff --git a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta4.yaml.j2 b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta4.yaml.j2 index be03b489f54..2646c4c0d96 100644 --- a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta4.yaml.j2 +++ b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta4.yaml.j2 @@ -142,8 +142,13 @@ apiServer: - name: anonymous-auth value: "{{ kube_api_anonymous_auth }}" {% endif %} +{% if kube_apiserver_use_authorization_config_file %} + - name: authorization-config + value: "{{ kube_config_dir }}/apiserver-authorization-config.yaml" +{% else %} - name: authorization-mode value: "{{ authorization_modes | join(',') }}" +{% endif %} - name: bind-address value: "{{ kube_apiserver_bind_address }}" {% if kube_apiserver_enable_admission_plugins | length > 0 %} @@ -212,7 +217,7 @@ apiServer: - name: authentication-token-webhook-config-file value: "{{ kube_config_dir }}/webhook-token-auth-config.yaml" {% endif %} -{% if kube_webhook_authorization | default(false) %} +{% if kube_webhook_authorization and not kube_apiserver_use_authorization_config_file %} - name: authorization-webhook-config-file value: "{{ kube_config_dir }}/webhook-authorization-config.yaml" {% endif %} @@ -299,6 +304,11 @@ apiServer: hostPath: {{ kube_config_dir }}/webhook-authorization-config.yaml mountPath: {{ kube_config_dir }}/webhook-authorization-config.yaml {% endif %} +{% if kube_apiserver_use_authorization_config_file %} + - name: authorization-config + hostPath: {{ kube_config_dir }}/apiserver-authorization-config.yaml + mountPath: {{ kube_config_dir }}/apiserver-authorization-config.yaml +{% endif %} {% if kubernetes_audit or kubernetes_audit_webhook %} - name: {{ audit_policy_name }} hostPath: {{ audit_policy_hostpath }} diff --git a/roles/kubespray-defaults/defaults/main/main.yml b/roles/kubespray-defaults/defaults/main/main.yml index 31ef21f768c..36e3ce60b73 100644 --- a/roles/kubespray-defaults/defaults/main/main.yml +++ b/roles/kubespray-defaults/defaults/main/main.yml @@ -487,7 +487,62 @@ external_hcloud_cloud: ## the k8s cluster. Only 'AlwaysAllow', 'AlwaysDeny', 'Node' and ## 'RBAC' modes are tested. Order is important. authorization_modes: ['Node', 'RBAC'] -rbac_enabled: "{{ 'RBAC' in authorization_modes }}" + +## Structured authorization config +## Structured AuthorizationConfiguration is a new feature in Kubernetes v1.29+ (GA in v1.32) that configures the API server's authorization modes with a structured configuration file. +## AuthorizationConfiguration files offer features not available with the `--authorization-mode` flag, although Kubespray supports both methods and authorization-mode remains the default for now. +## Note: Because the `--authorization-config` and `--authorization-mode` flags are mutually exclusive, the `authorization_modes` ansible variable is ignored when `kube_apiserver_use_authorization_config_file` is set to true. The two features cannot be used at the same time. +## Docs: https://kubernetes.io/docs/reference/access-authn-authz/authorization/#configuring-the-api-server-using-an-authorization-config-file +## Examples: https://kubernetes.io/blog/2024/04/26/multi-webhook-and-modular-authorization-made-much-easier/ +## KEP: https://github.com/kubernetes/enhancements/tree/master/keps/sig-auth/3221-structured-authorization-configuration +kube_apiserver_use_authorization_config_file: false +kube_apiserver_authorization_config_authorizers: +- type: Node + name: node +- type: RBAC + name: rbac +## Example for use with kube_webhook_authorization: true +# - type: Webhook +# name: webhook +# webhook: +# connectionInfo: +# type: KubeConfigFile +# kubeConfigFile: "{{ kube_config_dir }}/webhook-authorization-config.yaml" +# subjectAccessReviewVersion: v1beta1 +# matchConditionSubjectAccessReviewVersion: v1 +# timeout: 3s +# failurePolicy: NoOpinion +# matchConditions: +# # Documentation on CEL: https://kubernetes.io/docs/reference/using-api/cel/ +# # only send resource requests to the webhook +# - expression: has(request.resourceAttributes) +# # Don't intercept requests from kube-system service accounts +# - expression: "!('system:serviceaccounts:kube-system' in request.groups)" +# ## Below expressions avoid issues with kubeadm init and other system components that should be authorized by Node and RBAC +# # Don't process node and bootstrap token requests with the webhook +# - expression: "!('system:nodes' in request.groups)" +# - expression: "!('system:bootstrappers' in request.groups)" +# - expression: "!('system:bootstrappers:kubeadm:default-node-token' in request.groups)" +# # Don't process kubeadm requests with the webhook +# - expression: "!('kubeadm:cluster-admins' in request.groups)" +# - expression: "!('system:masters' in request.groups)" + +## Two workarounds are required to use AuthorizationConfiguration with kubeadm v1.29.x: +## 1. Enable the StructuredAuthorizationConfiguration feature gate: +# kube_apiserver_feature_gates: +# - StructuredAuthorizationConfiguration=true +## 2. Use the following kubeadm_patches to remove defaulted authorization-mode flags (Workaround for a kubeadm defaulting bug on v1.29.x. fixed in 1.30+ via: https://github.com/kubernetes/kubernetes/pull/123654) +# kubeadm_patches: +# - target: kube-apiserver +# type: strategic +# patch: +# spec: +# containers: +# - name: kube-apiserver +# $deleteFromPrimitiveList/command: +# - --authorization-mode=Node,RBAC + +rbac_enabled: "{{ ('RBAC' in authorization_modes and not kube_apiserver_use_authorization_config_file) or (kube_apiserver_use_authorization_config_file and kube_apiserver_authorization_config_authorizers | selectattr('type', 'equalto', 'RBAC') | list | length > 0) }}" # When enabled, API bearer tokens (including service account tokens) can be used to authenticate to the kubelet's HTTPS endpoint kubelet_authentication_token_webhook: true