From 5042e5c02dbb2f263bfdadd00133148c83de60c9 Mon Sep 17 00:00:00 2001 From: peefy Date: Wed, 4 Sep 2024 17:01:31 +0800 Subject: [PATCH 1/7] test: add kcl run tests Signed-off-by: peefy --- config/all.yaml | 11 ++------- config/rbac/role_binding.yaml | 4 ++-- config/rbac/service_account.yaml | 11 ++------- scripts/deploy_test.sh | 40 +++++++++++++++++++++++++++++++- 4 files changed, 45 insertions(+), 21 deletions(-) diff --git a/config/all.yaml b/config/all.yaml index 7fbd88f..60eafa7 100644 --- a/config/all.yaml +++ b/config/all.yaml @@ -63,14 +63,7 @@ spec: apiVersion: v1 kind: ServiceAccount metadata: - labels: - app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: kcl-operator - app.kubernetes.io/instance: controller-manager - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: serviceaccount - app.kubernetes.io/part-of: kcl-operator - name: controller-manager + name: webhook namespace: default --- apiVersion: rbac.authorization.k8s.io/v1 @@ -115,7 +108,7 @@ roleRef: name: webhook-role subjects: - kind: ServiceAccount - name: default + name: webhook namespace: default --- apiVersion: v1 diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml index 768da9b..d4ca7af 100644 --- a/config/rbac/role_binding.yaml +++ b/config/rbac/role_binding.yaml @@ -8,5 +8,5 @@ roleRef: name: webhook-role subjects: - kind: ServiceAccount - name: default - namespace: system + name: webhook + namespace: default diff --git a/config/rbac/service_account.yaml b/config/rbac/service_account.yaml index 7aa7bf3..3619d08 100644 --- a/config/rbac/service_account.yaml +++ b/config/rbac/service_account.yaml @@ -1,12 +1,5 @@ apiVersion: v1 kind: ServiceAccount metadata: - labels: - app.kubernetes.io/name: serviceaccount - app.kubernetes.io/instance: controller-manager - app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: kcl-operator - app.kubernetes.io/part-of: kcl-operator - app.kubernetes.io/managed-by: kustomize - name: controller-manager - namespace: system + name: webhook + namespace: default diff --git a/scripts/deploy_test.sh b/scripts/deploy_test.sh index 2c86b15..a5b44e1 100755 --- a/scripts/deploy_test.sh +++ b/scripts/deploy_test.sh @@ -29,8 +29,46 @@ if [ $ROLLOUT_STATUS -ne 0 ]; then echo -e "${YELLOW}Pod: $pod${NC}" kubectl describe pod/$pod echo -e "${YELLOW}Pod logs:${NC}" - kubectl logs $pod + kubectl logs $pod -c kcl-webhook-init + kubectl logs $pod -c kcl-webhook-server done else echo -e "${GREEN}Deployment rollout successful.${NC}" + echo -e "${YELLOW}Deploying the KCL source...${NC}" + kubectl apply -f- << EOF +apiVersion: krm.kcl.dev/v1alpha1 +kind: KCLRun +metadata: + name: set-annotation +spec: + params: + annotations: + managed-by: kcl-operator + source: | + items = [item | { + metadata.annotations: {"managed-by" = "kcl-operator"} + } for item in option("items")] +EOF + echo -e "${YELLOW}Validating the mutation result by creating a nginx Pod YAML...${NC}" + kubectl apply -f- << EOF +apiVersion: v1 +kind: Pod +metadata: + name: nginx + annotations: + app: nginx +spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 +EOF + echo -e "${YELLOW}Checking if the annotation 'managed-by: kcl-operator' is added to the pod...${NC}" + MANAGED_BY=$(kubectl get po nginx -o yaml | grep kcl-operator) + if [ -n "$MANAGED_BY" ]; then + echo -e "${GREEN}The annotation 'managed-by: kcl-operator' is added to the pod.${NC}" + else + echo -e "${RED}The annotation 'managed-by: kcl-operator' is not added to the pod.${NC}" + fi fi From 32aec04fa9d1173445af8256109c0a63edd9d100 Mon Sep 17 00:00:00 2001 From: peefy Date: Wed, 4 Sep 2024 17:09:15 +0800 Subject: [PATCH 2/7] feat: add mutate webhook config create Signed-off-by: peefy --- cmd/webhook-init/main.go | 2 ++ config/all.yaml | 12 ++++++++++++ config/rbac/role.yaml | 12 ++++++++++++ pkg/webhook/handler/mutation.go | 1 + 4 files changed, 27 insertions(+) diff --git a/cmd/webhook-init/main.go b/cmd/webhook-init/main.go index 10c8dfa..653d530 100644 --- a/cmd/webhook-init/main.go +++ b/cmd/webhook-init/main.go @@ -147,6 +147,8 @@ func WriteFile(filepath string, sCert *bytes.Buffer) error { return nil } +//+kubebuilder:rbac:groups=admissionregistration.k8s.io,resources=mutatingwebhookconfigurations,verbs=get;list;watch;create;update;patch;delete + func createMutationConfig(caCert *bytes.Buffer) { var ( webhookNamespace, _ = os.LookupEnv("WEBHOOK_NAMESPACE") diff --git a/config/all.yaml b/config/all.yaml index 60eafa7..33ba664 100644 --- a/config/all.yaml +++ b/config/all.yaml @@ -71,6 +71,18 @@ kind: ClusterRole metadata: name: webhook-role rules: +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - krm.kcl.dev resources: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 10df539..5ec84b7 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -4,6 +4,18 @@ kind: ClusterRole metadata: name: webhook-role rules: +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - krm.kcl.dev resources: diff --git a/pkg/webhook/handler/mutation.go b/pkg/webhook/handler/mutation.go index 3b75940..0b3aefe 100644 --- a/pkg/webhook/handler/mutation.go +++ b/pkg/webhook/handler/mutation.go @@ -22,6 +22,7 @@ import ( //+kubebuilder:rbac:groups=krm.kcl.dev,resources=kclruns,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=krm.kcl.dev,resources=kclruns/status,verbs=get;update;patch //+kubebuilder:rbac:groups=krm.kcl.dev,resources=kclruns/finalizers,verbs=update +//+kubebuilder:rbac:groups=admissionregistration.k8s.io,resources=mutatingwebhookconfigurations,verbs=get;list;watch;create;update;patch;delete // MutationHandler validates Kubernetes resources using the KCL source. type MutationHandler struct { From 0c96171ac7fe3c13a31182508f0c14be3e8c0dff Mon Sep 17 00:00:00 2001 From: peefy Date: Wed, 4 Sep 2024 17:30:08 +0800 Subject: [PATCH 3/7] test: update rbac resource names Signed-off-by: peefy --- config/all.yaml | 9 +++++---- config/rbac/role_binding.yaml | 6 +++--- config/rbac/service_account.yaml | 2 +- config/webhook/webhook.yaml | 1 + scripts/deploy_test.sh | 3 ++- 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/config/all.yaml b/config/all.yaml index 33ba664..b14ccbf 100644 --- a/config/all.yaml +++ b/config/all.yaml @@ -63,7 +63,7 @@ spec: apiVersion: v1 kind: ServiceAccount metadata: - name: webhook + name: kcl-webhook namespace: default --- apiVersion: rbac.authorization.k8s.io/v1 @@ -113,14 +113,14 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: webhook-rolebinding + name: kcl-webhook-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: webhook-role + name: kcl-webhook-role subjects: - kind: ServiceAccount - name: webhook + name: kcl-webhook namespace: default --- apiVersion: v1 @@ -181,6 +181,7 @@ spec: volumeMounts: - mountPath: /etc/webhook/certs name: webhook-certs + serviceAccountName: kcl-webhook volumes: - emptyDir: {} name: webhook-certs diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml index d4ca7af..a0df8cf 100644 --- a/config/rbac/role_binding.yaml +++ b/config/rbac/role_binding.yaml @@ -1,12 +1,12 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: webhook-rolebinding + name: kcl-webhook-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: webhook-role + name: kcl-webhook-role subjects: - kind: ServiceAccount - name: webhook + name: kcl-webhook namespace: default diff --git a/config/rbac/service_account.yaml b/config/rbac/service_account.yaml index 3619d08..77980b9 100644 --- a/config/rbac/service_account.yaml +++ b/config/rbac/service_account.yaml @@ -1,5 +1,5 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: webhook + name: kcl-webhook namespace: default diff --git a/config/webhook/webhook.yaml b/config/webhook/webhook.yaml index dda55b3..14b2ffa 100644 --- a/config/webhook/webhook.yaml +++ b/config/webhook/webhook.yaml @@ -14,6 +14,7 @@ spec: labels: app: kcl-webhook-server spec: + serviceAccountName: kcl-webhook initContainers: - name: kcl-webhook-init image: kcllang/webhook-init diff --git a/scripts/deploy_test.sh b/scripts/deploy_test.sh index a5b44e1..6922908 100755 --- a/scripts/deploy_test.sh +++ b/scripts/deploy_test.sh @@ -28,8 +28,9 @@ if [ $ROLLOUT_STATUS -ne 0 ]; then for pod in $PODS; do echo -e "${YELLOW}Pod: $pod${NC}" kubectl describe pod/$pod - echo -e "${YELLOW}Pod logs:${NC}" + echo -e "${YELLOW}Pod init container logs:${NC}" kubectl logs $pod -c kcl-webhook-init + echo -e "${YELLOW}Pod main container logs:${NC}" kubectl logs $pod -c kcl-webhook-server done else From b87dbaa0eb4381cf9e1a9a76b8c4e2fe3e124813 Mon Sep 17 00:00:00 2001 From: peefy Date: Wed, 4 Sep 2024 17:55:36 +0800 Subject: [PATCH 4/7] test: update rbac resource names Signed-off-by: peefy --- config/rbac/role.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 5ec84b7..04f05ad 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -2,7 +2,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: webhook-role + name: kcl-webhook-role rules: - apiGroups: - admissionregistration.k8s.io From e225a54f0ab5b3ac189692be1008336333cb5081 Mon Sep 17 00:00:00 2001 From: peefy Date: Thu, 5 Sep 2024 11:38:52 +0800 Subject: [PATCH 5/7] test: add more logs for kcl webhook Signed-off-by: peefy --- .github/workflows/ci.yaml | 3 ++ Makefile | 2 +- cmd/webhook-init/main.go | 55 +++++++++++++++++++++++-------- config/all.yaml | 2 +- config/webhook/kustomization.yaml | 2 +- 5 files changed, 47 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bc52859..f3592a8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -32,5 +32,8 @@ jobs: - name: Project Build & Check run: make + - name: Certs generate tests + run: CERTS_PATH="." go run cmd/webhook-init/main.go + - name: Run e2e tests run: ./scripts/deploy_test.sh diff --git a/Makefile b/Makefile index de38035..4875da0 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,7 @@ help: ## Display this help. .PHONY: manifests manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. - $(CONTROLLER_GEN) rbac:roleName=webhook-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases + $(CONTROLLER_GEN) rbac:roleName=kcl-webhook-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases .PHONY: generate generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. diff --git a/cmd/webhook-init/main.go b/cmd/webhook-init/main.go index 653d530..f51f4c9 100644 --- a/cmd/webhook-init/main.go +++ b/cmd/webhook-init/main.go @@ -26,17 +26,24 @@ import ( "fmt" "math/big" "os" + "path/filepath" "time" - log "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" + + "github.com/sirupsen/logrus" + kwhlogrus "github.com/slok/kubewebhook/v2/pkg/log/logrus" ) func main() { + logrusLogEntry := logrus.NewEntry(logrus.New()) + logrusLogEntry.Logger.SetLevel(logrus.DebugLevel) + logger := kwhlogrus.NewLogrus(logrusLogEntry) + var caPEM, serverCertPEM, serverPrivKeyPEM *bytes.Buffer // CA config ca := &x509.Certificate{ @@ -55,13 +62,15 @@ func main() { // CA private key caPrivKey, err := rsa.GenerateKey(cryptorand.Reader, 4096) if err != nil { - fmt.Println(err) + logger.Errorf("unable to generate CA private key: %v", err) + os.Exit(1) } // Self signed CA certificate caBytes, err := x509.CreateCertificate(cryptorand.Reader, ca, ca, &caPrivKey.PublicKey, caPrivKey) if err != nil { - fmt.Println(err) + logger.Errorf("unable to self signed CA certificate: %v", err) + os.Exit(1) } // PEM encode CA cert @@ -115,21 +124,35 @@ func main() { Bytes: x509.MarshalPKCS1PrivateKey(serverPrivKey), }) - err = os.MkdirAll("/etc/webhook/certs/", 0666) + var ( + certsPath, _ = os.LookupEnv("CERTS_PATH") + ) + if certsPath == "" { + certsPath = "/etc/webhook/certs/" + } + + err = os.MkdirAll(certsPath, 0666) if err != nil { - log.Panic(err) + logger.Errorf("unable to mkdir %s: %v", certsPath, err) + os.Exit(1) } - err = WriteFile("/etc/webhook/certs/tls.crt", serverCertPEM) + err = WriteFile(filepath.Join(certsPath, "tls.crt"), serverCertPEM) if err != nil { - log.Panic(err) + logger.Errorf("unable to generate tls.crt: %v", err) + os.Exit(1) } - err = WriteFile("/etc/webhook/certs/tls.key", serverPrivKeyPEM) + err = WriteFile(filepath.Join(certsPath, "tls.key"), serverPrivKeyPEM) if err != nil { - log.Panic(err) + logger.Errorf("unable to generate tls.key: %v", err) + os.Exit(1) } // Create `MutatingWebhookConfiguration` resources - createMutationConfig(serverCertPEM) + err = createMutationConfig(serverCertPEM) + if err != nil { + logger.Errorf("unable to create `MutatingWebhookConfiguration` resources: %v", err) + os.Exit(1) + } } // WriteFile writes data in the file at the given path @@ -149,7 +172,7 @@ func WriteFile(filepath string, sCert *bytes.Buffer) error { //+kubebuilder:rbac:groups=admissionregistration.k8s.io,resources=mutatingwebhookconfigurations,verbs=get;list;watch;create;update;patch;delete -func createMutationConfig(caCert *bytes.Buffer) { +func createMutationConfig(caCert *bytes.Buffer) error { var ( webhookNamespace, _ = os.LookupEnv("WEBHOOK_NAMESPACE") mutationCfgName, _ = os.LookupEnv("MUTATE_CONFIG") @@ -158,11 +181,12 @@ func createMutationConfig(caCert *bytes.Buffer) { config := ctrl.GetConfigOrDie() kubeClient, err := kubernetes.NewForConfig(config) if err != nil { - panic("failed to set go-client") + return err } path := "/mutate" fail := admissionregistrationv1.Fail + sideEffectClassNone := admissionregistrationv1.SideEffectClassNone mutateconfig := &admissionregistrationv1.MutatingWebhookConfiguration{ ObjectMeta: metav1.ObjectMeta{ @@ -186,11 +210,14 @@ func createMutationConfig(caCert *bytes.Buffer) { Resources: []string{"deployments", "pods"}, }, }}, - FailurePolicy: &fail, + SideEffects: &sideEffectClassNone, + AdmissionReviewVersions: []string{"v1", "v1beta1"}, + FailurePolicy: &fail, }}, } if _, err := kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.Background(), mutateconfig, metav1.CreateOptions{}); err != nil { - panic(err) + return err } + return nil } diff --git a/config/all.yaml b/config/all.yaml index b14ccbf..eb2c71b 100644 --- a/config/all.yaml +++ b/config/all.yaml @@ -69,7 +69,7 @@ metadata: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: webhook-role + name: kcl-webhook-role rules: - apiGroups: - admissionregistration.k8s.io diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml index 8a81591..7ae8994 100644 --- a/config/webhook/kustomization.yaml +++ b/config/webhook/kustomization.yaml @@ -3,5 +3,5 @@ resources: apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: -- name: controller +- name: kcl-webhook-server newName: kcllang/webhook-server From 171525946ec8d5bbb776f2889ab5a786cca24f41 Mon Sep 17 00:00:00 2001 From: peefy Date: Thu, 5 Sep 2024 11:44:54 +0800 Subject: [PATCH 6/7] test: generate certs Signed-off-by: peefy --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f3592a8..66d023a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -33,7 +33,7 @@ jobs: run: make - name: Certs generate tests - run: CERTS_PATH="." go run cmd/webhook-init/main.go + run: CERTS_PATH="." MUTATE_CONFIG="test" WEBHOOK_SERVICE="test" WEBHOOK_NAMESPACE="default" go run cmd/webhook-init/main.go - name: Run e2e tests run: ./scripts/deploy_test.sh From 4010c0e9fdc1fb9c88a29c3ade1011b277519369 Mon Sep 17 00:00:00 2001 From: peefy Date: Thu, 5 Sep 2024 11:50:43 +0800 Subject: [PATCH 7/7] chore: temp remove certs tests Signed-off-by: peefy --- .github/workflows/ci.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 66d023a..bc52859 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -32,8 +32,5 @@ jobs: - name: Project Build & Check run: make - - name: Certs generate tests - run: CERTS_PATH="." MUTATE_CONFIG="test" WEBHOOK_SERVICE="test" WEBHOOK_NAMESPACE="default" go run cmd/webhook-init/main.go - - name: Run e2e tests run: ./scripts/deploy_test.sh