Merge pull request #115 from spinkube/e2e

adds basic setup for e2e testing

.github/workflows/e2e.yaml

@@ -0,0 +1,18 @@
+name: e2e
+
+on:
+  pull_request:
+    branches: [main]
+
+jobs:
+  e2e:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.22.x"
+          cache: true
+      - name: run e2e
+        run: go test ./e2e -v

Makefile

@@ -291,3 +291,7 @@ $(ENVTEST): $(LOCALBIN)
 helmify: $(HELMIFY) ## Download helmify locally if necessary.
 $(HELMIFY): $(LOCALBIN)
 	@test -s $(LOCALBIN)/helmify || GOBIN=$(LOCALBIN) go install github.com/arttor/helmify/cmd/helmify@$(HELMIFY_VESRION)
+
+.PHONY: e2e
+e2e: ## Run e2e tests
+	go test -v ./e2e/...

e2e/README.md

@@ -0,0 +1,9 @@
+# E2e Testing
+
+This e2e test suite leverages the [Kubernetes e2e-framework](https://github.com/kubernetes-sigs/e2e-framework) project for writing and running e2e tests for the spin operator project.
+
+To run tests:
+
+```console
+go test ./e2e -v
+```

e2e/crd_installed_test.go

@@ -0,0 +1,49 @@
+package e2e
+
+import (
+	"context"
+	"testing"
+
+	apiextensionsV1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"sigs.k8s.io/e2e-framework/pkg/features"
+)
+
+func TestCRDInstalled(t *testing.T) {
+	crdInstalledFeature := features.New("crd installed").
+		Assess("spinapp crd installed", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			client := cfg.Client()
+			if err := apiextensionsV1.AddToScheme(client.Resources().GetScheme()); err != nil {
+				t.Fatalf("failed to register the v1 API extension types with Kuberenets scheme: %s", err)
+			}
+			name := "spinapps.core.spinoperator.dev"
+			var crd apiextensionsV1.CustomResourceDefinition
+			if err := client.Resources().Get(ctx, name, "", &crd); err != nil {
+				t.Fatalf("SpinApp CRD not found: %s", err)
+			}
+
+			if crd.Spec.Group != "core.spinoperator.dev" {
+				t.Fatalf("SpinApp CRD has unexpected group: %s", crd.Spec.Group)
+			}
+			return ctx
+
+		}).
+		Assess("spinappexecutor crd installed", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			client := cfg.Client()
+			if err := apiextensionsV1.AddToScheme(client.Resources().GetScheme()); err != nil {
+				t.Fatalf("failed to register the v1 API extension types with Kuberenets scheme: %s", err)
+			}
+
+			name := "spinappexecutors.core.spinoperator.dev"
+			var crd apiextensionsV1.CustomResourceDefinition
+			if err := client.Resources().Get(ctx, name, "", &crd); err != nil {
+				t.Fatalf("SpinApp CRD not found: %s", err)
+			}
+
+			if crd.Spec.Group != "core.spinoperator.dev" {
+				t.Fatalf("SpinAppExecutor CRD has unexpected group: %s", crd.Spec.Group)
+			}
+			return ctx
+		}).Feature()
+	testEnv.Test(t, crdInstalledFeature)
+}

e2e/default_test.go

@@ -0,0 +1,137 @@
+package e2e
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	v1 "k8s.io/api/core/v1"
+	nodev1 "k8s.io/api/node/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/e2e-framework/klient"
+	"sigs.k8s.io/e2e-framework/klient/k8s"
+	"sigs.k8s.io/e2e-framework/klient/wait"
+	"sigs.k8s.io/e2e-framework/klient/wait/conditions"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"sigs.k8s.io/e2e-framework/pkg/features"
+
+	spinapps_v1alpha1 "github.com/spinkube/spin-operator/api/v1alpha1"
+)
+
+var runtimeClassName = "wasmtime-spin-v2"
+
+// TestDefaultSetup is a test that checks that the minimal setup works
+// with the containerd wasm shim runtime as the default runtime.
+func TestDefaultSetup(t *testing.T) {
+	var client klient.Client
+
+	helloWorldImage := "ghcr.io/deislabs/containerd-wasm-shims/examples/spin-rust-hello:latest"
+	testSpinAppName := "test-spinapp"
+
+	defaultTest := features.New("default and most minimal setup").
+		Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+
+			client = cfg.Client()
+
+			if err := spinapps_v1alpha1.AddToScheme(client.Resources(testNamespace).GetScheme()); err != nil {
+				t.Fatalf("failed to register the spinapps_v1alpha1 types with Kuberenets scheme: %s", err)
+			}
+
+			runtimeClass := &nodev1.RuntimeClass{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: runtimeClassName,
+				},
+				Handler: "spin",
+			}
+
+			if err := client.Resources().Create(ctx, runtimeClass); err != nil {
+				t.Fatalf("Failed to create runtimeclass: %s", err)
+			}
+
+			return ctx
+		}).
+		Assess("spin app custom resource is created", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			testSpinApp := newSpinAppCR(testSpinAppName, helloWorldImage)
+
+			if err := client.Resources().Create(ctx, newContainerdShimExecutor(testNamespace)); err != nil {
+				t.Fatalf("Failed to create spinappexecutor: %s", err)
+			}
+
+			if err := client.Resources().Create(ctx, testSpinApp); err != nil {
+				t.Fatalf("Failed to create spinapp: %s", err)
+			}
+			// wait for spinapp to be created
+			if err := wait.For(
+				conditions.New(client.Resources()).ResourceMatch(testSpinApp, func(object k8s.Object) bool {
+					return true
+				}),
+				wait.WithTimeout(3*time.Minute),
+				wait.WithInterval(30*time.Second),
+			); err != nil {
+				t.Fatal(err)
+			}
+
+			return ctx
+		}).
+		Assess("spin app deployment and service are available", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+
+			// wait for deployment to be ready
+			if err := wait.For(
+				conditions.New(client.Resources()).DeploymentAvailable(testSpinAppName, testNamespace),
+				wait.WithTimeout(3*time.Minute),
+				wait.WithInterval(30*time.Second),
+			); err != nil {
+				t.Fatal(err)
+			}
+
+			svc := &v1.ServiceList{
+				Items: []v1.Service{
+					{ObjectMeta: metav1.ObjectMeta{Name: testSpinAppName, Namespace: testNamespace}},
+				},
+			}
+
+			if err := wait.For(
+				conditions.New(client.Resources()).ResourcesFound(svc),
+				wait.WithTimeout(3*time.Minute),
+				wait.WithInterval(30*time.Second),
+			); err != nil {
+				t.Fatal(err)
+			}
+			return ctx
+		}).Feature()
+	testEnv.Test(t, defaultTest)
+}
+
+func newSpinAppCR(name, image string) *spinapps_v1alpha1.SpinApp {
+	var testSpinApp = &spinapps_v1alpha1.SpinApp{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: testNamespace,
+		},
+		Spec: spinapps_v1alpha1.SpinAppSpec{
+			Replicas: 1,
+			Image:    image,
+			Executor: "containerd-shim-spin",
+		},
+	}
+
+	return testSpinApp
+
+}
+
+func newContainerdShimExecutor(namespace string) *spinapps_v1alpha1.SpinAppExecutor {
+	var testSpinAppExecutor = &spinapps_v1alpha1.SpinAppExecutor{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "containerd-shim-spin",
+			Namespace: namespace,
+		},
+		Spec: spinapps_v1alpha1.SpinAppExecutorSpec{
+			CreateDeployment: true,
+			DeploymentConfig: &spinapps_v1alpha1.ExecutorDeploymentConfig{
+				RuntimeClassName: runtimeClassName,
+			},
+		},
+	}
+
+	return testSpinAppExecutor
+}

e2e/k3d_provider.go

@@ -0,0 +1,123 @@
+package e2e
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"k8s.io/client-go/rest"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/e2e-framework/klient/conf"
+	"sigs.k8s.io/e2e-framework/support/utils"
+)
+
+const k3dImage = "ghcr.io/deislabs/containerd-wasm-shims/examples/k3d:v0.11.0"
+
+var k3dBin = "k3d"
+
+type Cluster struct {
+	name        string
+	kubecfgFile string
+	restConfig  *rest.Config
+}
+
+func (c *Cluster) Create(context.Context, string) (string, error) {
+	if err := findOrInstallK3d(); err != nil {
+		panic(err)
+	}
+
+	if _, ok := clusterExists(c.name); ok {
+		klog.V(4).Info("Skipping k3d Cluster creation. Cluster already created ", c.name)
+		return c.GetKubeconfig()
+	}
+
+	command := fmt.Sprintf("%s cluster create %s --image %s -p '8081:80@loadbalancer' --agents 2 --wait", k3dBin, c.name, k3dImage)
+	klog.V(4).Info("Launching:", command)
+	p := utils.RunCommand(command)
+	if p.Err() != nil {
+		outBytes, err := io.ReadAll(p.Out())
+		if err != nil {
+			klog.ErrorS(err, "failed to read data from the k3d create process output due to an error")
+		}
+		return "", fmt.Errorf("k3d: failed to create cluster %q: %w: %s: %s", c.name, p.Err(), p.Result(), string(outBytes))
+	}
+
+	clusters, ok := clusterExists(c.name)
+	if !ok {
+		return "", fmt.Errorf("k3d Cluster.Create: cluster %v still not in 'cluster list' after creation: %v", c.name, clusters)
+	}
+	klog.V(4).Info("k3d cluster available: ", clusters)
+
+	kConfig, err := c.GetKubeconfig()
+	if err != nil {
+		return "", err
+	}
+	return kConfig, c.initKubernetesAccessClients()
+}
+
+func (c *Cluster) Destroy() error {
+	p := utils.RunCommand(fmt.Sprintf(`%s cluster delete %s`, k3dBin, c.name))
+	if p.Err() != nil {
+		return fmt.Errorf("%s cluster delete: %w", k3dBin, p.Err())
+	}
+
+	return nil
+}
+
+func (c *Cluster) GetKubeconfig() (string, error) {
+	kubecfg := fmt.Sprintf("%s-kubecfg", c.name)
+
+	p := utils.RunCommand(fmt.Sprintf(`%s kubeconfig get %s`, k3dBin, c.name))
+
+	if p.Err() != nil {
+		return "", fmt.Errorf("k3d get kubeconfig: %w", p.Err())
+	}
+
+	var stdout bytes.Buffer
+	if _, err := stdout.ReadFrom(p.Out()); err != nil {
+		return "", fmt.Errorf("k3d kubeconfig stdout bytes: %w", err)
+	}
+
+	file, err := os.CreateTemp("", fmt.Sprintf("k3d-cluster-%s", kubecfg))
+	if err != nil {
+		return "", fmt.Errorf("k3d kubeconfig file: %w", err)
+	}
+	defer file.Close()
+
+	c.kubecfgFile = file.Name()
+
+	if n, err := io.Copy(file, &stdout); n == 0 || err != nil {
+		return "", fmt.Errorf("k3d kubecfg file: bytes copied: %d: %w]", n, err)
+	}
+
+	return file.Name(), nil
+}
+
+func (c *Cluster) initKubernetesAccessClients() error {
+	cfg, err := conf.New(c.kubecfgFile)
+	if err != nil {
+		return err
+	}
+	c.restConfig = cfg
+
+	return nil
+}
+
+func findOrInstallK3d() error {
+	_, err := utils.FindOrInstallGoBasedProvider(k3dBin, k3dBin, "github.com/k3d-io/k3d", "v5.6.0")
+	return err
+}
+
+func clusterExists(name string) (string, bool) {
+	clusters := utils.FetchCommandOutput(fmt.Sprintf("%s cluster list --no-headers", k3dBin))
+	for _, c := range strings.Split(clusters, "\n") {
+		cl := strings.Split(c, " ")[0]
+		if cl == name {
+			return clusters, true
+		}
+	}
+	return clusters, false
+}