Remove reliance on hiveclient set which in-turn relies on openshift i…

…nstaller (#2806) * chore(hive): Replace hive clientset with controller-runtime client * chore(hive): Replace clientset with controller-runtime * chore(vendor): Tidy dependencies up * chore(hive): Remove vendored clientset
Azure · Jun 2, 2023 · 386753a · 386753a
1 parent 8751724
commit 386753a
Show file tree

Hide file tree

Showing 74 changed files with 70 additions and 7,629 deletions.
diff --git a/go.mod b/go.mod
@@ -44,7 +44,6 @@ require (
 	github.com/openshift/api v3.9.1-0.20191111211345-a27ff30ebf09+incompatible
 	github.com/openshift/client-go v0.0.0-20220525160904-9e1acff93e4a
 	github.com/openshift/console-operator v0.0.0-20220407014945-45d37e70e0c2
-	github.com/openshift/hive v1.1.16
 	github.com/openshift/hive/apis v0.0.0
 	github.com/openshift/installer v0.16.1
 	github.com/openshift/library-go v0.0.0-20220525173854-9b950a41acdc
@@ -223,6 +222,7 @@ require (
 	github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace // indirect
 	github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980 // indirect
 	github.com/stoewer/go-strcase v1.2.0 // indirect
+	github.com/stretchr/objx v0.3.0 // indirect
 	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
 	github.com/ulikunitz/xz v0.5.10 // indirect
 	github.com/vbatts/tar-split v0.11.2 // indirect
@@ -235,6 +235,7 @@ require (
 	go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 // indirect
 	go.opencensus.io v0.23.0 // indirect
 	go.starlark.net v0.0.0-20220328144851-d1966c6b9fcd // indirect
+	go.uber.org/atomic v1.8.0 // indirect
 	golang.org/x/mod v0.8.0 // indirect
 	golang.org/x/sys v0.8.0 // indirect
 	golang.org/x/term v0.8.0 // indirect

diff --git a/go.sum b/go.sum
@@ -1285,8 +1285,6 @@ github.com/openshift/cluster-api-provider-ovirt v0.1.1-0.20211215231458-35ce9aaf
 github.com/openshift/cluster-autoscaler-operator v0.0.0-20190521201101-62768a6ba480/go.mod h1:/XmV44Fh28Vo3Ye93qFrxAbcFJ/Uy+7LPD+jGjmfJYc=
 github.com/openshift/console-operator v0.0.0-20220318130441-e44516b9c315 h1:zmwv8TgbOgZ5QoaPhLdOivqg706Z+VyuPs703jNMdrE=
 github.com/openshift/console-operator v0.0.0-20220318130441-e44516b9c315/go.mod h1:jCX07P5qFcuJrzd0xO5caxLjvSscehiaq6We/hGlcW8=
-github.com/openshift/hive v1.1.17-0.20220719141355-c63c9b0281d8 h1:qcy/cICI6s0CW86+VktQMpdM3u/6TgFzAaWBVRT8Yg8=
-github.com/openshift/hive v1.1.17-0.20220719141355-c63c9b0281d8/go.mod h1:R9ykqkQdLwAv+Zw244+qPGsqv85Htuy78NXdqQQGFSs=
 github.com/openshift/hive/apis v0.0.0-20220719141355-c63c9b0281d8 h1:7e4sMDIstjEKW6SmPv8VhusDaYinDBrspd1M7ybIHC8=
 github.com/openshift/hive/apis v0.0.0-20220719141355-c63c9b0281d8/go.mod h1:XWo9dsulk75E9xkfxS/GNpJrL5UHgn3wuSyPeO39NME=
 github.com/openshift/library-go v0.0.0-20220303081124-fb4e7a2872f0 h1:hiwAdZ5ishMe4qtUejv+CuBWra18cjZMHVFlVPOZnw0=
@@ -1544,6 +1542,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As=
+github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -1708,6 +1707,7 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/atomic v1.8.0 h1:CUhrE4N1rqSE6FM9ecihEjRkLQu8cDfgDyoOs83mEY4=
+go.uber.org/atomic v1.8.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
 go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA=

diff --git a/pkg/hive/manager.go b/pkg/hive/manager.go
@@ -8,14 +8,14 @@ import (
 	"fmt"
 
 	hivev1 "github.com/openshift/hive/apis/hive/v1"
-	hiveclient "github.com/openshift/hive/pkg/client/clientset/versioned"
 	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/util/retry"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/Azure/ARO-RP/pkg/api"
 	"github.com/Azure/ARO-RP/pkg/env"
@@ -46,7 +46,7 @@ type clusterManager struct {
 	log *logrus.Entry
 	env env.Core
 
-	hiveClientset hiveclient.Interface
+	hiveClientset client.Client
 	kubernetescli kubernetes.Interface
 
 	dh dynamichelper.Interface
@@ -81,7 +81,7 @@ func NewFromEnv(ctx context.Context, log *logrus.Entry, env env.Interface) (Clus
 // It MUST NOT take cluster or subscription document as values
 // in these structs can be change during the lifetime of the cluster manager.
 func NewFromConfig(log *logrus.Entry, _env env.Core, restConfig *rest.Config) (ClusterManager, error) {
-	hiveClientset, err := hiveclient.NewForConfig(restConfig)
+	hiveClientset, err := client.New(restConfig, client.Options{})
 	if err != nil {
 		return nil, err
 	}
@@ -186,7 +186,7 @@ func (hr *clusterManager) IsClusterDeploymentReady(ctx context.Context, doc *api
 }
 
 func (hr *clusterManager) IsClusterInstallationComplete(ctx context.Context, doc *api.OpenShiftClusterDocument) (bool, error) {
-	cd, err := hr.hiveClientset.HiveV1().ClusterDeployments(doc.OpenShiftCluster.Properties.HiveProfile.Namespace).Get(ctx, ClusterDeploymentName, metav1.GetOptions{})
+	cd, err := hr.GetClusterDeployment(ctx, doc)
 	if err != nil {
 		return false, err
 	}
@@ -210,12 +210,21 @@ func (hr *clusterManager) IsClusterInstallationComplete(ctx context.Context, doc
 }
 
 func (hr *clusterManager) GetClusterDeployment(ctx context.Context, doc *api.OpenShiftClusterDocument) (*hivev1.ClusterDeployment, error) {
-	return hr.hiveClientset.HiveV1().ClusterDeployments(doc.OpenShiftCluster.Properties.HiveProfile.Namespace).Get(ctx, ClusterDeploymentName, metav1.GetOptions{})
+	cd := &hivev1.ClusterDeployment{}
+	err := hr.hiveClientset.Get(ctx, client.ObjectKey{
+		Namespace: doc.OpenShiftCluster.Properties.HiveProfile.Namespace,
+		Name:      ClusterDeploymentName,
+	}, cd)
+	if err != nil {
+		return nil, err
+	}
+
+	return cd, nil
 }
 
 func (hr *clusterManager) ResetCorrelationData(ctx context.Context, doc *api.OpenShiftClusterDocument) error {
 	return retry.RetryOnConflict(retry.DefaultRetry, func() error {
-		cd, err := hr.hiveClientset.HiveV1().ClusterDeployments(doc.OpenShiftCluster.Properties.HiveProfile.Namespace).Get(ctx, ClusterDeploymentName, metav1.GetOptions{})
+		cd, err := hr.GetClusterDeployment(ctx, doc)
 		if err != nil {
 			return err
 		}
@@ -225,7 +234,6 @@ func (hr *clusterManager) ResetCorrelationData(ctx context.Context, doc *api.Ope
 			return err
 		}
 
-		_, err = hr.hiveClientset.HiveV1().ClusterDeployments(doc.OpenShiftCluster.Properties.HiveProfile.Namespace).Update(ctx, cd, metav1.UpdateOptions{})
-		return err
+		return hr.hiveClientset.Update(ctx, cd)
 	})
 }
diff --git a/pkg/hive/manager_test.go b/pkg/hive/manager_test.go
@@ -9,17 +9,17 @@ import (
 	"testing"
 
 	hivev1 "github.com/openshift/hive/apis/hive/v1"
-	hivefake "github.com/openshift/hive/pkg/client/clientset/versioned/fake"
 	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kruntime "k8s.io/apimachinery/pkg/runtime"
 	kubernetesfake "k8s.io/client-go/kubernetes/fake"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 
 	"github.com/Azure/ARO-RP/pkg/api"
 	"github.com/Azure/ARO-RP/pkg/util/cmp"
 	"github.com/Azure/ARO-RP/pkg/util/uuid"
-	"github.com/Azure/ARO-RP/pkg/util/uuid/fake"
+	uuidfake "github.com/Azure/ARO-RP/pkg/util/uuid/fake"
 	utilerror "github.com/Azure/ARO-RP/test/util/error"
 )
 
@@ -148,12 +148,12 @@ func TestIsClusterDeploymentReady(t *testing.T) {
 		},
 	} {
 		t.Run(tt.name, func(t *testing.T) {
-			fakeClientset := hivefake.NewSimpleClientset()
+			fakeClientBuilder := fake.NewClientBuilder()
 			if tt.cd != nil {
-				fakeClientset.Tracker().Add(tt.cd)
+				fakeClientBuilder.WithRuntimeObjects(tt.cd)
 			}
 			c := clusterManager{
-				hiveClientset: fakeClientset,
+				hiveClientset: fakeClientBuilder.Build(),
 				log:           logrus.NewEntry(logrus.StandardLogger()),
 			}
 
@@ -248,12 +248,12 @@ func TestIsClusterInstallationComplete(t *testing.T) {
 		},
 	} {
 		t.Run(tt.name, func(t *testing.T) {
-			fakeClientset := hivefake.NewSimpleClientset()
+			fakeClientBuilder := fake.NewClientBuilder()
 			if tt.cd != nil {
-				fakeClientset.Tracker().Add(tt.cd)
+				fakeClientBuilder = fakeClientBuilder.WithRuntimeObjects(tt.cd)
 			}
 			c := clusterManager{
-				hiveClientset: fakeClientset,
+				hiveClientset: fakeClientBuilder.Build(),
 				log:           logrus.NewEntry(logrus.StandardLogger()),
 			}
 
@@ -308,19 +308,19 @@ func TestResetCorrelationData(t *testing.T) {
 		},
 	} {
 		t.Run(tt.name, func(t *testing.T) {
-			fakeClientset := hivefake.NewSimpleClientset()
+			fakeClientBuilder := fake.NewClientBuilder()
 			if tt.cd != nil {
-				fakeClientset.Tracker().Add(tt.cd)
+				fakeClientBuilder = fakeClientBuilder.WithRuntimeObjects(tt.cd)
 			}
 			c := clusterManager{
-				hiveClientset: fakeClientset,
+				hiveClientset: fakeClientBuilder.Build(),
 			}
 
 			err := c.ResetCorrelationData(context.Background(), doc)
 			utilerror.AssertErrorMessage(t, err, tt.wantErr)
 
 			if err == nil {
-				cd, err := c.hiveClientset.HiveV1().ClusterDeployments(fakeNamespace).Get(context.Background(), ClusterDeploymentName, metav1.GetOptions{})
+				cd, err := c.GetClusterDeployment(context.Background(), doc)
 				if err != nil {
 					t.Fatal(err)
 				}
@@ -371,7 +371,7 @@ func TestCreateNamespace(t *testing.T) {
 			}
 
 			if tc.useFakeGenerator {
-				uuid.DefaultGenerator = fake.NewGenerator(tc.nsNames)
+				uuid.DefaultGenerator = uuidfake.NewGenerator(tc.nsNames)
 			}
 
 			ns, err := c.CreateNamespace(context.Background())
@@ -419,12 +419,12 @@ func TestGetClusterDeployment(t *testing.T) {
 		{name: "cd does not exist err returned", wantErr: `clusterdeployments.hive.openshift.io "cluster" not found`},
 	} {
 		t.Run(tt.name, func(t *testing.T) {
-			fakeClientset := hivefake.NewSimpleClientset()
+			fakeClientBuilder := fake.NewClientBuilder()
 			if tt.wantErr == "" {
-				_ = fakeClientset.Tracker().Add(cd)
+				fakeClientBuilder = fakeClientBuilder.WithRuntimeObjects(cd)
 			}
 			c := clusterManager{
-				hiveClientset: fakeClientset,
+				hiveClientset: fakeClientBuilder.Build(),
 				log:           logrus.NewEntry(logrus.StandardLogger()),
 			}
 

diff --git a/pkg/monitor/cluster/cluster.go b/pkg/monitor/cluster/cluster.go
@@ -11,13 +11,13 @@ import (
 	configv1 "github.com/openshift/api/config/v1"
 	configclient "github.com/openshift/client-go/config/clientset/versioned"
 	machineclient "github.com/openshift/client-go/machine/clientset/versioned"
-	hiveclient "github.com/openshift/hive/pkg/client/clientset/versioned"
 	mcoclient "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned"
 	"github.com/sirupsen/logrus"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/Azure/ARO-RP/pkg/api"
 	"github.com/Azure/ARO-RP/pkg/metrics"
@@ -41,7 +41,7 @@ type Monitor struct {
 	m          metrics.Emitter
 	arocli     aroclient.Interface
 
-	hiveclientset hiveclient.Interface
+	hiveclientset client.Client
 
 	// access below only via the helper functions in cache.go
 	cache struct {
@@ -91,10 +91,10 @@ func NewMonitor(log *logrus.Entry, restConfig *rest.Config, oc *api.OpenShiftClu
 		return nil, err
 	}
 
-	var hiveclientset hiveclient.Interface
+	var hiveclientset client.Client
 	if hiveRestConfig != nil {
 		var err error
-		hiveclientset, err = hiveclient.NewForConfig(hiveRestConfig)
+		hiveclientset, err = client.New(hiveRestConfig, client.Options{})
 		if err != nil {
 			// TODO(hive): Update to fail once we have Hive everywhere in prod and dev
 			log.Error(err)

diff --git a/pkg/monitor/cluster/hiveregistrationstatus.go b/pkg/monitor/cluster/hiveregistrationstatus.go
@@ -9,7 +9,7 @@ import (
 
 	hivev1 "github.com/openshift/hive/apis/hive/v1"
 	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/Azure/ARO-RP/pkg/hive"
 )
@@ -42,7 +42,11 @@ func (mon *Monitor) emitHiveRegistrationStatus(ctx context.Context) error {
 }
 
 func (mon *Monitor) retrieveClusterDeployment(ctx context.Context) (*hivev1.ClusterDeployment, error) {
-	cd, err := mon.hiveclientset.HiveV1().ClusterDeployments(mon.oc.Properties.HiveProfile.Namespace).Get(ctx, hive.ClusterDeploymentName, metav1.GetOptions{})
+	cd := &hivev1.ClusterDeployment{}
+	err := mon.hiveclientset.Get(ctx, client.ObjectKey{
+		Namespace: mon.oc.Properties.HiveProfile.Namespace,
+		Name:      hive.ClusterDeploymentName,
+	}, cd)
 	if err != nil {
 		return nil, err
 	}

diff --git a/pkg/monitor/cluster/hiveregistrationstatus_test.go b/pkg/monitor/cluster/hiveregistrationstatus_test.go
@@ -10,14 +10,14 @@ import (
 
 	"github.com/golang/mock/gomock"
 	hivev1 "github.com/openshift/hive/apis/hive/v1"
-	hiveclient "github.com/openshift/hive/pkg/client/clientset/versioned"
-	hivefake "github.com/openshift/hive/pkg/client/clientset/versioned/fake"
 	"github.com/sirupsen/logrus"
 	"github.com/sirupsen/logrus/hooks/test"
 	"github.com/stretchr/testify/assert"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kruntime "k8s.io/apimachinery/pkg/runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake"
 
 	"github.com/Azure/ARO-RP/pkg/api"
 	"github.com/Azure/ARO-RP/pkg/hive"
@@ -87,13 +87,13 @@ func TestEmitHiveRegistrationStatus(t *testing.T) {
 		},
 	} {
 		t.Run(tt.name, func(t *testing.T) {
-			var hiveclient hiveclient.Interface
+			var hiveclient client.Client
 			if tt.withClient {
-				fakeclient := hivefake.NewSimpleClientset()
+				fakeclient := fakeclient.NewClientBuilder()
 				if tt.cd != nil {
-					fakeclient.Tracker().Add(tt.cd)
+					fakeclient = fakeclient.WithRuntimeObjects(tt.cd)
 				}
-				hiveclient = fakeclient
+				hiveclient = fakeclient.Build()
 			}
 
 			logger, hook := test.NewNullLogger()
@@ -136,13 +136,13 @@ func TestRetrieveClusterDeployment(t *testing.T) {
 		wantErr: fmt.Sprintf("clusterdeployments.hive.openshift.io %q not found", hive.ClusterDeploymentName),
 	}} {
 		t.Run(tt.name, func(t *testing.T) {
-			fakeclient := hivefake.NewSimpleClientset()
+			fakeclient := fakeclient.NewClientBuilder()
 			if tt.cd != nil {
-				fakeclient.Tracker().Add(tt.cd)
+				fakeclient = fakeclient.WithRuntimeObjects(tt.cd)
 			}
 
 			mon := &Monitor{
-				hiveclientset: fakeclient,
+				hiveclientset: fakeclient.Build(),
 				oc: &api.OpenShiftCluster{
 					Name: "testcluster",
 					Properties: api.OpenShiftClusterProperties{

diff --git a/test/e2e/hive.go b/test/e2e/hive.go
@@ -12,7 +12,8 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	hivev1 "github.com/openshift/hive/apis/hive/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/Azure/ARO-RP/pkg/api/admin"
 	"github.com/Azure/ARO-RP/pkg/hive"
@@ -44,7 +45,11 @@ var _ = Describe("Hive-managed ARO cluster", func() {
 
 	It("has been properly created/adopted by Hive", func(ctx context.Context) {
 		By("verifying that a corresponding ClusterDeployment object exists in the expected namespace in the Hive cluster")
-		cd, err := clients.Hive.HiveV1().ClusterDeployments(adminAPICluster.Properties.HiveProfile.Namespace).Get(ctx, hive.ClusterDeploymentName, metav1.GetOptions{})
+		cd := &hivev1.ClusterDeployment{}
+		err := clients.Hive.Get(ctx, client.ObjectKey{
+			Namespace: adminAPICluster.Properties.HiveProfile.Namespace,
+			Name:      hive.ClusterDeploymentName,
+		}, cd)
 		Expect(err).NotTo(HaveOccurred())
 
 		By("verifying that the ClusterDeployment object has the expected name and labels")