diff --git a/bundle/next/eclipse-che/manifests/che-operator.clusterserviceversion.yaml b/bundle/next/eclipse-che/manifests/che-operator.clusterserviceversion.yaml index f7bef87540..0b61d5e05e 100644 --- a/bundle/next/eclipse-che/manifests/che-operator.clusterserviceversion.yaml +++ b/bundle/next/eclipse-che/manifests/che-operator.clusterserviceversion.yaml @@ -104,7 +104,7 @@ metadata: operators.operatorframework.io/project_layout: go.kubebuilder.io/v3 repository: https://github.com/eclipse-che/che-operator support: Eclipse Foundation - name: eclipse-che.v7.94.0-888.next + name: eclipse-che.v7.94.0-889.next namespace: placeholder spec: apiservicedefinitions: {} @@ -1035,7 +1035,7 @@ spec: minKubeVersion: 1.19.0 provider: name: Eclipse Foundation - version: 7.94.0-888.next + version: 7.94.0-889.next webhookdefinitions: - admissionReviewVersions: - v1 diff --git a/pkg/deploy/container-build/container_build.go b/pkg/deploy/container-build/container_build.go index 30889e315e..15499a7733 100644 --- a/pkg/deploy/container-build/container_build.go +++ b/pkg/deploy/container-build/container_build.go @@ -47,28 +47,28 @@ func (cb *ContainerBuildReconciler) Reconcile(ctx *chetypes.DeployContext) (reco // The check below to avoid NPE while CheCluster is not updated with defaults. if ctx.CheCluster.IsOpenShiftSecurityContextConstraintSet() { if done, err := cb.syncSCC(ctx); !done { - return reconcile.Result{}, false, err + return reconcile.Result{Requeue: true}, false, err } if done, err := cb.syncRBAC(ctx); !done { - return reconcile.Result{}, false, err + return reconcile.Result{Requeue: true}, false, err } if err := deploy.AppendFinalizer(ctx, cb.getFinalizerName()); err != nil { - return reconcile.Result{}, false, err + return reconcile.Result{Requeue: true}, false, err } } } else { if done, err := cb.removeRBAC(ctx); !done { - return reconcile.Result{}, false, err + return reconcile.Result{Requeue: true}, false, err } if done, err := cb.removeSCC(ctx); !done { - return reconcile.Result{}, false, err + return reconcile.Result{Requeue: true}, false, err } if err := deploy.DeleteFinalizer(ctx, cb.getFinalizerName()); err != nil { - return reconcile.Result{}, false, err + return reconcile.Result{Requeue: true}, false, err } } diff --git a/pkg/deploy/dev-workspace-config/dev_workspace_config.go b/pkg/deploy/dev-workspace-config/dev_workspace_config.go index c2ae4dd049..cae5483b08 100644 --- a/pkg/deploy/dev-workspace-config/dev_workspace_config.go +++ b/pkg/deploy/dev-workspace-config/dev_workspace_config.go @@ -67,11 +67,8 @@ func (d *DevWorkspaceConfigReconciler) Reconcile(ctx *chetypes.DeployContext) (r return reconcile.Result{}, false, err } - if done, err := deploy.Sync(ctx, dwoc); !done { - return reconcile.Result{}, false, err - } - - return reconcile.Result{}, true, nil + done, err := deploy.Sync(ctx, dwoc) + return reconcile.Result{Requeue: !done}, done, err } func (d *DevWorkspaceConfigReconciler) Finalize(ctx *chetypes.DeployContext) bool { diff --git a/pkg/deploy/reconcile_manager.go b/pkg/deploy/reconcile_manager.go index f52f337216..ae94d25ba1 100644 --- a/pkg/deploy/reconcile_manager.go +++ b/pkg/deploy/reconcile_manager.go @@ -14,13 +14,20 @@ package deploy import ( "fmt" + "log" + "os" + + ctrl "sigs.k8s.io/controller-runtime" "github.com/eclipse-che/che-operator/pkg/common/chetypes" "github.com/eclipse-che/che-operator/pkg/common/constants" - "github.com/sirupsen/logrus" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) +var ( + reconcilerLogger = ctrl.Log.WithName("reconciler_manager") +) + const Finalizer = "cluster-resources." + constants.FinalizerSuffix type Reconcilable interface { @@ -33,11 +40,15 @@ type Reconcilable interface { type ReconcileManager struct { reconcilers []Reconcilable failedReconciler Reconcilable + + // track reconciler invocations + operationLogger *log.Logger } func NewReconcileManager() *ReconcileManager { return &ReconcileManager{ reconcilers: make([]Reconcilable, 0), + operationLogger: initOperationLogger(), failedReconciler: nil, } } @@ -46,7 +57,7 @@ func (manager *ReconcileManager) RegisterReconciler(reconciler Reconcilable) { manager.reconcilers = append(manager.reconcilers, reconciler) } -// Reconcile all objects in a order they have been added +// ReconcileAll reconciles all objects in an order they have been added. // If reconciliation failed then CheCluster status will be updated accordingly. func (manager *ReconcileManager) ReconcileAll(ctx *chetypes.DeployContext) (reconcile.Result, bool, error) { if err := AppendFinalizer(ctx, Finalizer); err != nil { @@ -54,21 +65,36 @@ func (manager *ReconcileManager) ReconcileAll(ctx *chetypes.DeployContext) (reco } for _, reconciler := range manager.reconcilers { + reconcilerName := GetObjectType(reconciler) + + if manager.operationLogger != nil { + manager.operationLogger.Printf("Reconciler [%s] started.", reconcilerName) + } + result, done, err := reconciler.Reconcile(ctx) + + if manager.operationLogger != nil { + manager.operationLogger.Printf("Reconciler [%s] done: %t", reconcilerName, done) + } + if err != nil { + // set failed reconciler manager.failedReconciler = reconciler - reconcilerName := GetObjectType(reconciler) + errMsg := fmt.Sprintf("Reconciler failed %s, cause: %v", reconcilerName, err) if err := SetStatusDetails(ctx, constants.InstallOrUpdateFailed, errMsg); err != nil { - logrus.Errorf("Failed to update checluster status, cause: %v", err) + reconcilerLogger.Error(err, "Failed to update checluster status") } } else if manager.failedReconciler == reconciler { + // cleanup failed reconciler manager.failedReconciler = nil + if err := SetStatusDetails(ctx, "", ""); err != nil { - logrus.Errorf("Failed to update checluster status, cause: %v", err) + reconcilerLogger.Error(err, "Failed to update checluster status") } } + // don't continue if reconciliation failed if !done { return result, done, err } @@ -80,16 +106,17 @@ func (manager *ReconcileManager) ReconcileAll(ctx *chetypes.DeployContext) (reco func (manager *ReconcileManager) FinalizeAll(ctx *chetypes.DeployContext) (done bool) { done = true for _, reconciler := range manager.reconcilers { - if completed := reconciler.Finalize(ctx); !completed { - reconcilerName := GetObjectType(reconciler) - ctx.CheCluster.Status.Message = fmt.Sprintf("Finalization failed for reconciler: %s", reconcilerName) - _ = UpdateCheCRStatus(ctx, "Message", ctx.CheCluster.Status.Message) + completed := reconciler.Finalize(ctx) + done = done && completed - done = false + if !completed { + // don't prevent from invoking other finalizers, just log the error + reconcilerLogger.Error(nil, fmt.Sprintf("Finalization failed for reconciler: %s", GetObjectType(reconciler))) } } if done { + // Removes remaining finalizers not to prevent CheCluster object from being deleted if err := CleanUpAllFinalizers(ctx); err != nil { return false } @@ -97,3 +124,12 @@ func (manager *ReconcileManager) FinalizeAll(ctx *chetypes.DeployContext) (done return done } + +func initOperationLogger() *log.Logger { + logFile, err := os.OpenFile("/tmp/reconciler_manager.log", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666) + if err == nil { + return log.New(logFile, "INFO: ", log.Ldate|log.Ltime|log.Lshortfile) + } + + return nil +}