Optionally deploy cluster resource sync in standalone mode (#2098)

* wip Signed-off-by: Katrina Rogan <[email protected]> * fix? Signed-off-by: Katrina Rogan <[email protected]> * ofc Signed-off-by: Katrina Rogan <[email protected]> * more Signed-off-by: Katrina Rogan <[email protected]>
flyteorg · Jan 25, 2022 · d07726c · d07726c
1 parent 3dd3b25
commit d07726c
Show file tree

Hide file tree

Showing 12 changed files with 422 additions and 169 deletions.
diff --git a/charts/flyte-core/README.md b/charts/flyte-core/README.md
@@ -49,10 +49,11 @@ helm install gateway bitnami/contour -n flyte
 
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
-| cluster_resource_manager | object | `{"config":{"cluster_resources":{"customData":[{"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}]},{"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}]}],"refresh":"5m","refreshInterval":"5m","templatePath":"/etc/flyte/clusterresource/templates"}},"enabled":true,"templates":[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n  name: {{ namespace }}\nspec:\n  finalizers:\n  - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n  name: project-quota\n  namespace: {{ namespace }}\nspec:\n  hard:\n    limits.cpu: {{ projectQuotaCpu }}\n    limits.memory: {{ projectQuotaMemory }}\n"}]}` | Configuration for the Cluster resource manager component. This is an optional component, that enables automatic cluster configuration. This is useful to set default quotas, manage namespaces etc that map to a project/domain |
+| cluster_resource_manager | object | `{"config":{"cluster_resources":{"customData":[{"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}]},{"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}]}],"refresh":"5m","refreshInterval":"5m","templatePath":"/etc/flyte/clusterresource/templates"}},"enabled":true,"standalone_deploy":false,"templates":[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n  name: {{ namespace }}\nspec:\n  finalizers:\n  - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n  name: project-quota\n  namespace: {{ namespace }}\nspec:\n  hard:\n    limits.cpu: {{ projectQuotaCpu }}\n    limits.memory: {{ projectQuotaMemory }}\n"}]}` | Configuration for the Cluster resource manager component. This is an optional component, that enables automatic cluster configuration. This is useful to set default quotas, manage namespaces etc that map to a project/domain |
 | cluster_resource_manager.config | object | `{"cluster_resources":{"customData":[{"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}]},{"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}]}],"refresh":"5m","refreshInterval":"5m","templatePath":"/etc/flyte/clusterresource/templates"}}` | Configmap for ClusterResource parameters |
 | cluster_resource_manager.config.cluster_resources | object | `{"customData":[{"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}]},{"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}]}],"refresh":"5m","refreshInterval":"5m","templatePath":"/etc/flyte/clusterresource/templates"}` | ClusterResource parameters Refer to the [structure](https://pkg.go.dev/github.com/lyft/[email protected]/pkg/runtime/interfaces#ClusterResourceConfig) to customize. |
 | cluster_resource_manager.enabled | bool | `true` | Enables the Cluster resource manager component |
+| cluster_resource_manager.standalone_deploy | bool | `false` | Starts the cluster resource manager in standalone mode with requisite auth credentials to call flyteadmin service endpoints |
 | cluster_resource_manager.templates | list | `[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n  name: {{ namespace }}\nspec:\n  finalizers:\n  - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n  name: project-quota\n  namespace: {{ namespace }}\nspec:\n  hard:\n    limits.cpu: {{ projectQuotaCpu }}\n    limits.memory: {{ projectQuotaMemory }}\n"}]` | Resource templates that should be applied |
 | cluster_resource_manager.templates[0] | object | `{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n  name: {{ namespace }}\nspec:\n  finalizers:\n  - kubernetes\n"}` | Template for namespaces resources |
 | common | object | `{"databaseSecret":{"name":"","secretManifest":{}},"flyteNamespaceTemplate":{"enabled":false},"ingress":{"albSSLRedirect":false,"annotations":{"nginx.ingress.kubernetes.io/app-root":"/console"},"enabled":true,"separateGrpcIngress":false,"separateGrpcIngressAnnotations":{"nginx.ingress.kubernetes.io/backend-protocol":"GRPC"},"tls":{"enabled":false},"webpackHMR":false}}` | ---------------------------------------------- COMMON SETTINGS |

diff --git a/charts/flyte-core/templates/admin/deployment.yaml b/charts/flyte-core/templates/admin/deployment.yaml
@@ -62,7 +62,7 @@ spec:
           - mountPath: /etc/flyte/config
             name: config-volume
         {{- end }}
-        {{- if .Values.cluster_resource_manager.enabled }}
+        {{- if and (.Values.cluster_resource_manager.enabled) (not .Values.cluster_resource_manager.standalone_deploy) }}
         - command:
           - flyteadmin
           - --config

diff --git a/...tes/admin/cluster_resource_configmap.yaml → ...ourcesync/cluster_resource_configmap.yaml b/...tes/admin/cluster_resource_configmap.yaml → ...ourcesync/cluster_resource_configmap.yaml
diff --git a/charts/flyte-core/templates/clusterresourcesync/configmap.yaml b/charts/flyte-core/templates/clusterresourcesync/configmap.yaml
@@ -0,0 +1,30 @@
+{{- if .Values.cluster_resource_manager.enabled }}
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: flyte-clusterresourcesync-config
+  namespace: {{ template "flyte.namespace" . }}
+  labels: {{ include "flyteadmin.labels" . | nindent 4 }}
+data:
+  {{- with .Values.cluster_resource_manager.config }}
+  cluster_resources.yaml: | {{ tpl (toYaml .) $ | nindent 4 }}
+  {{- end }}
+
+  {{- if .Values.cluster_resource_manager.standalone_deploy }}
+  {{- with .Values.configmap.admin }}
+  admin.yaml: | {{ tpl (toYaml .) $ | nindent 4 }}
+  {{- end }}
+  {{- end }}
+
+  {{- if not .Values.cluster_resource_manager.standalone_deploy }}
+  {{- with .Values.db.admin }}
+  db.yaml: | {{ tpl (toYaml .) $ | nindent 4 }}
+  {{- end }}
+  {{- end }}
+
+  {{- with .Values.configmap.domain }}
+  domain.yaml: | {{ tpl (toYaml .) $ | nindent 4 }}
+  {{- end }}
+
+  {{- end }}
diff --git a/...s/flyte-core/templates/admin/cronjob.yaml → ...emplates/clusterresourcesync/cronjob.yaml b/...s/flyte-core/templates/admin/cronjob.yaml → ...emplates/clusterresourcesync/cronjob.yaml
@@ -26,7 +26,10 @@ spec:
             image: "{{ .Values.flyteadmin.image.repository }}:{{ .Values.flyteadmin.image.tag }}"
             imagePullPolicy: "{{ .Values.flyteadmin.image.pullPolicy }}"
             name: sync-cluster-resources
-            volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 12 }}
+            volumeMounts:
+            {{- if not .Values.cluster_resource_manager.standalone_deploy}}
+            {{- include "databaseSecret.volumeMount" . | nindent 12 }}
+            {{- end }}
             - mountPath: /etc/flyte/clusterresource/templates
               name: resource-templates
             - mountPath: /etc/flyte/config
@@ -38,7 +41,7 @@ spec:
               name: clusterresource-template
             name: resource-templates
           - configMap:
-              name: flyte-admin-config
+              name: flyte-clusterresourcesync-config
             name: config-volume
   {{- end }}
   {{- end }}
diff --git a/charts/flyte-core/values-eks.yaml b/charts/flyte-core/values-eks.yaml
@@ -360,6 +360,8 @@ workflow_notifications:
 cluster_resource_manager:
   # -- Enables the Cluster resource manager component
   enabled: true
+  # -- Starts the cluster resource manager in standalone mode with requisite auth credentials to call flyteadmin service endpoints
+  standalone_deploy: false
   config:
     cluster_resources:
       customData:

diff --git a/charts/flyte-core/values-gcp.yaml b/charts/flyte-core/values-gcp.yaml
@@ -319,6 +319,8 @@ configmap:
 cluster_resource_manager:
   # -- Enables the Cluster resource manager component
   enabled: true
+  # -- Starts the cluster resource manager in standalone mode with requisite auth credentials to call flyteadmin service endpoints
+  standalone_deploy: false
   config:
     cluster_resources:
       customData:

diff --git a/charts/flyte-core/values.yaml b/charts/flyte-core/values.yaml
@@ -638,6 +638,8 @@ external_events:
 cluster_resource_manager:
   # -- Enables the Cluster resource manager component
   enabled: true
+  # -- Starts the cluster resource manager in standalone mode with requisite auth credentials to call flyteadmin service endpoints
+  standalone_deploy: false
   # -- Configmap for ClusterResource parameters
   config:
     # -- ClusterResource parameters

diff --git a/deployment/eks/flyte_aws_scheduler_helm_generated.yaml b/deployment/eks/flyte_aws_scheduler_helm_generated.yaml
@@ -86,47 +86,6 @@ metadata:
   namespace: flyte
 type: Opaque
 ---
-# Source: flyte-core/templates/admin/cluster_resource_configmap.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: clusterresource-template
-  namespace: flyte
-  labels: 
-    app.kubernetes.io/name: flyteadmin
-    app.kubernetes.io/instance: flyte
-    helm.sh/chart: flyte-core-v0.1.10
-    app.kubernetes.io/managed-by: Helm
-data:
-  aa_namespace.yaml: | 
-    apiVersion: v1
-    kind: Namespace
-    metadata:
-      name: {{ namespace }}
-    spec:
-      finalizers:
-      - kubernetes
-    
-  aab_default_service_account.yaml: | 
-    apiVersion: v1
-    kind: ServiceAccount
-    metadata:
-      name: default
-      namespace: {{ namespace }}
-      annotations:
-        eks.amazonaws.com/role-arn: {{ defaultIamRole }}
-    
-  ab_project_resource_quota.yaml: | 
-    apiVersion: v1
-    kind: ResourceQuota
-    metadata:
-      name: project-quota
-      namespace: {{ namespace }}
-    spec:
-      hard:
-        limits.cpu: {{ projectQuotaCpu }}
-        limits.memory: {{ projectQuotaMemory }}
----
 # Source: flyte-core/templates/admin/configmap.yaml
 apiVersion: v1
 kind: ConfigMap
@@ -269,6 +228,102 @@ data:
         scheduleQueueName: flyte-helm-test-cron-scheduler-queue
         scheme: aws
 ---
+# Source: flyte-core/templates/clusterresourcesync/cluster_resource_configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: clusterresource-template
+  namespace: flyte
+  labels: 
+    app.kubernetes.io/name: flyteadmin
+    app.kubernetes.io/instance: flyte
+    helm.sh/chart: flyte-core-v0.1.10
+    app.kubernetes.io/managed-by: Helm
+data:
+  aa_namespace.yaml: | 
+    apiVersion: v1
+    kind: Namespace
+    metadata:
+      name: {{ namespace }}
+    spec:
+      finalizers:
+      - kubernetes
+    
+  aab_default_service_account.yaml: | 
+    apiVersion: v1
+    kind: ServiceAccount
+    metadata:
+      name: default
+      namespace: {{ namespace }}
+      annotations:
+        eks.amazonaws.com/role-arn: {{ defaultIamRole }}
+    
+  ab_project_resource_quota.yaml: | 
+    apiVersion: v1
+    kind: ResourceQuota
+    metadata:
+      name: project-quota
+      namespace: {{ namespace }}
+    spec:
+      hard:
+        limits.cpu: {{ projectQuotaCpu }}
+        limits.memory: {{ projectQuotaMemory }}
+---
+# Source: flyte-core/templates/clusterresourcesync/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: flyte-clusterresourcesync-config
+  namespace: flyte
+  labels: 
+    app.kubernetes.io/name: flyteadmin
+    app.kubernetes.io/instance: flyte
+    helm.sh/chart: flyte-core-v0.1.10
+    app.kubernetes.io/managed-by: Helm
+data:
+  cluster_resources.yaml: | 
+    cluster_resources:
+      customData:
+      - production:
+        - projectQuotaCpu:
+            value: "5"
+        - projectQuotaMemory:
+            value: 4000Mi
+        - defaultIamRole:
+            value: arn:aws:iam::<ACCOUNT_NUMBER>:role/flyte-user-role
+      - staging:
+        - projectQuotaCpu:
+            value: "2"
+        - projectQuotaMemory:
+            value: 3000Mi
+        - defaultIamRole:
+            value: arn:aws:iam::<ACCOUNT_NUMBER>:role/flyte-user-role
+      - development:
+        - projectQuotaCpu:
+            value: "4"
+        - projectQuotaMemory:
+            value: 3000Mi
+        - defaultIamRole:
+            value: arn:aws:iam::<ACCOUNT_NUMBER>:role/flyte-user-role
+      refresh: 5m
+      refreshInterval: 5m
+      templatePath: /etc/flyte/clusterresource/templates
+  db.yaml: | 
+    database:
+      dbname: flyteadmin
+      host: '<RDS_HOST>'
+      passwordPath: /etc/db/pass.txt
+      port: 5432
+      username: flyteadmin
+  domain.yaml: | 
+    domains:
+    - id: development
+      name: development
+    - id: staging
+      name: staging
+    - id: production
+      name: production
+---
 # Source: flyte-core/templates/console/configmap.yaml
 apiVersion: v1
 kind: ConfigMap
@@ -1227,7 +1282,7 @@ spec:
           secret:
             secretName: flyte-pod-webhook
 ---
-# Source: flyte-core/templates/admin/cronjob.yaml
+# Source: flyte-core/templates/clusterresourcesync/cronjob.yaml
 apiVersion: batch/v1beta1
 kind: CronJob
 metadata:
@@ -1271,7 +1326,7 @@ spec:
               name: clusterresource-template
             name: resource-templates
           - configMap:
-              name: flyte-admin-config
+              name: flyte-clusterresourcesync-config
             name: config-volume
 ---
 # Source: flyte-core/templates/common/ingress.yaml