From 5fe518533e7bd8f995215e17d70a43644320e074 Mon Sep 17 00:00:00 2001
From: Erika Hunhoff <hunhoff.erika@gmail.com>
Date: Sun, 10 Jul 2022 21:21:58 -0600
Subject: [PATCH 01/10] Kubernetes Akka bootstrap for controller

---
 build.gradle                                               | 2 +-
 core/controller/build.gradle                               | 1 +
 core/controller/src/main/resources/application.conf        | 5 ++++-
 .../org/apache/openwhisk/core/controller/Controller.scala  | 7 +++++++
 4 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/build.gradle b/build.gradle
index a0a49e8a7d2..cebe54c7bc7 100644
--- a/build.gradle
+++ b/build.gradle
@@ -42,7 +42,7 @@ subprojects {
         def cons = project.getDependencies().getConstraints()
         def akka = ['akka-actor', 'akka-cluster', 'akka-cluster-metrics', 'akka-cluster-tools', 'akka-coordination',
                     'akka-discovery', 'akka-distributed-data', 'akka-protobuf', 'akka-remote', 'akka-slf4j',
-                    'akka-stream', 'akka-stream-testkit', 'akka-testkit']
+                    'akka-stream', 'akka-stream-testkit', 'akka-testkit', 'akka-persistence', 'akka-cluster-sharding']
         def akkaHttp = ['akka-http', 'akka-http-core', 'akka-http-spray-json', 'akka-http-testkit', 'akka-http-xml',
                         'akka-parsing', 'akka-http2-support']
 
diff --git a/core/controller/build.gradle b/core/controller/build.gradle
index 7d5163438fc..110eb99b14d 100644
--- a/core/controller/build.gradle
+++ b/core/controller/build.gradle
@@ -41,6 +41,7 @@ dependencies {
     implementation project(':core:invoker')
     implementation project(':core:scheduler')
 
+    implementation "com.lightbend.akka.management:akka-management-cluster-http_${gradle.scala.depVersion}:${gradle.akka_management.version}"  
     implementation "com.lightbend.akka.management:akka-management-cluster-bootstrap_${gradle.scala.depVersion}:${gradle.akka_management.version}"
     implementation "com.lightbend.akka.discovery:akka-discovery-kubernetes-api_${gradle.scala.depVersion}:${gradle.akka_management.version}"
     implementation "com.lightbend.akka.discovery:akka-discovery-marathon-api_${gradle.scala.depVersion}:${gradle.akka_management.version}"
diff --git a/core/controller/src/main/resources/application.conf b/core/controller/src/main/resources/application.conf
index e7bf1a9566d..2c8f4fd8d52 100644
--- a/core/controller/src/main/resources/application.conf
+++ b/core/controller/src/main/resources/application.conf
@@ -91,14 +91,17 @@ akka {
     artery {
       enabled = on
       transport = tcp
+      canonical.port = 25520
     }
     log-remote-lifecycle-events = DEBUG
     log-received-messages = on
     log-sent-messages = on
   }
   cluster {
+    shutdown-after-unsuccessful-join-seed-nodes = 60s
+
     # Disable legacy metrics in akka-cluster.
-    metrics.enabled=off
+    metrics.enabled = off
   }
 }
 
diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Controller.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Controller.scala
index 0e662faa454..ff42cfc056a 100644
--- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Controller.scala
+++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Controller.scala
@@ -24,6 +24,8 @@ import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
 import akka.http.scaladsl.model.StatusCodes._
 import akka.http.scaladsl.model.Uri
 import akka.http.scaladsl.server.Route
+import akka.management.scaladsl.AkkaManagement
+import akka.management.cluster.bootstrap.ClusterBootstrap
 import kamon.Kamon
 import org.apache.openwhisk.common.Https.HttpsConfig
 import org.apache.openwhisk.common._
@@ -185,6 +187,7 @@ object Controller {
   protected val protocol = loadConfigOrThrow[String]("whisk.controller.protocol")
   protected val interface = loadConfigOrThrow[String]("whisk.controller.interface")
   protected val readinessThreshold = loadConfig[Double]("whisk.controller.readiness-fraction")
+  protected val useClusterBootstrap = loadConfigOrThrow[Boolean]("whisk.cluster.use-cluster-bootstrap")
 
   val topicPrefix = loadConfigOrThrow[String](ConfigKeys.kafkaTopicsPrefix)
   val userEventTopicPrefix = loadConfigOrThrow[String](ConfigKeys.kafkaTopicsUserEventPrefix)
@@ -230,6 +233,10 @@ object Controller {
   def main(args: Array[String]): Unit = {
     implicit val actorSystem = ActorSystem("controller-actor-system")
     implicit val logger = new AkkaLogging(akka.event.Logging.getLogger(actorSystem, this))
+    if (useClusterBootstrap) {
+      AkkaManagement(actorSystem).start()
+      ClusterBootstrap(actorSystem).start()
+    }
     start(args)
   }
 

From 3e1a331818123c3ab378d6c5d1a84eb990acbb6c Mon Sep 17 00:00:00 2001
From: Erika Hunhoff <hunhoff.erika@gmail.com>
Date: Mon, 11 Jul 2022 19:38:05 -0600
Subject: [PATCH 02/10] Update cluster management for the scheduler to help
 with k8s deployment

---
 core/scheduler/build.gradle                          |  3 +++
 core/scheduler/src/main/resources/application.conf   | 12 ++++++++++++
 .../apache/openwhisk/core/scheduler/Scheduler.scala  |  8 ++++++++
 3 files changed, 23 insertions(+)

diff --git a/core/scheduler/build.gradle b/core/scheduler/build.gradle
index 02c4d2be23f..7f95912a4a9 100644
--- a/core/scheduler/build.gradle
+++ b/core/scheduler/build.gradle
@@ -65,6 +65,9 @@ dependencies {
     implementation "org.scala-lang:scala-library:${gradle.scala.version}"
     implementation project(':common:scala')
     implementation "io.altoo:akka-kryo-serialization_${gradle.scala.depVersion}:1.0.0"
+    implementation "com.lightbend.akka.management:akka-management-cluster-http_${gradle.scala.depVersion}:${gradle.akka_management.version}" 
+    implementation "com.lightbend.akka.management:akka-management-cluster-bootstrap_${gradle.scala.depVersion}:${gradle.akka_management.version}"
+    implementation "com.lightbend.akka.discovery:akka-discovery-kubernetes-api_${gradle.scala.depVersion}:${gradle.akka_management.version}"
 }
 
 // workaround for akka-grpc
diff --git a/core/scheduler/src/main/resources/application.conf b/core/scheduler/src/main/resources/application.conf
index e73f764c1c8..f57c558499d 100644
--- a/core/scheduler/src/main/resources/application.conf
+++ b/core/scheduler/src/main/resources/application.conf
@@ -42,11 +42,23 @@ akka {
     artery {
       enabled = on
       transport = tcp
+      canonical.port = 25520
     }
   }
+
+  cluster {
+    shutdown-after-unsuccessful-join-seed-nodes = 60s
+
+    # Disable legacy metrics in akka-cluster.
+    metrics.enabled = off
+  }
 }
 
 whisk {
+  cluster {
+    use-cluster-bootstrap: false
+  }
+
   # tracing configuration
   tracing {
     component = "Scheduler"
diff --git a/core/scheduler/src/main/scala/org/apache/openwhisk/core/scheduler/Scheduler.scala b/core/scheduler/src/main/scala/org/apache/openwhisk/core/scheduler/Scheduler.scala
index 128c004ef53..f32520cc756 100644
--- a/core/scheduler/src/main/scala/org/apache/openwhisk/core/scheduler/Scheduler.scala
+++ b/core/scheduler/src/main/scala/org/apache/openwhisk/core/scheduler/Scheduler.scala
@@ -21,6 +21,8 @@ import akka.Done
 import akka.actor.{ActorRef, ActorRefFactory, ActorSelection, ActorSystem, CoordinatedShutdown, Props}
 import akka.http.scaladsl.Http
 import akka.http.scaladsl.model.{HttpRequest, HttpResponse}
+import akka.management.scaladsl.AkkaManagement
+import akka.management.cluster.bootstrap.ClusterBootstrap
 import akka.pattern.ask
 import akka.util.Timeout
 import com.typesafe.config.ConfigValueFactory
@@ -258,6 +260,7 @@ trait SchedulerCore {
 object Scheduler {
 
   protected val protocol = loadConfigOrThrow[String]("whisk.scheduler.protocol")
+  protected val useClusterBootstrap = loadConfigOrThrow[Boolean]("whisk.cluster.use-cluster-bootstrap")
 
   val topicPrefix = loadConfigOrThrow[String](ConfigKeys.kafkaTopicsPrefix)
 
@@ -289,6 +292,11 @@ object Scheduler {
 
     implicit val logger = new AkkaLogging(akka.event.Logging.getLogger(actorSystem, this))
 
+    if (useClusterBootstrap) {
+      AkkaManagement(actorSystem).start()
+      ClusterBootstrap(actorSystem).start()
+    }
+
     // Prepare Kamon shutdown
     CoordinatedShutdown(actorSystem).addTask(CoordinatedShutdown.PhaseActorSystemTerminate, "shutdownKamon") { () =>
       logger.info(this, s"Shutting down Kamon with coordinated shutdown")

From 97a60fd4a8d5338d6acea5d7d6369397150f22bc Mon Sep 17 00:00:00 2001
From: Erika Hunhoff <hunhoff.erika@gmail.com>
Date: Mon, 11 Jul 2022 20:43:18 -0600
Subject: [PATCH 03/10] Made changes to tools to try to integrate scheduler
 into travis/CI build process

---
 Jenkinsfile                           | 2 +-
 tools/admin/wskadmin                  | 2 +-
 tools/build/checkLogs.py              | 1 +
 tools/build/citool                    | 2 ++
 tools/build/redo                      | 5 +++++
 tools/jenkins/apache/dockerhub.groovy | 2 +-
 tools/travis/distDocker.sh            | 1 +
 7 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 3e54c99c490..7f3863514c4 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -49,7 +49,7 @@ timeout(time: 12, unit: 'HOURS') {
                         sh "docker run -d --restart=always --name registry -v \"$HOME\"/certs:/certs \
                                 -e REGISTRY_HTTP_ADDR=0.0.0.0:${port} -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/${cert} \
                                 -e REGISTRY_HTTP_TLS_KEY=/certs/${key} -p ${port}:${port} registry:2"
-                        // Build the controller and invoker images.
+                        // Build the controller, scheduler, and invoker images.
                         sh "./gradlew distDocker -PdockerRegistry=${domainName}:${port}"
                         //Install the various modules like standalone
                         sh "./gradlew install"
diff --git a/tools/admin/wskadmin b/tools/admin/wskadmin
index f65d7c29b32..bd37916f48e 100755
--- a/tools/admin/wskadmin
+++ b/tools/admin/wskadmin
@@ -169,7 +169,7 @@ def parseArgs():
     subparser.required = True
 
     subcmd = subparser.add_parser('get', help='get logs')
-    subcmd.add_argument('components', help='components, one or more of [controllerN, invokerN] where N is the instance', nargs='*', default=['controller0', 'invoker0'])
+    subcmd.add_argument('components', help='components, one or more of [controllerN, schedulerN, invokerN] where N is the instance', nargs='*', default=['controller0', 'invoker0', 'scheduler0'])
     subcmd.add_argument('-t', '--tid', help='retrieve logs for the transaction id')
     subcmd.add_argument('-g', '--grep', help='retrieve logs that match grep expression')
 
diff --git a/tools/build/checkLogs.py b/tools/build/checkLogs.py
index b436042a3cf..9d202242a4a 100755
--- a/tools/build/checkLogs.py
+++ b/tools/build/checkLogs.py
@@ -96,6 +96,7 @@ def colorize(code, string):
         ("db-triggers.log", {"db"}, [ partial(database_has_at_most_x_entries, 0) ]),
         # Assert that stdout of the container is correctly piped and empty
         ("controller0.log", {"system"}, [ partial(file_has_at_most_x_bytes, 0) ]),
+        ("scheduler0.log", {"system"}, [ partial(file_has_at_most_x_bytes, 0) ]), 
         ("invoker0.log", {"system"}, [ partial(file_has_at_most_x_bytes, 0) ])
     ]
 
diff --git a/tools/build/citool b/tools/build/citool
index 8a5403d3fae..1202e2f8470 100755
--- a/tools/build/citool
+++ b/tools/build/citool
@@ -87,6 +87,7 @@ def parseArgs():
     subparser.add_argument('-s', '--sort', help='sort logs by timestamp', action='store_true')
     subparser.add_argument('-n', '--invokers', help='number of invokers', type=int, default=3)
     subparser.add_argument('-c', '--controllers', help='number of controllers', type=int, default=1)
+    subparser.add_argument('-c', '--schedulers', help='number of schedulers', type=int, default=1)
 
     return parser.parse_args()
 
@@ -312,6 +313,7 @@ def cat(args):
     else:
         components = {
             'controller': args.controllers,
+            'scheduler': args.schedulers,
             'invoker': args.invokers
         }
         logs = map(getComponentLogs, getComponentList(components))
diff --git a/tools/build/redo b/tools/build/redo
index b149c1aa3fa..b2ec0881289 100755
--- a/tools/build/redo
+++ b/tools/build/redo
@@ -272,6 +272,11 @@ Components = [
                   modes = 'clean',
                   gradle = 'core:controller'),
 
+    makeComponent('scheduler',
+                  'build/deploy scheduler',
+                  modes = 'clean',
+                  gradle = 'core:scheduler'),
+
     makeComponent('invoker',
                   'build/deploy invoker',
                   modes = 'clean',
diff --git a/tools/jenkins/apache/dockerhub.groovy b/tools/jenkins/apache/dockerhub.groovy
index 6ff1c591d9d..65734952294 100644
--- a/tools/jenkins/apache/dockerhub.groovy
+++ b/tools/jenkins/apache/dockerhub.groovy
@@ -30,7 +30,7 @@ node('ubuntu') {
       withCredentials([usernamePassword(credentialsId: 'openwhisk_dockerhub', passwordVariable: 'DOCKER_PASSWORD', usernameVariable: 'DOCKER_USER')]) {
           sh 'HOME="$WORKSPACE/local-docker-cfg" docker login -u ${DOCKER_USER} -p ${DOCKER_PASSWORD}'
       }
-      def PUSH_CMD = "./gradlew :core:controller:distDocker :core:invoker:distDocker :core:standalone:distDocker :core:monitoring:user-events:distDocker :tools:ow-utils:distDocker :core:cosmos:cache-invalidator:distDocker -PdockerRegistry=docker.io -PdockerImagePrefix=openwhisk"
+      def PUSH_CMD = "./gradlew :core:controller:distDocker :core:scheduler:distDocker :core:invoker:distDocker :core:standalone:distDocker :core:monitoring:user-events:distDocker :tools:ow-utils:distDocker :core:cosmos:cache-invalidator:distDocker -PdockerRegistry=docker.io -PdockerImagePrefix=openwhisk"
       def gitCommit = sh(returnStdout: true, script: 'git rev-parse HEAD').trim()
       def shortCommit = gitCommit.take(7)
       sh "./gradlew clean"
diff --git a/tools/travis/distDocker.sh b/tools/travis/distDocker.sh
index a859beca289..13c8450d98b 100755
--- a/tools/travis/distDocker.sh
+++ b/tools/travis/distDocker.sh
@@ -29,6 +29,7 @@ TERM=dumb ./gradlew clean # Run a clean step before build
 TERM=dumb ./gradlew distDocker -PdockerImagePrefix=testing $GRADLE_PROJS_SKIP
 
 TERM=dumb ./gradlew :core:controller:distDockerCoverage -PdockerImagePrefix=testing
+TERM=dumb ./gradlew :core:scheduler:distDockerCoverage -PdockerImagePrefix=testing
 TERM=dumb ./gradlew :core:invoker:distDockerCoverage -PdockerImagePrefix=testing
 TERM=dumb ./gradlew :core:standalone:build
 

From 7c05fabd800a8d08f30a0db64e52788080cf2077 Mon Sep 17 00:00:00 2001
From: Erika Hunhoff <hunhoff.erika@gmail.com>
Date: Tue, 12 Jul 2022 21:48:37 -0600
Subject: [PATCH 04/10] Added scheduler Dockerfile.cov

---
 core/scheduler/Dockerfile.cov | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 core/scheduler/Dockerfile.cov

diff --git a/core/scheduler/Dockerfile.cov b/core/scheduler/Dockerfile.cov
new file mode 100644
index 00000000000..ef3a09471a4
--- /dev/null
+++ b/core/scheduler/Dockerfile.cov
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+FROM scheduler
+
+ARG OW_ROOT_DIR
+
+USER root
+RUN mkdir -p /coverage/common && \
+    mkdir -p /coverage/scheduler && \
+    mkdir -p "${OW_ROOT_DIR}/common/scala/build" && \
+    mkdir -p "${OW_ROOT_DIR}/core/scheduler/build" && \
+    ln -s /coverage/common "${OW_ROOT_DIR}/common/scala/build/scoverage" && \
+    ln -s /coverage/scheduler "${OW_ROOT_DIR}/core/scheduler/build/scoverage"
+
+COPY build/tmp/docker-coverage /scheduler/

From ccd652a9497e9475563a3671cb0dbd709460cb57 Mon Sep 17 00:00:00 2001
From: Erika Hunhoff <hunhoff.erika@gmail.com>
Date: Tue, 12 Jul 2022 22:10:06 -0600
Subject: [PATCH 05/10] Use consistent ordering of components

---
 tools/admin/wskadmin | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/admin/wskadmin b/tools/admin/wskadmin
index bd37916f48e..126e7847d57 100755
--- a/tools/admin/wskadmin
+++ b/tools/admin/wskadmin
@@ -169,7 +169,7 @@ def parseArgs():
     subparser.required = True
 
     subcmd = subparser.add_parser('get', help='get logs')
-    subcmd.add_argument('components', help='components, one or more of [controllerN, schedulerN, invokerN] where N is the instance', nargs='*', default=['controller0', 'invoker0', 'scheduler0'])
+    subcmd.add_argument('components', help='components, one or more of [controllerN, schedulerN, invokerN] where N is the instance', nargs='*', default=['controller0', 'scheduler0', 'invoker0'])
     subcmd.add_argument('-t', '--tid', help='retrieve logs for the transaction id')
     subcmd.add_argument('-g', '--grep', help='retrieve logs that match grep expression')
 

From c43837ae5dacb7bdbee44d0326324d834a6cb301 Mon Sep 17 00:00:00 2001
From: Erika Hunhoff <hunhoff.erika@gmail.com>
Date: Tue, 12 Jul 2022 22:28:27 -0600
Subject: [PATCH 06/10] remove canonical.port setting in scheduler, controller

---
 core/controller/src/main/resources/application.conf | 1 -
 core/scheduler/src/main/resources/application.conf  | 1 -
 2 files changed, 2 deletions(-)

diff --git a/core/controller/src/main/resources/application.conf b/core/controller/src/main/resources/application.conf
index 2c8f4fd8d52..c894c7397ca 100644
--- a/core/controller/src/main/resources/application.conf
+++ b/core/controller/src/main/resources/application.conf
@@ -91,7 +91,6 @@ akka {
     artery {
       enabled = on
       transport = tcp
-      canonical.port = 25520
     }
     log-remote-lifecycle-events = DEBUG
     log-received-messages = on
diff --git a/core/scheduler/src/main/resources/application.conf b/core/scheduler/src/main/resources/application.conf
index f57c558499d..7e670cc1463 100644
--- a/core/scheduler/src/main/resources/application.conf
+++ b/core/scheduler/src/main/resources/application.conf
@@ -42,7 +42,6 @@ akka {
     artery {
       enabled = on
       transport = tcp
-      canonical.port = 25520
     }
   }
 

From f6ed475b487fa967820b594a6d253c161a8601d4 Mon Sep 17 00:00:00 2001
From: Erika Hunhoff <hunhoff.erika@gmail.com>
Date: Thu, 21 Jul 2022 10:07:22 -0600
Subject: [PATCH 07/10] Remove unneeded dependency from controller, scheduler

---
 core/controller/build.gradle | 1 -
 core/scheduler/build.gradle  | 1 -
 2 files changed, 2 deletions(-)

diff --git a/core/controller/build.gradle b/core/controller/build.gradle
index 110eb99b14d..7d5163438fc 100644
--- a/core/controller/build.gradle
+++ b/core/controller/build.gradle
@@ -41,7 +41,6 @@ dependencies {
     implementation project(':core:invoker')
     implementation project(':core:scheduler')
 
-    implementation "com.lightbend.akka.management:akka-management-cluster-http_${gradle.scala.depVersion}:${gradle.akka_management.version}"  
     implementation "com.lightbend.akka.management:akka-management-cluster-bootstrap_${gradle.scala.depVersion}:${gradle.akka_management.version}"
     implementation "com.lightbend.akka.discovery:akka-discovery-kubernetes-api_${gradle.scala.depVersion}:${gradle.akka_management.version}"
     implementation "com.lightbend.akka.discovery:akka-discovery-marathon-api_${gradle.scala.depVersion}:${gradle.akka_management.version}"
diff --git a/core/scheduler/build.gradle b/core/scheduler/build.gradle
index 7f95912a4a9..ae5068ce914 100644
--- a/core/scheduler/build.gradle
+++ b/core/scheduler/build.gradle
@@ -65,7 +65,6 @@ dependencies {
     implementation "org.scala-lang:scala-library:${gradle.scala.version}"
     implementation project(':common:scala')
     implementation "io.altoo:akka-kryo-serialization_${gradle.scala.depVersion}:1.0.0"
-    implementation "com.lightbend.akka.management:akka-management-cluster-http_${gradle.scala.depVersion}:${gradle.akka_management.version}" 
     implementation "com.lightbend.akka.management:akka-management-cluster-bootstrap_${gradle.scala.depVersion}:${gradle.akka_management.version}"
     implementation "com.lightbend.akka.discovery:akka-discovery-kubernetes-api_${gradle.scala.depVersion}:${gradle.akka_management.version}"
 }

From 57b44b988f9b83bb993959a336cf76b6c6d36595 Mon Sep 17 00:00:00 2001
From: Erika Hunhoff <hunhoff.erika@gmail.com>
Date: Mon, 25 Jul 2022 12:45:27 -0600
Subject: [PATCH 08/10] Remove cluster creation from
 ShardingContainerPoolBalancer

---
 .../core/loadBalancer/ShardingContainerPoolBalancer.scala | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/loadBalancer/ShardingContainerPoolBalancer.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/loadBalancer/ShardingContainerPoolBalancer.scala
index 5f7b9f05cd6..b33cfa4d753 100644
--- a/core/controller/src/main/scala/org/apache/openwhisk/core/loadBalancer/ShardingContainerPoolBalancer.scala
+++ b/core/controller/src/main/scala/org/apache/openwhisk/core/loadBalancer/ShardingContainerPoolBalancer.scala
@@ -24,8 +24,6 @@ import java.util.concurrent.ThreadLocalRandom
 import akka.actor.{Actor, ActorSystem, Cancellable, Props}
 import akka.cluster.ClusterEvent._
 import akka.cluster.{Cluster, Member, MemberStatus}
-import akka.management.scaladsl.AkkaManagement
-import akka.management.cluster.bootstrap.ClusterBootstrap
 import org.apache.openwhisk.common.InvokerState.{Healthy, Offline, Unhealthy, Unresponsive}
 import pureconfig._
 import pureconfig.generic.auto._
@@ -154,11 +152,7 @@ class ShardingContainerPoolBalancer(
     extends CommonLoadBalancer(config, feedFactory, controllerInstance) {
 
   /** Build a cluster of all loadbalancers */
-  private val cluster: Option[Cluster] = if (loadConfigOrThrow[ClusterConfig](ConfigKeys.cluster).useClusterBootstrap) {
-    AkkaManagement(actorSystem).start()
-    ClusterBootstrap(actorSystem).start()
-    Some(Cluster(actorSystem))
-  } else if (loadConfigOrThrow[Seq[String]]("akka.cluster.seed-nodes").nonEmpty) {
+  private val cluster: Option[Cluster] = if (loadConfigOrThrow[ClusterConfig](ConfigKeys.cluster).useClusterBootstrap || loadConfigOrThrow[Seq[String]]("akka.cluster.seed-nodes").nonEmpty) {
     Some(Cluster(actorSystem))
   } else {
     None

From 1890a95b99f1d400f3100bf209a32367cd0d6fed Mon Sep 17 00:00:00 2001
From: Erika Hunhoff <hunhoff.erika@gmail.com>
Date: Thu, 1 Sep 2022 17:10:18 -0600
Subject: [PATCH 09/10] Remove trailing whitespace

Signed-off-by: Erika Hunhoff <hunhoff.erika@gmail.com>
---
 tools/build/checkLogs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/build/checkLogs.py b/tools/build/checkLogs.py
index 9d202242a4a..049156b32b8 100755
--- a/tools/build/checkLogs.py
+++ b/tools/build/checkLogs.py
@@ -96,7 +96,7 @@ def colorize(code, string):
         ("db-triggers.log", {"db"}, [ partial(database_has_at_most_x_entries, 0) ]),
         # Assert that stdout of the container is correctly piped and empty
         ("controller0.log", {"system"}, [ partial(file_has_at_most_x_bytes, 0) ]),
-        ("scheduler0.log", {"system"}, [ partial(file_has_at_most_x_bytes, 0) ]), 
+        ("scheduler0.log", {"system"}, [ partial(file_has_at_most_x_bytes, 0) ]),
         ("invoker0.log", {"system"}, [ partial(file_has_at_most_x_bytes, 0) ])
     ]
 

From 806cbe7d58a5cb23e99ab0986270698d37cd28b6 Mon Sep 17 00:00:00 2001
From: Erika Hunhoff <hunhoff.erika@gmail.com>
Date: Thu, 1 Sep 2022 17:11:00 -0600
Subject: [PATCH 10/10] Revert akka cluster changes to controller

Signed-off-by: Erika Hunhoff <hunhoff.erika@gmail.com>
---
 core/controller/src/main/resources/application.conf       | 4 +---
 .../org/apache/openwhisk/core/controller/Controller.scala | 7 -------
 .../core/loadBalancer/ShardingContainerPoolBalancer.scala | 8 +++++++-
 3 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/core/controller/src/main/resources/application.conf b/core/controller/src/main/resources/application.conf
index c894c7397ca..e7bf1a9566d 100644
--- a/core/controller/src/main/resources/application.conf
+++ b/core/controller/src/main/resources/application.conf
@@ -97,10 +97,8 @@ akka {
     log-sent-messages = on
   }
   cluster {
-    shutdown-after-unsuccessful-join-seed-nodes = 60s
-
     # Disable legacy metrics in akka-cluster.
-    metrics.enabled = off
+    metrics.enabled=off
   }
 }
 
diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Controller.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Controller.scala
index ff42cfc056a..0e662faa454 100644
--- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Controller.scala
+++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Controller.scala
@@ -24,8 +24,6 @@ import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
 import akka.http.scaladsl.model.StatusCodes._
 import akka.http.scaladsl.model.Uri
 import akka.http.scaladsl.server.Route
-import akka.management.scaladsl.AkkaManagement
-import akka.management.cluster.bootstrap.ClusterBootstrap
 import kamon.Kamon
 import org.apache.openwhisk.common.Https.HttpsConfig
 import org.apache.openwhisk.common._
@@ -187,7 +185,6 @@ object Controller {
   protected val protocol = loadConfigOrThrow[String]("whisk.controller.protocol")
   protected val interface = loadConfigOrThrow[String]("whisk.controller.interface")
   protected val readinessThreshold = loadConfig[Double]("whisk.controller.readiness-fraction")
-  protected val useClusterBootstrap = loadConfigOrThrow[Boolean]("whisk.cluster.use-cluster-bootstrap")
 
   val topicPrefix = loadConfigOrThrow[String](ConfigKeys.kafkaTopicsPrefix)
   val userEventTopicPrefix = loadConfigOrThrow[String](ConfigKeys.kafkaTopicsUserEventPrefix)
@@ -233,10 +230,6 @@ object Controller {
   def main(args: Array[String]): Unit = {
     implicit val actorSystem = ActorSystem("controller-actor-system")
     implicit val logger = new AkkaLogging(akka.event.Logging.getLogger(actorSystem, this))
-    if (useClusterBootstrap) {
-      AkkaManagement(actorSystem).start()
-      ClusterBootstrap(actorSystem).start()
-    }
     start(args)
   }
 
diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/loadBalancer/ShardingContainerPoolBalancer.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/loadBalancer/ShardingContainerPoolBalancer.scala
index b33cfa4d753..20f1581de7f 100644
--- a/core/controller/src/main/scala/org/apache/openwhisk/core/loadBalancer/ShardingContainerPoolBalancer.scala
+++ b/core/controller/src/main/scala/org/apache/openwhisk/core/loadBalancer/ShardingContainerPoolBalancer.scala
@@ -24,6 +24,8 @@ import java.util.concurrent.ThreadLocalRandom
 import akka.actor.{Actor, ActorSystem, Cancellable, Props}
 import akka.cluster.ClusterEvent._
 import akka.cluster.{Cluster, Member, MemberStatus}
+import akka.management.scaladsl.AkkaManagement
+import akka.management.cluster.bootstrap.ClusterBootstrap
 import org.apache.openwhisk.common.InvokerState.{Healthy, Offline, Unhealthy, Unresponsive}
 import pureconfig._
 import pureconfig.generic.auto._
@@ -152,7 +154,11 @@ class ShardingContainerPoolBalancer(
     extends CommonLoadBalancer(config, feedFactory, controllerInstance) {
 
   /** Build a cluster of all loadbalancers */
-  private val cluster: Option[Cluster] = if (loadConfigOrThrow[ClusterConfig](ConfigKeys.cluster).useClusterBootstrap || loadConfigOrThrow[Seq[String]]("akka.cluster.seed-nodes").nonEmpty) {
+  private val cluster: Option[Cluster] = if (loadConfigOrThrow[ClusterConfig](ConfigKeys.cluster).useClusterBootstrap) {
+    AkkaManagement(actorSystem).start()
+    ClusterBootstrap(actorSystem).start()
+    Some(Cluster(actorSystem))
+  } else if (loadConfigOrThrow[Seq[String]]("akka.cluster.seed-nodes").nonEmpty) {  
     Some(Cluster(actorSystem))
   } else {
     None