From 9cc14954eaffd874a035c6d029b4489bb80eda84 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Fri, 26 Jan 2024 16:04:54 +0100 Subject: [PATCH] chore: Attempt to fix tests by starting single servers before --- build.sbt | 47 +++++--- .../renku/redis/client/util/RedisServer.scala | 93 -------------- .../renku/redis/client/util/RedisSpec.scala | 1 + .../renku/solr/client/util/SolrServer.scala | 113 ------------------ .../io/renku/solr/client/util/SolrSpec.scala | 4 +- project/DbTestPlugin.scala | 48 ++++++++ project/RedisServer.scala | 78 +++++++++--- project/SolrServer.scala | 98 ++++++++++++--- 8 files changed, 230 insertions(+), 252 deletions(-) delete mode 100644 modules/redis-client/src/test/scala/io/renku/redis/client/util/RedisServer.scala delete mode 100644 modules/solr-client/src/test/scala/io/renku/solr/client/util/SolrServer.scala create mode 100644 project/DbTestPlugin.scala diff --git a/build.sbt b/build.sbt index e449bce5..f27b43f8 100644 --- a/build.sbt +++ b/build.sbt @@ -27,7 +27,7 @@ releaseVersionBump := sbtrelease.Version.Bump.Minor releaseIgnoreUntrackedFiles := true releaseTagName := (ThisBuild / version).value -addCommandAlias("ci", "; lint; test; publishLocal") +addCommandAlias("ci", "; lint; dbTests; publishLocal") addCommandAlias( "lint", "; scalafmtSbtCheck; scalafmtCheckAll;" // Compile/scalafix --check; Test/scalafix --check @@ -37,6 +37,7 @@ addCommandAlias("fix", "; scalafmtSbt; scalafmtAll") // ; Compile/scalafix; Test lazy val root = project .in(file(".")) .withId("renku-search") + .enablePlugins(DbTestPlugin) .settings( publish / skip := true, publishTo := Some( @@ -63,13 +64,29 @@ lazy val commons = project Dependencies.catsEffect ++ Dependencies.fs2Core ++ Dependencies.scodecBits ++ - Dependencies.scribe + Dependencies.scribe, + Test / sourceGenerators += Def.task { + val sourceDir = + (LocalRootProject / baseDirectory).value / "project" + val sources = Seq( + sourceDir / "RedisServer.scala", + sourceDir / "SolrServer.scala" + ) // IO.listFiles(sourceDir) + val targetDir = (Test / sourceManaged).value / "servers" + IO.createDirectory(targetDir) + + val targets = sources.map(s => targetDir / s.name) + IO.copy(sources.zip(targets)) + targets + }.taskValue ) .enablePlugins(AutomateHeaderPlugin) + .disablePlugins(DbTestPlugin) lazy val http4sBorer = project .in(file("modules/http4s-borer")) .enablePlugins(AutomateHeaderPlugin) + .disablePlugins(DbTestPlugin) .withId("http4s-borer") .settings(commonSettings) .settings( @@ -85,6 +102,7 @@ lazy val httpClient = project .in(file("modules/http-client")) .withId("http-client") .enablePlugins(AutomateHeaderPlugin) + .disablePlugins(DbTestPlugin) .settings(commonSettings) .settings( name := "http-client", @@ -104,8 +122,6 @@ lazy val redisClient = project .settings(commonSettings) .settings( name := "redis-client", - Test / testOptions += Tests.Setup(RedisServer.start), - Test / testOptions += Tests.Cleanup(RedisServer.stop), libraryDependencies ++= Dependencies.catsCore ++ Dependencies.catsEffect ++ @@ -113,6 +129,9 @@ lazy val redisClient = project Dependencies.redis4CatsStreams ) .enablePlugins(AutomateHeaderPlugin) + .dependsOn( + commons % "test->test" + ) lazy val solrClient = project .in(file("modules/solr-client")) @@ -121,15 +140,14 @@ lazy val solrClient = project .settings(commonSettings) .settings( name := "solr-client", - Test / testOptions += Tests.Setup(SolrServer.start), - Test / testOptions += Tests.Cleanup(SolrServer.stop), libraryDependencies ++= Dependencies.catsCore ++ Dependencies.catsEffect ++ Dependencies.http4sClient ) .dependsOn( - httpClient % "compile->compile;test->test" + httpClient % "compile->compile;test->test", + commons % "test->test" ) lazy val searchSolrClient = project @@ -139,19 +157,19 @@ lazy val searchSolrClient = project .settings(commonSettings) .settings( name := "search-solr-client", - Test / testOptions += Tests.Setup(SolrServer.start), - Test / testOptions += Tests.Cleanup(SolrServer.stop), libraryDependencies ++= Dependencies.catsCore ++ Dependencies.catsEffect ) .dependsOn( avroCodec % "compile->compile;test->test", - solrClient % "compile->compile;test->test" + solrClient % "compile->compile;test->test", + commons % "test->test" ) lazy val avroCodec = project .in(file("modules/avro-codec")) + .disablePlugins(DbTestPlugin) .settings(commonSettings) .settings( name := "avro-codec", @@ -171,19 +189,14 @@ lazy val messages = project avroCodec % "compile->compile;test->test" ) .enablePlugins(AvroCodeGen, AutomateHeaderPlugin) + .disablePlugins(DbTestPlugin) lazy val searchProvision = project .in(file("modules/search-provision")) .withId("search-provision") .settings(commonSettings) .settings( - name := "search-provision", - Test / testOptions += Tests.Setup { cl => - RedisServer.start(cl); SolrServer.start(cl) - }, - Test / testOptions += Tests.Cleanup { cl => - RedisServer.stop(cl); SolrServer.stop(cl) - } + name := "search-provision" ) .dependsOn( commons % "compile->compile;test->test", diff --git a/modules/redis-client/src/test/scala/io/renku/redis/client/util/RedisServer.scala b/modules/redis-client/src/test/scala/io/renku/redis/client/util/RedisServer.scala deleted file mode 100644 index f5b8d541..00000000 --- a/modules/redis-client/src/test/scala/io/renku/redis/client/util/RedisServer.scala +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright 2024 Swiss Data Science Center (SDSC) - * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and - * Eidgenössische Technische Hochschule Zürich (ETHZ). - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.renku.redis.client.util - -import cats.syntax.all._ - -import java.util.concurrent.atomic.AtomicBoolean -import scala.sys.process._ -import scala.util.Try - -object RedisServer extends RedisServer("graph", port = 6379) - -class RedisServer(module: String, port: Int) { - - val url: String = s"redis://localhost:$port" - - // When using a local Redis for development, use this env variable - // to not start a Redis server via docker for the tests - private val skipServer: Boolean = sys.env.contains("NO_REDIS") - - private val containerName = s"$module-test-redis" - private val image = "redis:7.2.4-alpine" - private val startCmd = s"""|docker run --rm - |--name $containerName - |-p $port:6379 - |-d $image""".stripMargin - private val isRunningCmd = s"docker container ls --filter 'name=$containerName'" - private val stopCmd = s"docker stop -t5 $containerName" - private val readyCmd = "redis-cli -h 127.0.0.1 -p 6379 PING" - private val isReadyCmd = s"docker exec $containerName sh -c '$readyCmd'" - private val wasRunning = new AtomicBoolean(false) - - def start(): Unit = synchronized { - if (skipServer) println("Not starting Redis via docker") - else if (checkRunning) () - else { - println(s"Starting Redis container for '$module' from '$image' image") - startContainer() - var rc = 1 - while (rc != 0) { - Thread.sleep(500) - rc = isReadyCmd.! - if (rc == 0) println(s"Redis container for '$module' started on port $port") - } - } - } - - private def checkRunning: Boolean = { - val out = isRunningCmd.lazyLines.toList - val isRunning = out.exists(_ contains containerName) - wasRunning.set(isRunning) - isRunning - } - - private def startContainer(): Unit = { - val retryOnContainerFailedToRun: Throwable => Unit = { - case ex if ex.getMessage contains "Nonzero exit value: 125" => - Thread.sleep(500); start() - case ex => throw ex - } - Try(startCmd.!!).fold(retryOnContainerFailedToRun, _ => ()) - } - - def stop(): Unit = - if (!skipServer && !wasRunning.get()) { - println(s"Stopping Redis container for '$module'") - stopCmd.!! - () - } - - def forceStop(): Unit = - if (!skipServer) { - println(s"Stopping Redis container for '$module'") - stopCmd.!! - () - } -} diff --git a/modules/redis-client/src/test/scala/io/renku/redis/client/util/RedisSpec.scala b/modules/redis-client/src/test/scala/io/renku/redis/client/util/RedisSpec.scala index 25c2ecb6..3d49065a 100644 --- a/modules/redis-client/src/test/scala/io/renku/redis/client/util/RedisSpec.scala +++ b/modules/redis-client/src/test/scala/io/renku/redis/client/util/RedisSpec.scala @@ -28,6 +28,7 @@ import dev.profunktor.redis4cats.{Redis, RedisCommands} import io.lettuce.core.RedisConnectionException import io.renku.queue.client.QueueClient import io.renku.redis.client.RedisQueueClient +import io.renku.servers.RedisServer trait RedisSpec: self: munit.Suite => diff --git a/modules/solr-client/src/test/scala/io/renku/solr/client/util/SolrServer.scala b/modules/solr-client/src/test/scala/io/renku/solr/client/util/SolrServer.scala deleted file mode 100644 index 311e809e..00000000 --- a/modules/solr-client/src/test/scala/io/renku/solr/client/util/SolrServer.scala +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright 2024 Swiss Data Science Center (SDSC) - * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and - * Eidgenössische Technische Hochschule Zürich (ETHZ). - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.renku.solr.client.util - -import cats.syntax.all.* -import org.http4s.Uri - -import java.util.concurrent.atomic.AtomicBoolean -import scala.sys.process.* -import scala.util.Try - -object SolrServer extends SolrServer("graph", port = 8983) - -class SolrServer(module: String, port: Int) { - - val url: Uri = Uri.unsafeFromString(s"http://localhost:$port") - - // When using a local Solr for development, use this env variable - // to not start a Solr server via docker for the tests - private val skipServer: Boolean = sys.env.contains("NO_SOLR") - - private val containerName = s"$module-test-solr" - private val image = "solr:9.4.1-slim" - val genericCoreName = "core-test" - val searchCoreName = "search-core-test" - private val cores = Set(genericCoreName, searchCoreName) - private val startCmd = s"""|docker run --rm - |--name $containerName - |-p $port:8983 - |-d $image""".stripMargin - private val isRunningCmd = s"docker container ls --filter 'name=$containerName'" - private val stopCmd = s"docker stop -t5 $containerName" - private def readyCmd(core: String) = - s"curl http://localhost:8983/solr/$core/select?q=*:* --no-progress-meter --fail 1> /dev/null" - private def isReadyCmd(core: String) = - s"docker exec $containerName sh -c '${readyCmd(core)}'" - private def createCore(core: String) = s"precreate-core $core" - private def createCoreCmd(core: String) = - s"docker exec $containerName sh -c '${createCore(core)}'" - private val wasRunning = new AtomicBoolean(false) - - def start(): Unit = synchronized { - if (skipServer) println("Not starting Solr via docker") - else if (checkRunning) () - else { - println(s"Starting Solr container for '$module' from '$image' image") - startContainer() - waitForCoresToBeReady() - } - } - - private def waitForCoresToBeReady(): Unit = - var rc = 1 - while (rc != 0) { - Thread.sleep(500) - rc = checkCoresReady - if (rc == 0) println(s"Solr container for '$module' ready on port $port") - } - - private def checkCoresReady = - cores.foldLeft(0)((rc, core) => if (rc == 0) isReadyCmd(core).! else rc) - - private def checkRunning: Boolean = { - val out = isRunningCmd.lazyLines.toList - val isRunning = out.exists(_ contains containerName) - wasRunning.set(isRunning) - if (isRunning) waitForCoresToBeReady() - isRunning - } - - private def startContainer(): Unit = { - val retryOnContainerFailedToRun: Throwable => Unit = { - case ex if ex.getMessage contains "Nonzero exit value: 125" => - Thread.sleep(500); start() - case ex => throw ex - } - Try(startCmd.!!).fold(retryOnContainerFailedToRun, _ => ()) - val rcs = cores.map(c => c -> createCoreCmd(c).!) - println( - s"Created solr cores: ${rcs.map { case (core, rc) => s"'$core' ($rc)" }.mkString(", ")}" - ) - } - - def stop(): Unit = - if (!skipServer && !wasRunning.get()) { - println(s"Stopping Solr container for '$module'") - stopCmd.!! - () - } - - def forceStop(): Unit = - if (!skipServer) { - println(s"Stopping Solr container for '$module'") - stopCmd.!! - () - } -} diff --git a/modules/solr-client/src/test/scala/io/renku/solr/client/util/SolrSpec.scala b/modules/solr-client/src/test/scala/io/renku/solr/client/util/SolrSpec.scala index 7ce5abb0..06249b87 100644 --- a/modules/solr-client/src/test/scala/io/renku/solr/client/util/SolrSpec.scala +++ b/modules/solr-client/src/test/scala/io/renku/solr/client/util/SolrSpec.scala @@ -19,7 +19,9 @@ package io.renku.solr.client.util import cats.effect.* +import io.renku.servers.SolrServer import io.renku.solr.client.{SolrClient, SolrConfig} +import org.http4s.Uri import scala.concurrent.duration.Duration @@ -28,7 +30,7 @@ trait SolrSpec: protected lazy val server: SolrServer = SolrServer protected lazy val solrConfig: SolrConfig = SolrConfig( - server.url / "solr", + Uri.unsafeFromString(server.url) / "solr", server.genericCoreName, commitWithin = Some(Duration.Zero), logMessageBodies = true diff --git a/project/DbTestPlugin.scala b/project/DbTestPlugin.scala new file mode 100644 index 00000000..3ea9421d --- /dev/null +++ b/project/DbTestPlugin.scala @@ -0,0 +1,48 @@ +import sbt._ +import sbt.Keys._ +import _root_.io.renku.servers._ + +object DbTestPlugin extends AutoPlugin { + + object autoImport { + val dbTests = taskKey[Unit]("Run the tests with databases turned on") + } + + import autoImport._ + + // AllRequirements makes it enabled on all sub projects by default + // It is possible to use `.disablePlugins(DbTestPlugin)` to disable + // it + override def trigger = PluginTrigger.AllRequirements + + override def projectSettings: Seq[Def.Setting[_]] = Seq( + Test / dbTests := { + Def + .sequential( + Def.task { + val logger = streams.value.log + logger.info("Starting REDIS server") + RedisServer.start() + logger.info("Starting SOLR server") + SolrServer.start() + logger.info("Running tests") + }, + (Test / test).all(ScopeFilter(inAggregates(ThisProject))), + Def.task { + val logger = streams.value.log + logger.info("Stopping SOLR server") + SolrServer.forceStop() + logger.info("Stopping REDIS server") + RedisServer.forceStop() + } + ) + .value + }, + // We need to disable running the `dbTests` on all aggregates, + // otherwise it would try starting/stopping servers again and + // again. The `all(ScopeFilter(inAggregates(ThisProject)))` makes + // sure that tests run on all aggregates anyways- but + // starting/stopping servers only once. + dbTests / aggregate := false + ) +} diff --git a/project/RedisServer.scala b/project/RedisServer.scala index dd22014f..19042ea5 100644 --- a/project/RedisServer.scala +++ b/project/RedisServer.scala @@ -15,28 +15,78 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package io.renku.servers -import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicBoolean +import scala.sys.process.* import scala.util.Try -object RedisServer { +object RedisServer extends RedisServer("graph", port = 6379) - private val startRequests = new AtomicInteger(0) +@annotation.nowarn() +class RedisServer(module: String, port: Int) { - def start: ClassLoader => Unit = { cl => - if (startRequests.getAndIncrement() == 0) call("start")(cl) + val url: String = s"redis://localhost:$port" + + // When using a local Redis for development, use this env variable + // to not start a Redis server via docker for the tests + private val skipServer: Boolean = sys.env.contains("NO_REDIS") + + private val containerName = s"$module-test-redis" + private val image = "redis:7.2.4-alpine" + private val startCmd = s"""|docker run --rm + |--name $containerName + |-p $port:6379 + |-d $image""".stripMargin + private val isRunningCmd = + Seq("docker", "container", "ls", "--filter", s"name=$containerName") + private val stopCmd = s"docker stop -t5 $containerName" + private val readyCmd = "redis-cli -h 127.0.0.1 -p 6379 PING" + private val isReadyCmd = + Seq("docker", "exec", containerName, "sh", "-c", readyCmd) + private val wasStartedHere = new AtomicBoolean(false) + + def start(): Unit = synchronized { + if (skipServer) println("Not starting Redis via docker") + else if (checkRunning) () + else { + println(s"Starting Redis container for '$module' from '$image' image") + startContainer() + var rc = 1 + while (rc != 0) { + Thread.sleep(500) + rc = Process(isReadyCmd).! + if (rc == 0) println(s"Redis container for '$module' started on port $port") + else println(s"IsReadyCmd returned $rc") + } + } } - def stop: ClassLoader => Unit = { cl => - if (startRequests.decrementAndGet() == 0) - Try(call("forceStop")(cl)) - .recover { case err => err.printStackTrace() } + private def checkRunning: Boolean = { + val out = isRunningCmd.lineStream_!.take(200).toList + out.exists(_ contains containerName) } - private def call(methodName: String): ClassLoader => Unit = classLoader => { - val clazz = classLoader.loadClass("io.renku.redis.client.util.RedisServer$") - val method = clazz.getMethod(methodName) - val instance = clazz.getField("MODULE$").get(null) - method.invoke(instance) + private def startContainer(): Unit = { + val retryOnContainerFailedToRun: Throwable => Unit = { + case ex if ex.getMessage contains "Nonzero exit value: 125" => + Thread.sleep(500); start() + case ex => throw ex + } + Try(startCmd.!!).fold(retryOnContainerFailedToRun, _ => wasStartedHere.set(true)) } + + def stop(): Unit = + if (!skipServer && wasStartedHere.get()) { + println(s"Stopping Redis container for '$module'") + stopCmd.!! + () + } + + def forceStop(): Unit = + if (!skipServer) { + println(s"Stopping Redis container for '$module'") + stopCmd.!! + () + } } diff --git a/project/SolrServer.scala b/project/SolrServer.scala index 460fe70b..05727524 100644 --- a/project/SolrServer.scala +++ b/project/SolrServer.scala @@ -16,27 +16,97 @@ * limitations under the License. */ -import java.util.concurrent.atomic.AtomicInteger +package io.renku.servers + +import java.util.concurrent.atomic.AtomicBoolean +import scala.sys.process.* import scala.util.Try -object SolrServer { +object SolrServer extends SolrServer("graph", port = 8983) + +@annotation.nowarn() +class SolrServer(module: String, port: Int) { + + val url: String = s"http://localhost:$port" + + // When using a local Solr for development, use this env variable + // to not start a Solr server via docker for the tests + private val skipServer: Boolean = sys.env.contains("NO_SOLR") - private val startRequests = new AtomicInteger(0) + private val containerName = s"$module-test-solr" + private val image = "solr:9.4.1-slim" + val genericCoreName = "core-test" + val searchCoreName = "search-core-test" + private val cores = Set(genericCoreName, searchCoreName) + private val startCmd = s"""|docker run --rm + |--name $containerName + |-p $port:8983 + |-d $image""".stripMargin + private val isRunningCmd = + Seq("docker", "container", "ls", "--filter", s"name=$containerName") + private val stopCmd = s"docker stop -t5 $containerName" + private def readyCmd(core: String) = + s"curl http://localhost:8983/solr/$core/select?q=*:* --no-progress-meter --fail 1> /dev/null" + private def isReadyCmd(core: String) = + Seq("docker", "exec", containerName, "sh", "-c", readyCmd(core)) + private def createCore(core: String) = s"precreate-core $core" + private def createCoreCmd(core: String) = + Seq("docker", "exec", containerName, "sh", "-c", createCore(core)) + private val wasStartedHere = new AtomicBoolean(false) - def start: ClassLoader => Unit = { cl => - if (startRequests.getAndIncrement() == 0) call("start")(cl) + def start(): Unit = + if (skipServer) println("Not starting Solr via docker") + else if (checkRunning) () + else { + println(s"Starting Solr container for '$module' from '$image' image") + startContainer() + waitForCoresToBeReady() + } + + private def waitForCoresToBeReady(): Unit = { + var rc = 1 + while (rc != 0) { + Thread.sleep(500) + rc = checkCoresReady + if (rc == 0) println(s"Solr container for '$module' ready on port $port") + } } - def stop: ClassLoader => Unit = { cl => - if (startRequests.decrementAndGet() == 0) - Try(call("forceStop")(cl)) - .recover { case err => err.printStackTrace() } + private def checkCoresReady = + cores.foldLeft(0)((rc, core) => if (rc == 0) isReadyCmd(core).! else rc) + + private def checkRunning: Boolean = { + val out = isRunningCmd.lineStream_!.take(20).toList + val isRunning = out.exists(_ contains containerName) + if (isRunning) waitForCoresToBeReady() + isRunning } - private def call(methodName: String): ClassLoader => Unit = classLoader => { - val clazz = classLoader.loadClass("io.renku.solr.client.util.SolrServer$") - val method = clazz.getMethod(methodName) - val instance = clazz.getField("MODULE$").get(null) - method.invoke(instance) + private def startContainer(): Unit = { + val retryOnContainerFailedToRun: Throwable => Unit = { + case ex if ex.getMessage contains "Nonzero exit value: 125" => + Thread.sleep(500); start() + case ex => throw ex + } + Try(startCmd.!!).fold(retryOnContainerFailedToRun, _ => wasStartedHere.set(true)) + val rcs = cores.map(c => c -> createCoreCmd(c).!) + println( + s"Created solr cores: ${rcs.map { case (core, rc) => s"'$core' ($rc)" }.mkString(", ")}" + ) } + + def stop(): Unit = + if (!skipServer && !wasStartedHere.get()) () + else { + println(s"Stopping Solr container for '$module'") + stopCmd.!! + () + } + + def forceStop(): Unit = + if (!skipServer) { + println(s"Stopping Solr container for '$module'") + stopCmd.!! + () + } }