From 02230d04a8ed0ad54c8a9851386552b955e20d51 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Tue, 30 Jan 2024 14:13:30 +0100 Subject: [PATCH 1/3] Publish docker images for microservices --- .github/workflows/ci.yml | 16 ++++---- .github/workflows/release.yml | 72 +++++++++++++++++++++++++++++++++ build.sbt | 13 ++++-- project/DockerImagePlugin.scala | 42 +++++++++++++++++++ project/plugins.sbt | 2 + 5 files changed, 134 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100644 project/DockerImagePlugin.scala diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ad204f1..852a3c70 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - java: [ 'openjdk@1.17' ] + java: [ 17 ] steps: - uses: actions/checkout@v4.1.1 with: @@ -24,13 +24,13 @@ jobs: # uses: coursier/cache-action@v6 - name: sbt ci ${{ github.ref }} run: sbt -mem 2048 ci - # - name: Log in to Docker Hub - # uses: docker/login-action@v2 - # with: - # username: ${{ secrets.RENKU_DOCKER_USERNAME }} - # password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - # - name: sbt docker:publishLocal - # run: sbt -mem 2048 cli/Docker/publishLocal + - name: Log in to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.RENKU_DOCKER_USERNAME }} + password: ${{ secrets.RENKU_DOCKER_PASSWORD }} + - name: sbt docker:publishLocal + run: sbt -mem 2048 search-provision/Docker/publishLocal search-api/Docker/publishLocal ci: runs-on: ubuntu-latest needs: [ci-matrix] diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..d2085286 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,72 @@ +name: Release +on: + push: + branches: [ main ] + release: + types: [ published ] + +jobs: + release: + runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + java: [ 17 ] + steps: + - uses: actions/checkout@v4.1.1 + with: + fetch-depth: 0 + - uses: olafurpg/setup-scala@v14 + with: + java-version: ${{ matrix.java }} + + - name: Set current version + id: version + shell: bash + run: | + sbt 'renku-search/writeVersion' + RS_VERSION=$(cat target/version.txt) + echo "RS_VERSION=${RS_VERSION}" >> $GITHUB_ENV + + if [ -z "${RS_VERSION}" ]; then + echo "Version not set!" + exit 1 + fi + + - name: Create zip packages + run: sbt -mem 2048 search-provision/Universal/packageBin search-api/Universal/packageBin + + - name: Publish Release (${{ env.RS_VERSION }}) + uses: softprops/action-gh-release@v1 + if: startsWith(github.ref, 'refs/tags/') + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + with: + files: | + modules/search-provision/target/universal/search-provision-${{ env.RS_VERSION }}.zip + modules/search-api/target/universal/search-api-${{ env.RS_VERSION }}.zip + - name: Publish Pre-Release + uses: ncipollo/release-action@v1 + if: ${{ github.ref }} == 'refs/heads/main' + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + with: + token: "${{ secrets.GITHUB_TOKEN }}" + prerelease: true + allowUpdates: true + tag: "nightly" + commit: "main" + body: "Floating tag associating the latest build from the main branch" + name: "renku search nightly" + replacesArtifacts: true + removeArtifacts: true + artifacts: | + modules/search-provision/target/universal/search-provision-${{ env.RS_VERSION }}.zip, + modules/search-api/target/universal/search-api-${{ env.RS_VERSION }}.zip + - name: Log in to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.RENKU_DOCKER_USERNAME }} + password: ${{ secrets.RENKU_DOCKER_PASSWORD }} + - name: sbt Docker/publish + run: sbt -mem 2048 search-provision/Docker/publish search-api/Docker/publish diff --git a/build.sbt b/build.sbt index 574bcea7..6a09d629 100644 --- a/build.sbt +++ b/build.sbt @@ -34,6 +34,8 @@ addCommandAlias( ) addCommandAlias("fix", "; scalafmtSbt; scalafmtAll") // ; Compile/scalafix; Test/scalafix +val writeVersion = taskKey[Unit]("Write version into a file for CI to pick up") + lazy val root = project .in(file(".")) .withId("renku-search") @@ -42,7 +44,12 @@ lazy val root = project publish / skip := true, publishTo := Some( Resolver.file("Unused transient repository", file("target/unusedrepo")) - ) + ), + writeVersion := { + val out = (LocalRootProject / target).value / "version.txt" + val versionStr = version.value + IO.write(out, versionStr) + } ) .aggregate( commons, @@ -222,7 +229,7 @@ lazy val searchProvision = project redisClient % "compile->compile;test->test", searchSolrClient % "compile->compile;test->test" ) - .enablePlugins(AutomateHeaderPlugin) + .enablePlugins(AutomateHeaderPlugin, DockerImagePlugin) lazy val searchApi = project .in(file("modules/search-api")) @@ -240,7 +247,7 @@ lazy val searchApi = project http4sAvro % "compile->compile;test->test", searchSolrClient % "compile->compile;test->test" ) - .enablePlugins(AutomateHeaderPlugin) + .enablePlugins(AutomateHeaderPlugin, DockerImagePlugin) lazy val commonSettings = Seq( organization := "io.renku", diff --git a/project/DockerImagePlugin.scala b/project/DockerImagePlugin.scala new file mode 100644 index 00000000..59cd9ec9 --- /dev/null +++ b/project/DockerImagePlugin.scala @@ -0,0 +1,42 @@ +import com.typesafe.sbt.packager.docker._ +import com.typesafe.sbt.packager.Keys._ +import com.typesafe.sbt.packager.archetypes.JavaServerAppPackaging +import sbt._ +import sbt.Keys._ +import com.github.sbt.git.SbtGit.git +import sbtdynver._ +import sbtdynver.DynVerPlugin.autoImport._ + +/** Sets default docker image settings for sbt-native-packager's `DockerPlugin`. */ +object DockerImagePlugin extends AutoPlugin { + + // Load DockerPlugin and JavaServerAppPackaging whenever this plugin is enabled + override def requires = DockerPlugin && JavaServerAppPackaging && DynVerPlugin + override def trigger = allRequirements + + import DockerPlugin.autoImport._ + + val dockerSettings = Seq( + dockerUpdateLatest := true, + // dockerEntrypoint := Seq(s"bin/${executableScriptName.value}", "-Duser.timezone=UTC", "$JAVA_OPTS"), + dockerBaseImage := s"eclipse-temurin:21-jre", + // derive a package name + Docker / packageName := (Compile / name).value, + dockerRepository := Some("docker.io"), + dockerUsername := Some("renku"), + + // temporarily use git hash as image tag + Docker / version := git.gitHeadCommit.value + .map(_.take(12)) + .getOrElse((Compile / version).value) + ) + + private def imageTag(out: Option[GitDescribeOutput], headCommit: Option[String]) = + out match { + case Some(d) if d.isCleanAfterTag => Some(d.ref.dropPrefix) + case _ => headCommit.map(_.take(12)) + } + + override def projectSettings = + dockerSettings +} diff --git a/project/plugins.sbt b/project/plugins.sbt index 84418a0f..80f5b1f0 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -16,6 +16,8 @@ * limitations under the License. */ +addSbtPlugin("com.github.sbt" % "sbt-dynver" % "5.0.1") +addSbtPlugin("com.github.sbt" % "sbt-git" % "2.0.1") addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.9.16") addSbtPlugin("com.github.sbt" % "sbt-release" % "1.4.0") addSbtPlugin("com.julianpeeters" % "sbt-avrohugger" % "2.8.2") From ea87d286f387f3c2dea661ce4ac57faa17501f47 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Tue, 30 Jan 2024 14:42:01 +0100 Subject: [PATCH 2/3] Avoid going in an endless loop --- project/RedisServer.scala | 7 ++++++- project/SolrServer.scala | 6 +++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/project/RedisServer.scala b/project/RedisServer.scala index 19042ea5..223f646b 100644 --- a/project/RedisServer.scala +++ b/project/RedisServer.scala @@ -53,12 +53,17 @@ class RedisServer(module: String, port: Int) { println(s"Starting Redis container for '$module' from '$image' image") startContainer() var rc = 1 - while (rc != 0) { + val maxTries = 500 + var counter = 0 + while (rc != 0 && counter < maxTries) { + counter += 1 Thread.sleep(500) rc = Process(isReadyCmd).! if (rc == 0) println(s"Redis container for '$module' started on port $port") else println(s"IsReadyCmd returned $rc") } + if (rc != 0) + sys.error(s"Redis container for '$module' could not be started on port $port") } } diff --git a/project/SolrServer.scala b/project/SolrServer.scala index 05727524..654c7565 100644 --- a/project/SolrServer.scala +++ b/project/SolrServer.scala @@ -65,11 +65,15 @@ class SolrServer(module: String, port: Int) { private def waitForCoresToBeReady(): Unit = { var rc = 1 - while (rc != 0) { + val maxTries = 500 + var counter = 0 + while (rc != 0 && counter < maxTries) { + counter += 1 Thread.sleep(500) rc = checkCoresReady if (rc == 0) println(s"Solr container for '$module' ready on port $port") } + if (rc != 0) sys.error("Solr container for '$module' could not be started") } private def checkCoresReady = From ab2f0d9c11c37ff1ab6905cb0861f770770eff0c Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Tue, 30 Jan 2024 15:59:44 +0100 Subject: [PATCH 3/3] Configure microservices by env variables --- build.sbt | 27 ++++++++-- .../renku/search/config/ConfigDecoders.scala | 44 ++++++++++++++++ .../io/renku/search/config/ConfigValues.scala | 51 +++++++++++++++++++ .../scala/io/renku/queue/client/types.scala | 10 ++-- .../io/renku/redis/client/RedisUrl.scala | 2 +- .../io/renku/search/api/Microservice.scala | 21 +++----- .../io/renku/search/api/SearchApiConfig.scala | 31 +++++++++++ .../renku/search/provision/Microservice.scala | 39 +++++--------- .../provision/SearchProvisionConfig.scala | 47 +++++++++++++++++ project/Dependencies.scala | 5 ++ 10 files changed, 227 insertions(+), 50 deletions(-) create mode 100644 modules/config-values/src/main/scala/io/renku/search/config/ConfigDecoders.scala create mode 100644 modules/config-values/src/main/scala/io/renku/search/config/ConfigValues.scala create mode 100644 modules/search-api/src/main/scala/io/renku/search/api/SearchApiConfig.scala create mode 100644 modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisionConfig.scala diff --git a/build.sbt b/build.sbt index 6a09d629..f6fd2aea 100644 --- a/build.sbt +++ b/build.sbt @@ -216,18 +216,35 @@ lazy val messages = project .enablePlugins(AvroCodeGen, AutomateHeaderPlugin) .disablePlugins(DbTestPlugin) +lazy val configValues = project + .in(file("modules/config-values")) + .withId("config-values") + .settings(commonSettings) + .settings( + name := "config-values", + libraryDependencies ++= Dependencies.ciris + ) + .dependsOn( + commons % "compile->compile;test->test", + messages % "compile->compile;test->test", + redisClient % "compile->compile;test->test", + searchSolrClient % "compile->compile;test->test" + ) + lazy val searchProvision = project .in(file("modules/search-provision")) .withId("search-provision") .settings(commonSettings) .settings( - name := "search-provision" + name := "search-provision", + libraryDependencies ++= Dependencies.ciris ) .dependsOn( commons % "compile->compile;test->test", messages % "compile->compile;test->test", redisClient % "compile->compile;test->test", - searchSolrClient % "compile->compile;test->test" + searchSolrClient % "compile->compile;test->test", + configValues % "compile->compile;test->test" ) .enablePlugins(AutomateHeaderPlugin, DockerImagePlugin) @@ -239,13 +256,15 @@ lazy val searchApi = project name := "search-api", libraryDependencies ++= Dependencies.http4sDsl ++ - Dependencies.http4sServer + Dependencies.http4sServer ++ + Dependencies.ciris ) .dependsOn( commons % "compile->compile;test->test", messages % "compile->compile;test->test", http4sAvro % "compile->compile;test->test", - searchSolrClient % "compile->compile;test->test" + searchSolrClient % "compile->compile;test->test", + configValues % "compile->compile;test->test" ) .enablePlugins(AutomateHeaderPlugin, DockerImagePlugin) diff --git a/modules/config-values/src/main/scala/io/renku/search/config/ConfigDecoders.scala b/modules/config-values/src/main/scala/io/renku/search/config/ConfigDecoders.scala new file mode 100644 index 00000000..54d3bf75 --- /dev/null +++ b/modules/config-values/src/main/scala/io/renku/search/config/ConfigDecoders.scala @@ -0,0 +1,44 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.config + +import cats.syntax.all.* +import ciris.{ConfigDecoder, ConfigError} +import io.renku.queue.client.QueueName +import io.renku.redis.client.RedisUrl +import org.http4s.Uri + +import scala.concurrent.duration.{Duration, FiniteDuration} + +trait ConfigDecoders: + given ConfigDecoder[String, Uri] = + ConfigDecoder[String].mapEither { (_, s) => + Uri.fromString(s).leftMap(err => ConfigError(err.getMessage)) + } + + given ConfigDecoder[String, FiniteDuration] = + ConfigDecoder[String].mapOption("duration") { s => + Duration.unapply(s).map(Duration.apply.tupled).filter(_.isFinite) + } + + given ConfigDecoder[String, RedisUrl] = + ConfigDecoder[String].map(s => RedisUrl(s)) + + given ConfigDecoder[String, QueueName] = + ConfigDecoder[String].map(s => QueueName(s)) diff --git a/modules/config-values/src/main/scala/io/renku/search/config/ConfigValues.scala b/modules/config-values/src/main/scala/io/renku/search/config/ConfigValues.scala new file mode 100644 index 00000000..bccf0a68 --- /dev/null +++ b/modules/config-values/src/main/scala/io/renku/search/config/ConfigValues.scala @@ -0,0 +1,51 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.config + +import cats.syntax.all.* +import ciris.* +import io.renku.queue.client.QueueName +import io.renku.redis.client.RedisUrl +import io.renku.solr.client.SolrConfig +import org.http4s.Uri + +import scala.concurrent.duration.FiniteDuration + +object ConfigValues extends ConfigDecoders: + + private val prefix = "RS" + + val redisUrl: ConfigValue[Effect, RedisUrl] = + env(s"${prefix}_REDIS_URL").default("redis://localhost:6379").as[RedisUrl] + + val eventsQueueName: ConfigValue[Effect, QueueName] = + env(s"${prefix}_REDIS_QUEUE_NAME").default("events").as[QueueName] + + val retryOnErrorDelay: ConfigValue[Effect, FiniteDuration] = + env(s"${prefix}_RETRY_ON_ERROR_DELAY").default("2 seconds").as[FiniteDuration] + + val solrConfig: ConfigValue[Effect, SolrConfig] = { + val url = env(s"${prefix}_SOLR_URL").default("http://localhost:8983/solr").as[Uri] + val core = env(s"${prefix}_SOLR_CORE").default("search-core-test") + val defaultCommit = + env(s"${prefix}_SOLR_DEFAULT_COMMIT_WITHIN").default("0").as[FiniteDuration].option + val logMessageBodies = + env(s"${prefix}_SOLR_LOG_MESSAGE_BODIES").default("false").as[Boolean] + (url, core, defaultCommit, logMessageBodies).mapN(SolrConfig.apply) + } diff --git a/modules/redis-client/src/main/scala/io/renku/queue/client/types.scala b/modules/redis-client/src/main/scala/io/renku/queue/client/types.scala index 918c283d..e0043777 100644 --- a/modules/redis-client/src/main/scala/io/renku/queue/client/types.scala +++ b/modules/redis-client/src/main/scala/io/renku/queue/client/types.scala @@ -19,11 +19,9 @@ package io.renku.queue.client opaque type QueueName = String -object QueueName { - def apply(v: String): QueueName = new QueueName(v) -} +object QueueName: + def apply(v: String): QueueName = v opaque type ClientId = String -object ClientId { - def apply(v: String): ClientId = new ClientId(v) -} +object ClientId: + def apply(v: String): ClientId = v diff --git a/modules/redis-client/src/main/scala/io/renku/redis/client/RedisUrl.scala b/modules/redis-client/src/main/scala/io/renku/redis/client/RedisUrl.scala index 47885f3d..956c2a3c 100644 --- a/modules/redis-client/src/main/scala/io/renku/redis/client/RedisUrl.scala +++ b/modules/redis-client/src/main/scala/io/renku/redis/client/RedisUrl.scala @@ -20,5 +20,5 @@ package io.renku.redis.client opaque type RedisUrl = String object RedisUrl { - def apply(v: String): RedisUrl = new RedisUrl(v) + def apply(v: String): RedisUrl = v } diff --git a/modules/search-api/src/main/scala/io/renku/search/api/Microservice.scala b/modules/search-api/src/main/scala/io/renku/search/api/Microservice.scala index ef4ae132..249bab14 100644 --- a/modules/search-api/src/main/scala/io/renku/search/api/Microservice.scala +++ b/modules/search-api/src/main/scala/io/renku/search/api/Microservice.scala @@ -19,22 +19,15 @@ package io.renku.search.api import cats.effect.{ExitCode, IO, IOApp} -import cats.syntax.all.* -import io.renku.solr.client.SolrConfig -import org.http4s.implicits.* - -import scala.concurrent.duration.Duration object Microservice extends IOApp: - private val solrConfig = SolrConfig( - baseUrl = uri"http://localhost:8983" / "solr", - core = "search-core-test", - commitWithin = Some(Duration.Zero), - logMessageBodies = true - ) + private val loadConfig = SearchApiConfig.config.load[IO] override def run(args: List[String]): IO[ExitCode] = - (createHttpApp >>= HttpServer.build).use(_ => IO.never).as(ExitCode.Success) - - private def createHttpApp = HttpApplication[IO](solrConfig) + for { + config <- loadConfig + _ <- HttpApplication[IO](config.solrConfig) + .flatMap(HttpServer.build) + .use(_ => IO.never) + } yield ExitCode.Success diff --git a/modules/search-api/src/main/scala/io/renku/search/api/SearchApiConfig.scala b/modules/search-api/src/main/scala/io/renku/search/api/SearchApiConfig.scala new file mode 100644 index 00000000..86261ac5 --- /dev/null +++ b/modules/search-api/src/main/scala/io/renku/search/api/SearchApiConfig.scala @@ -0,0 +1,31 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.api + +import ciris.{ConfigValue, Effect} +import io.renku.search.config.ConfigValues +import io.renku.solr.client.SolrConfig + +final case class SearchApiConfig( + solrConfig: SolrConfig +) + +object SearchApiConfig: + val config: ConfigValue[Effect, SearchApiConfig] = + ConfigValues.solrConfig.map(SearchApiConfig.apply) diff --git a/modules/search-provision/src/main/scala/io/renku/search/provision/Microservice.scala b/modules/search-provision/src/main/scala/io/renku/search/provision/Microservice.scala index b932a76e..a912af65 100644 --- a/modules/search-provision/src/main/scala/io/renku/search/provision/Microservice.scala +++ b/modules/search-provision/src/main/scala/io/renku/search/provision/Microservice.scala @@ -19,47 +19,36 @@ package io.renku.search.provision import cats.effect.{ExitCode, IO, IOApp, Temporal} -import io.renku.queue.client.QueueName -import io.renku.redis.client.RedisUrl import io.renku.search.solr.schema.Migrations -import io.renku.solr.client.SolrConfig import io.renku.solr.client.migration.SchemaMigrator -import org.http4s.Uri -import org.http4s.implicits.* import scribe.Scribe import scribe.cats.* -import scala.concurrent.duration.* - object Microservice extends IOApp: - private val queueName = QueueName("events") - private val redisUrl = RedisUrl("redis://localhost:6379") - private val solrConfig = SolrConfig( - baseUrl = uri"http://localhost:8983" / "solr", - core = "search-core-test", - commitWithin = Some(Duration.Zero), - logMessageBodies = true - ) - private val retryOnErrorDelay = 2 seconds + private val loadConfig: IO[SearchProvisionConfig] = + SearchProvisionConfig.config.load[IO] override def run(args: List[String]): IO[ExitCode] = - (runSolrMigrations >> startProvisioning) - .as(ExitCode.Success) - - private def startProvisioning: IO[Unit] = - SearchProvisioner[IO](queueName, redisUrl, solrConfig) + for { + config <- loadConfig + _ <- runSolrMigrations(config) + _ <- startProvisioning(config) + } yield ExitCode.Success + + private def startProvisioning(cfg: SearchProvisionConfig): IO[Unit] = + SearchProvisioner[IO](cfg.queueName, cfg.redisUrl, cfg.solrConfig) .evalMap(_.provisionSolr.start) .use(_ => IO.never) .handleErrorWith { err => Scribe[IO].error("Starting provisioning failure, retrying", err) >> - Temporal[IO].delayBy(startProvisioning, retryOnErrorDelay) + Temporal[IO].delayBy(startProvisioning(cfg), cfg.retryOnErrorDelay) } - private def runSolrMigrations: IO[Unit] = - SchemaMigrator[IO](solrConfig) + private def runSolrMigrations(cfg: SearchProvisionConfig): IO[Unit] = + SchemaMigrator[IO](cfg.solrConfig) .use(_.migrate(Migrations.all)) .handleErrorWith { err => Scribe[IO].error("Running solr migrations failure, retrying", err) >> - Temporal[IO].delayBy(runSolrMigrations, retryOnErrorDelay) + Temporal[IO].delayBy(runSolrMigrations(cfg), cfg.retryOnErrorDelay) } diff --git a/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisionConfig.scala b/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisionConfig.scala new file mode 100644 index 00000000..9ad6517f --- /dev/null +++ b/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisionConfig.scala @@ -0,0 +1,47 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.provision + +import cats.syntax.all.* +import ciris.{ConfigValue, Effect} +import io.renku.queue.client.QueueName +import io.renku.redis.client.RedisUrl +import io.renku.search.config.ConfigValues +import io.renku.solr.client.SolrConfig + +import scala.concurrent.duration.FiniteDuration + +final case class SearchProvisionConfig( + redisUrl: RedisUrl, + queueName: QueueName, + solrConfig: SolrConfig, + retryOnErrorDelay: FiniteDuration +) + +object SearchProvisionConfig: + + val config: ConfigValue[Effect, SearchProvisionConfig] = + ( + ConfigValues.redisUrl, + ConfigValues.eventsQueueName, + ConfigValues.solrConfig, + ConfigValues.retryOnErrorDelay + ).mapN( + SearchProvisionConfig.apply + ) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 6b494de4..8b15cdf9 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -10,6 +10,7 @@ object Dependencies { val catsCore = "2.10.0" val catsEffect = "3.5.3" val catsEffectMunit = "1.0.7" + val ciris = "3.5.0" val fs2 = "3.9.4" val http4s = "0.23.25" val redis4Cats = "1.5.2" @@ -19,6 +20,10 @@ object Dependencies { val scribe = "3.13.0" } + val ciris = Seq( + "is.cir" %% "ciris" % V.ciris + ) + val borer = Seq( "io.bullet" %% "borer-core" % V.borer, "io.bullet" %% "borer-derivation" % V.borer,