diff --git a/build.sbt b/build.sbt index 6a09d629..f6fd2aea 100644 --- a/build.sbt +++ b/build.sbt @@ -216,18 +216,35 @@ lazy val messages = project .enablePlugins(AvroCodeGen, AutomateHeaderPlugin) .disablePlugins(DbTestPlugin) +lazy val configValues = project + .in(file("modules/config-values")) + .withId("config-values") + .settings(commonSettings) + .settings( + name := "config-values", + libraryDependencies ++= Dependencies.ciris + ) + .dependsOn( + commons % "compile->compile;test->test", + messages % "compile->compile;test->test", + redisClient % "compile->compile;test->test", + searchSolrClient % "compile->compile;test->test" + ) + lazy val searchProvision = project .in(file("modules/search-provision")) .withId("search-provision") .settings(commonSettings) .settings( - name := "search-provision" + name := "search-provision", + libraryDependencies ++= Dependencies.ciris ) .dependsOn( commons % "compile->compile;test->test", messages % "compile->compile;test->test", redisClient % "compile->compile;test->test", - searchSolrClient % "compile->compile;test->test" + searchSolrClient % "compile->compile;test->test", + configValues % "compile->compile;test->test" ) .enablePlugins(AutomateHeaderPlugin, DockerImagePlugin) @@ -239,13 +256,15 @@ lazy val searchApi = project name := "search-api", libraryDependencies ++= Dependencies.http4sDsl ++ - Dependencies.http4sServer + Dependencies.http4sServer ++ + Dependencies.ciris ) .dependsOn( commons % "compile->compile;test->test", messages % "compile->compile;test->test", http4sAvro % "compile->compile;test->test", - searchSolrClient % "compile->compile;test->test" + searchSolrClient % "compile->compile;test->test", + configValues % "compile->compile;test->test" ) .enablePlugins(AutomateHeaderPlugin, DockerImagePlugin) diff --git a/modules/config-values/src/main/scala/io/renku/search/config/ConfigDecoders.scala b/modules/config-values/src/main/scala/io/renku/search/config/ConfigDecoders.scala new file mode 100644 index 00000000..54d3bf75 --- /dev/null +++ b/modules/config-values/src/main/scala/io/renku/search/config/ConfigDecoders.scala @@ -0,0 +1,44 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.config + +import cats.syntax.all.* +import ciris.{ConfigDecoder, ConfigError} +import io.renku.queue.client.QueueName +import io.renku.redis.client.RedisUrl +import org.http4s.Uri + +import scala.concurrent.duration.{Duration, FiniteDuration} + +trait ConfigDecoders: + given ConfigDecoder[String, Uri] = + ConfigDecoder[String].mapEither { (_, s) => + Uri.fromString(s).leftMap(err => ConfigError(err.getMessage)) + } + + given ConfigDecoder[String, FiniteDuration] = + ConfigDecoder[String].mapOption("duration") { s => + Duration.unapply(s).map(Duration.apply.tupled).filter(_.isFinite) + } + + given ConfigDecoder[String, RedisUrl] = + ConfigDecoder[String].map(s => RedisUrl(s)) + + given ConfigDecoder[String, QueueName] = + ConfigDecoder[String].map(s => QueueName(s)) diff --git a/modules/config-values/src/main/scala/io/renku/search/config/ConfigValues.scala b/modules/config-values/src/main/scala/io/renku/search/config/ConfigValues.scala new file mode 100644 index 00000000..bccf0a68 --- /dev/null +++ b/modules/config-values/src/main/scala/io/renku/search/config/ConfigValues.scala @@ -0,0 +1,51 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.config + +import cats.syntax.all.* +import ciris.* +import io.renku.queue.client.QueueName +import io.renku.redis.client.RedisUrl +import io.renku.solr.client.SolrConfig +import org.http4s.Uri + +import scala.concurrent.duration.FiniteDuration + +object ConfigValues extends ConfigDecoders: + + private val prefix = "RS" + + val redisUrl: ConfigValue[Effect, RedisUrl] = + env(s"${prefix}_REDIS_URL").default("redis://localhost:6379").as[RedisUrl] + + val eventsQueueName: ConfigValue[Effect, QueueName] = + env(s"${prefix}_REDIS_QUEUE_NAME").default("events").as[QueueName] + + val retryOnErrorDelay: ConfigValue[Effect, FiniteDuration] = + env(s"${prefix}_RETRY_ON_ERROR_DELAY").default("2 seconds").as[FiniteDuration] + + val solrConfig: ConfigValue[Effect, SolrConfig] = { + val url = env(s"${prefix}_SOLR_URL").default("http://localhost:8983/solr").as[Uri] + val core = env(s"${prefix}_SOLR_CORE").default("search-core-test") + val defaultCommit = + env(s"${prefix}_SOLR_DEFAULT_COMMIT_WITHIN").default("0").as[FiniteDuration].option + val logMessageBodies = + env(s"${prefix}_SOLR_LOG_MESSAGE_BODIES").default("false").as[Boolean] + (url, core, defaultCommit, logMessageBodies).mapN(SolrConfig.apply) + } diff --git a/modules/redis-client/src/main/scala/io/renku/queue/client/types.scala b/modules/redis-client/src/main/scala/io/renku/queue/client/types.scala index 918c283d..e0043777 100644 --- a/modules/redis-client/src/main/scala/io/renku/queue/client/types.scala +++ b/modules/redis-client/src/main/scala/io/renku/queue/client/types.scala @@ -19,11 +19,9 @@ package io.renku.queue.client opaque type QueueName = String -object QueueName { - def apply(v: String): QueueName = new QueueName(v) -} +object QueueName: + def apply(v: String): QueueName = v opaque type ClientId = String -object ClientId { - def apply(v: String): ClientId = new ClientId(v) -} +object ClientId: + def apply(v: String): ClientId = v diff --git a/modules/redis-client/src/main/scala/io/renku/redis/client/RedisUrl.scala b/modules/redis-client/src/main/scala/io/renku/redis/client/RedisUrl.scala index 47885f3d..956c2a3c 100644 --- a/modules/redis-client/src/main/scala/io/renku/redis/client/RedisUrl.scala +++ b/modules/redis-client/src/main/scala/io/renku/redis/client/RedisUrl.scala @@ -20,5 +20,5 @@ package io.renku.redis.client opaque type RedisUrl = String object RedisUrl { - def apply(v: String): RedisUrl = new RedisUrl(v) + def apply(v: String): RedisUrl = v } diff --git a/modules/search-api/src/main/scala/io/renku/search/api/Microservice.scala b/modules/search-api/src/main/scala/io/renku/search/api/Microservice.scala index ef4ae132..249bab14 100644 --- a/modules/search-api/src/main/scala/io/renku/search/api/Microservice.scala +++ b/modules/search-api/src/main/scala/io/renku/search/api/Microservice.scala @@ -19,22 +19,15 @@ package io.renku.search.api import cats.effect.{ExitCode, IO, IOApp} -import cats.syntax.all.* -import io.renku.solr.client.SolrConfig -import org.http4s.implicits.* - -import scala.concurrent.duration.Duration object Microservice extends IOApp: - private val solrConfig = SolrConfig( - baseUrl = uri"http://localhost:8983" / "solr", - core = "search-core-test", - commitWithin = Some(Duration.Zero), - logMessageBodies = true - ) + private val loadConfig = SearchApiConfig.config.load[IO] override def run(args: List[String]): IO[ExitCode] = - (createHttpApp >>= HttpServer.build).use(_ => IO.never).as(ExitCode.Success) - - private def createHttpApp = HttpApplication[IO](solrConfig) + for { + config <- loadConfig + _ <- HttpApplication[IO](config.solrConfig) + .flatMap(HttpServer.build) + .use(_ => IO.never) + } yield ExitCode.Success diff --git a/modules/search-api/src/main/scala/io/renku/search/api/SearchApiConfig.scala b/modules/search-api/src/main/scala/io/renku/search/api/SearchApiConfig.scala new file mode 100644 index 00000000..86261ac5 --- /dev/null +++ b/modules/search-api/src/main/scala/io/renku/search/api/SearchApiConfig.scala @@ -0,0 +1,31 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.api + +import ciris.{ConfigValue, Effect} +import io.renku.search.config.ConfigValues +import io.renku.solr.client.SolrConfig + +final case class SearchApiConfig( + solrConfig: SolrConfig +) + +object SearchApiConfig: + val config: ConfigValue[Effect, SearchApiConfig] = + ConfigValues.solrConfig.map(SearchApiConfig.apply) diff --git a/modules/search-provision/src/main/scala/io/renku/search/provision/Microservice.scala b/modules/search-provision/src/main/scala/io/renku/search/provision/Microservice.scala index b932a76e..a912af65 100644 --- a/modules/search-provision/src/main/scala/io/renku/search/provision/Microservice.scala +++ b/modules/search-provision/src/main/scala/io/renku/search/provision/Microservice.scala @@ -19,47 +19,36 @@ package io.renku.search.provision import cats.effect.{ExitCode, IO, IOApp, Temporal} -import io.renku.queue.client.QueueName -import io.renku.redis.client.RedisUrl import io.renku.search.solr.schema.Migrations -import io.renku.solr.client.SolrConfig import io.renku.solr.client.migration.SchemaMigrator -import org.http4s.Uri -import org.http4s.implicits.* import scribe.Scribe import scribe.cats.* -import scala.concurrent.duration.* - object Microservice extends IOApp: - private val queueName = QueueName("events") - private val redisUrl = RedisUrl("redis://localhost:6379") - private val solrConfig = SolrConfig( - baseUrl = uri"http://localhost:8983" / "solr", - core = "search-core-test", - commitWithin = Some(Duration.Zero), - logMessageBodies = true - ) - private val retryOnErrorDelay = 2 seconds + private val loadConfig: IO[SearchProvisionConfig] = + SearchProvisionConfig.config.load[IO] override def run(args: List[String]): IO[ExitCode] = - (runSolrMigrations >> startProvisioning) - .as(ExitCode.Success) - - private def startProvisioning: IO[Unit] = - SearchProvisioner[IO](queueName, redisUrl, solrConfig) + for { + config <- loadConfig + _ <- runSolrMigrations(config) + _ <- startProvisioning(config) + } yield ExitCode.Success + + private def startProvisioning(cfg: SearchProvisionConfig): IO[Unit] = + SearchProvisioner[IO](cfg.queueName, cfg.redisUrl, cfg.solrConfig) .evalMap(_.provisionSolr.start) .use(_ => IO.never) .handleErrorWith { err => Scribe[IO].error("Starting provisioning failure, retrying", err) >> - Temporal[IO].delayBy(startProvisioning, retryOnErrorDelay) + Temporal[IO].delayBy(startProvisioning(cfg), cfg.retryOnErrorDelay) } - private def runSolrMigrations: IO[Unit] = - SchemaMigrator[IO](solrConfig) + private def runSolrMigrations(cfg: SearchProvisionConfig): IO[Unit] = + SchemaMigrator[IO](cfg.solrConfig) .use(_.migrate(Migrations.all)) .handleErrorWith { err => Scribe[IO].error("Running solr migrations failure, retrying", err) >> - Temporal[IO].delayBy(runSolrMigrations, retryOnErrorDelay) + Temporal[IO].delayBy(runSolrMigrations(cfg), cfg.retryOnErrorDelay) } diff --git a/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisionConfig.scala b/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisionConfig.scala new file mode 100644 index 00000000..9ad6517f --- /dev/null +++ b/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisionConfig.scala @@ -0,0 +1,47 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.provision + +import cats.syntax.all.* +import ciris.{ConfigValue, Effect} +import io.renku.queue.client.QueueName +import io.renku.redis.client.RedisUrl +import io.renku.search.config.ConfigValues +import io.renku.solr.client.SolrConfig + +import scala.concurrent.duration.FiniteDuration + +final case class SearchProvisionConfig( + redisUrl: RedisUrl, + queueName: QueueName, + solrConfig: SolrConfig, + retryOnErrorDelay: FiniteDuration +) + +object SearchProvisionConfig: + + val config: ConfigValue[Effect, SearchProvisionConfig] = + ( + ConfigValues.redisUrl, + ConfigValues.eventsQueueName, + ConfigValues.solrConfig, + ConfigValues.retryOnErrorDelay + ).mapN( + SearchProvisionConfig.apply + ) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 6b494de4..8b15cdf9 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -10,6 +10,7 @@ object Dependencies { val catsCore = "2.10.0" val catsEffect = "3.5.3" val catsEffectMunit = "1.0.7" + val ciris = "3.5.0" val fs2 = "3.9.4" val http4s = "0.23.25" val redis4Cats = "1.5.2" @@ -19,6 +20,10 @@ object Dependencies { val scribe = "3.13.0" } + val ciris = Seq( + "is.cir" %% "ciris" % V.ciris + ) + val borer = Seq( "io.bullet" %% "borer-core" % V.borer, "io.bullet" %% "borer-derivation" % V.borer,