Merge pull request #10 from SwissDataScienceCenter/docker-image

Publish docker images for microservices

.github/workflows/ci.yml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        java: [ '[email protected]' ]
+        java: [ 17 ]
     steps:
       - uses: actions/[email protected]
         with:
@@ -24,13 +24,13 @@ jobs:
       #   uses: coursier/cache-action@v6
       - name: sbt ci ${{ github.ref }}
         run: sbt -mem 2048 ci
-      # - name: Log in to Docker Hub
-      #   uses: docker/login-action@v2
-      #   with:
-      #     username: ${{ secrets.RENKU_DOCKER_USERNAME }}
-      #     password: ${{ secrets.RENKU_DOCKER_PASSWORD }}
-      # - name: sbt docker:publishLocal
-      #   run: sbt -mem 2048 cli/Docker/publishLocal
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.RENKU_DOCKER_USERNAME }}
+          password: ${{ secrets.RENKU_DOCKER_PASSWORD }}
+      - name: sbt docker:publishLocal
+        run: sbt -mem 2048 search-provision/Docker/publishLocal search-api/Docker/publishLocal
   ci:
     runs-on: ubuntu-latest
     needs: [ci-matrix]

.github/workflows/release.yml

@@ -0,0 +1,72 @@
+name: Release
+on:
+  push:
+    branches: [ main ]
+  release:
+    types: [ published ]
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        java: [ 17 ]
+    steps:
+      - uses: actions/[email protected]
+        with:
+          fetch-depth: 0
+      - uses: olafurpg/setup-scala@v14
+        with:
+          java-version: ${{ matrix.java }}
+
+      - name: Set current version
+        id: version
+        shell: bash
+        run: |
+          sbt 'renku-search/writeVersion'
+          RS_VERSION=$(cat target/version.txt)
+          echo "RS_VERSION=${RS_VERSION}" >> $GITHUB_ENV
+
+          if [ -z "${RS_VERSION}" ]; then
+            echo "Version not set!"
+            exit 1
+          fi
+
+      - name: Create zip packages
+        run: sbt -mem 2048 search-provision/Universal/packageBin search-api/Universal/packageBin
+
+      - name: Publish Release (${{ env.RS_VERSION }})
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        env:
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+        with:
+          files: |
+            modules/search-provision/target/universal/search-provision-${{ env.RS_VERSION }}.zip
+            modules/search-api/target/universal/search-api-${{ env.RS_VERSION }}.zip
+      - name: Publish Pre-Release
+        uses: ncipollo/release-action@v1
+        if: ${{ github.ref }} == 'refs/heads/main'
+        env:
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+        with:
+          token: "${{ secrets.GITHUB_TOKEN }}"
+          prerelease: true
+          allowUpdates: true
+          tag: "nightly"
+          commit: "main"
+          body: "Floating tag associating the latest build from the main branch"
+          name: "renku search nightly"
+          replacesArtifacts: true
+          removeArtifacts: true
+          artifacts: |
+            modules/search-provision/target/universal/search-provision-${{ env.RS_VERSION }}.zip,
+            modules/search-api/target/universal/search-api-${{ env.RS_VERSION }}.zip
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.RENKU_DOCKER_USERNAME }}
+          password: ${{ secrets.RENKU_DOCKER_PASSWORD }}
+      - name: sbt Docker/publish
+        run: sbt -mem 2048 search-provision/Docker/publish search-api/Docker/publish

build.sbt

@@ -34,6 +34,8 @@ addCommandAlias(
 )
 addCommandAlias("fix", "; scalafmtSbt; scalafmtAll") // ; Compile/scalafix; Test/scalafix
 
+val writeVersion = taskKey[Unit]("Write version into a file for CI to pick up")
+
 lazy val root = project
   .in(file("."))
   .withId("renku-search")
@@ -42,7 +44,12 @@ lazy val root = project
     publish / skip := true,
     publishTo := Some(
       Resolver.file("Unused transient repository", file("target/unusedrepo"))
-    )
+    ),
+    writeVersion := {
+      val out = (LocalRootProject / target).value / "version.txt"
+      val versionStr = version.value
+      IO.write(out, versionStr)
+    }
   )
   .aggregate(
     commons,
@@ -209,20 +216,37 @@ lazy val messages = project
   .enablePlugins(AvroCodeGen, AutomateHeaderPlugin)
   .disablePlugins(DbTestPlugin)
 
+lazy val configValues = project
+  .in(file("modules/config-values"))
+  .withId("config-values")
+  .settings(commonSettings)
+  .settings(
+    name := "config-values",
+    libraryDependencies ++= Dependencies.ciris
+  )
+  .dependsOn(
+    commons % "compile->compile;test->test",
+    messages % "compile->compile;test->test",
+    redisClient % "compile->compile;test->test",
+    searchSolrClient % "compile->compile;test->test"
+  )
+
 lazy val searchProvision = project
   .in(file("modules/search-provision"))
   .withId("search-provision")
   .settings(commonSettings)
   .settings(
-    name := "search-provision"
+    name := "search-provision",
+    libraryDependencies ++= Dependencies.ciris
   )
   .dependsOn(
     commons % "compile->compile;test->test",
     messages % "compile->compile;test->test",
     redisClient % "compile->compile;test->test",
-    searchSolrClient % "compile->compile;test->test"
+    searchSolrClient % "compile->compile;test->test",
+    configValues % "compile->compile;test->test"
   )
-  .enablePlugins(AutomateHeaderPlugin)
+  .enablePlugins(AutomateHeaderPlugin, DockerImagePlugin)
 
 lazy val searchApi = project
   .in(file("modules/search-api"))
@@ -232,15 +256,17 @@ lazy val searchApi = project
     name := "search-api",
     libraryDependencies ++=
       Dependencies.http4sDsl ++
-        Dependencies.http4sServer
+        Dependencies.http4sServer ++
+        Dependencies.ciris
   )
   .dependsOn(
     commons % "compile->compile;test->test",
     messages % "compile->compile;test->test",
     http4sAvro % "compile->compile;test->test",
-    searchSolrClient % "compile->compile;test->test"
+    searchSolrClient % "compile->compile;test->test",
+    configValues % "compile->compile;test->test"
   )
-  .enablePlugins(AutomateHeaderPlugin)
+  .enablePlugins(AutomateHeaderPlugin, DockerImagePlugin)
 
 lazy val commonSettings = Seq(
   organization := "io.renku",

modules/config-values/src/main/scala/io/renku/search/config/ConfigDecoders.scala

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2024 Swiss Data Science Center (SDSC)
+ * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+ * Eidgenössische Technische Hochschule Zürich (ETHZ).
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.renku.search.config
+
+import cats.syntax.all.*
+import ciris.{ConfigDecoder, ConfigError}
+import io.renku.queue.client.QueueName
+import io.renku.redis.client.RedisUrl
+import org.http4s.Uri
+
+import scala.concurrent.duration.{Duration, FiniteDuration}
+
+trait ConfigDecoders:
+  given ConfigDecoder[String, Uri] =
+    ConfigDecoder[String].mapEither { (_, s) =>
+      Uri.fromString(s).leftMap(err => ConfigError(err.getMessage))
+    }
+
+  given ConfigDecoder[String, FiniteDuration] =
+    ConfigDecoder[String].mapOption("duration") { s =>
+      Duration.unapply(s).map(Duration.apply.tupled).filter(_.isFinite)
+    }
+
+  given ConfigDecoder[String, RedisUrl] =
+    ConfigDecoder[String].map(s => RedisUrl(s))
+
+  given ConfigDecoder[String, QueueName] =
+    ConfigDecoder[String].map(s => QueueName(s))

modules/config-values/src/main/scala/io/renku/search/config/ConfigValues.scala

@@ -0,0 +1,51 @@
+/*
+ * Copyright 2024 Swiss Data Science Center (SDSC)
+ * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+ * Eidgenössische Technische Hochschule Zürich (ETHZ).
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.renku.search.config
+
+import cats.syntax.all.*
+import ciris.*
+import io.renku.queue.client.QueueName
+import io.renku.redis.client.RedisUrl
+import io.renku.solr.client.SolrConfig
+import org.http4s.Uri
+
+import scala.concurrent.duration.FiniteDuration
+
+object ConfigValues extends ConfigDecoders:
+
+  private val prefix = "RS"
+
+  val redisUrl: ConfigValue[Effect, RedisUrl] =
+    env(s"${prefix}_REDIS_URL").default("redis://localhost:6379").as[RedisUrl]
+
+  val eventsQueueName: ConfigValue[Effect, QueueName] =
+    env(s"${prefix}_REDIS_QUEUE_NAME").default("events").as[QueueName]
+
+  val retryOnErrorDelay: ConfigValue[Effect, FiniteDuration] =
+    env(s"${prefix}_RETRY_ON_ERROR_DELAY").default("2 seconds").as[FiniteDuration]
+
+  val solrConfig: ConfigValue[Effect, SolrConfig] = {
+    val url = env(s"${prefix}_SOLR_URL").default("http://localhost:8983/solr").as[Uri]
+    val core = env(s"${prefix}_SOLR_CORE").default("search-core-test")
+    val defaultCommit =
+      env(s"${prefix}_SOLR_DEFAULT_COMMIT_WITHIN").default("0").as[FiniteDuration].option
+    val logMessageBodies =
+      env(s"${prefix}_SOLR_LOG_MESSAGE_BODIES").default("false").as[Boolean]
+    (url, core, defaultCommit, logMessageBodies).mapN(SolrConfig.apply)
+  }

modules/redis-client/src/main/scala/io/renku/queue/client/types.scala

@@ -19,11 +19,9 @@
 package io.renku.queue.client
 
 opaque type QueueName = String
-object QueueName {
-  def apply(v: String): QueueName = new QueueName(v)
-}
+object QueueName:
+  def apply(v: String): QueueName = v
 
 opaque type ClientId = String
-object ClientId {
-  def apply(v: String): ClientId = new ClientId(v)
-}
+object ClientId:
+  def apply(v: String): ClientId = v

modules/redis-client/src/main/scala/io/renku/redis/client/RedisUrl.scala

@@ -20,5 +20,5 @@ package io.renku.redis.client
 
 opaque type RedisUrl = String
 object RedisUrl {
-  def apply(v: String): RedisUrl = new RedisUrl(v)
+  def apply(v: String): RedisUrl = v
 }

modules/search-api/src/main/scala/io/renku/search/api/Microservice.scala

@@ -19,22 +19,15 @@
 package io.renku.search.api
 
 import cats.effect.{ExitCode, IO, IOApp}
-import cats.syntax.all.*
-import io.renku.solr.client.SolrConfig
-import org.http4s.implicits.*
-
-import scala.concurrent.duration.Duration
 
 object Microservice extends IOApp:
 
-  private val solrConfig = SolrConfig(
-    baseUrl = uri"http://localhost:8983" / "solr",
-    core = "search-core-test",
-    commitWithin = Some(Duration.Zero),
-    logMessageBodies = true
-  )
+  private val loadConfig = SearchApiConfig.config.load[IO]
 
   override def run(args: List[String]): IO[ExitCode] =
-    (createHttpApp >>= HttpServer.build).use(_ => IO.never).as(ExitCode.Success)
-
-  private def createHttpApp = HttpApplication[IO](solrConfig)
+    for {
+      config <- loadConfig
+      _ <- HttpApplication[IO](config.solrConfig)
+        .flatMap(HttpServer.build)
+        .use(_ => IO.never)
+    } yield ExitCode.Success

modules/search-api/src/main/scala/io/renku/search/api/SearchApiConfig.scala

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2024 Swiss Data Science Center (SDSC)
+ * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+ * Eidgenössische Technische Hochschule Zürich (ETHZ).
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.renku.search.api
+
+import ciris.{ConfigValue, Effect}
+import io.renku.search.config.ConfigValues
+import io.renku.solr.client.SolrConfig
+
+final case class SearchApiConfig(
+    solrConfig: SolrConfig
+)
+
+object SearchApiConfig:
+  val config: ConfigValue[Effect, SearchApiConfig] =
+    ConfigValues.solrConfig.map(SearchApiConfig.apply)