feat: Search API Microservice and manual testing facilities

modules/redis-client/src/test/scala/io/renku/redis/client/util/TestSearchRedisServer.scala

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2024 Swiss Data Science Center (SDSC)
+ * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+ * Eidgenössische Technische Hochschule Zürich (ETHZ).
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.renku.redis.client.util
+
+import cats.effect.{ExitCode, IO, IOApp}
+import io.renku.servers.RedisServer
+
+/** This is a utility to start a Redis server for manual testing */
+object TestSearchRedisServer extends IOApp:
+
+  override def run(args: List[String]): IO[ExitCode] =
+    (IO(RedisServer.start()) >> IO.never[ExitCode]).as(ExitCode.Success)

modules/search-api/src/main/scala/io/renku/search/api/HttpApplication.scala

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2024 Swiss Data Science Center (SDSC)
+ * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+ * Eidgenössische Technische Hochschule Zürich (ETHZ).
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.renku.search.api
+
+import cats.Monad
+import cats.effect.{Async, Resource}
+import fs2.io.net.Network
+import io.renku.solr.client.SolrConfig
+import org.http4s.dsl.Http4sDsl
+import org.http4s.server.Router
+import org.http4s.{HttpApp, HttpRoutes, Request, Response}
+import scribe.Scribe
+
+object HttpApplication:
+  def apply[F[_]: Async: Network: Scribe](
+      solrConfig: SolrConfig
+  ): Resource[F, HttpApp[F]] =
+    SearchApi[F](solrConfig).map(new HttpApplication[F](_).router)
+
+class HttpApplication[F[_]: Monad](searchApi: SearchApi[F]) extends Http4sDsl[F]:
+
+  lazy val router: HttpApp[F] =
+    Router[F]("/" -> routes).orNotFound
+
+  private lazy val routes: HttpRoutes[F] = HttpRoutes.of[F] {
+    case GET -> Root / "api" / phrase => searchApi.find(phrase)
+    case GET -> Root / "ping"         => Ok("pong")
+  }

modules/search-api/src/main/scala/io/renku/search/api/HttpServer.scala

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2024 Swiss Data Science Center (SDSC)
+ * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+ * Eidgenössische Technische Hochschule Zürich (ETHZ).
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.renku.search.api
+
+import cats.effect.{Async, Resource}
+import com.comcast.ip4s.{Port, ipv4, port}
+import fs2.io.net.Network
+import org.http4s.HttpApp
+import org.http4s.ember.server.EmberServerBuilder
+import org.http4s.server.Server
+
+object HttpServer:
+
+  val port: Port = port"8080"
+
+  def build[F[_]: Async: Network](app: HttpApp[F]): Resource[F, Server] =
+    EmberServerBuilder
+      .default[F]
+      .withHost(ipv4"0.0.0.0")
+      .withPort(port)
+      .withHttpApp(app)
+      .build

modules/search-api/src/main/scala/io/renku/search/api/Microservice.scala

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2024 Swiss Data Science Center (SDSC)
+ * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+ * Eidgenössische Technische Hochschule Zürich (ETHZ).
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.renku.search.api
+
+import cats.effect.{ExitCode, IO, IOApp}
+import cats.syntax.all.*
+import io.renku.solr.client.SolrConfig
+import org.http4s.implicits.*
+import scribe.Scribe
+
+import scala.concurrent.duration.Duration
+
+object Microservice extends IOApp:
+
+  private given Scribe[IO] = scribe.cats[IO]
+
+  private val solrConfig = SolrConfig(
+    baseUrl = uri"http://localhost:8983" / "solr",
+    core = "search-core-test",
+    commitWithin = Some(Duration.Zero),
+    logMessageBodies = true
+  )
+
+  override def run(args: List[String]): IO[ExitCode] =
+    (createHttpApp >>= HttpServer.build).use(_ => IO.never).as(ExitCode.Success)
+
+  private def createHttpApp = HttpApplication[IO](solrConfig)

modules/search-api/src/main/scala/io/renku/search/api/SearchApi.scala

@@ -23,10 +23,13 @@ import fs2.io.net.Network
 import io.renku.search.solr.client.SearchSolrClient
 import io.renku.solr.client.SolrConfig
 import org.http4s.Response
+import scribe.Scribe
 
 trait SearchApi[F[_]]:
   def find(phrase: String): F[Response[F]]
 
 object SearchApi:
-  def apply[F[_]: Async: Network](solrConfig: SolrConfig): Resource[F, SearchApi[F]] =
+  def apply[F[_]: Async: Network: Scribe](
+      solrConfig: SolrConfig
+  ): Resource[F, SearchApi[F]] =
     SearchSolrClient[F](solrConfig).map(new SearchApiImpl[F](_))

modules/search-api/src/main/scala/io/renku/search/api/SearchApiImpl.scala

@@ -28,13 +28,18 @@ import io.renku.search.solr.client.SearchSolrClient
 import io.renku.search.solr.documents.Project as SolrProject
 import org.http4s.Response
 import org.http4s.dsl.Http4sDsl
+import scribe.Scribe
 
-private class SearchApiImpl[F[_]: Async](solrClient: SearchSolrClient[F])
+private class SearchApiImpl[F[_]: Async: Scribe](solrClient: SearchSolrClient[F])
     extends Http4sDsl[F]
     with SearchApi[F]:
 
   override def find(phrase: String): F[Response[F]] =
-    solrClient.findProjects(phrase).map(toApiModel).map(toAvroResponse)
+    solrClient
+      .findProjects(phrase)
+      .map(toApiModel)
+      .map(toAvroResponse)
+      .handleErrorWith(errorResponse(phrase))
 
   private given AvroJsonEncoder[List[ApiProject]] =
     AvroJsonEncoder.encodeList[ApiProject](ApiProject.SCHEMA$)
@@ -43,5 +48,11 @@ private class SearchApiImpl[F[_]: Async](solrClient: SearchSolrClient[F])
     Response[F](Ok)
       .withEntity(entities)
 
+  private def errorResponse(phrase: String): Throwable => F[Response[F]] =
+    err =>
+      Scribe[F]
+        .error(s"Finding by '$phrase' phrase failed", err)
+        .map(_ => Response[F](InternalServerError).withEntity(err.getMessage))
+
   private def toApiModel(entities: List[SolrProject]): List[ApiProject] =
     entities.map(p => ApiProject(p.id, p.name, p.description))

modules/search-api/src/test/scala/io/renku/search/api/SearchApiSpec.scala

@@ -29,9 +29,12 @@ import io.renku.search.solr.client.SearchSolrClientGenerators.*
 import io.renku.search.solr.client.SearchSolrSpec
 import io.renku.search.solr.documents.Project as SolrProject
 import munit.CatsEffectSuite
+import scribe.Scribe
 
 class SearchApiSpec extends CatsEffectSuite with SearchSolrSpec:
 
+  private given Scribe[IO] = scribe.cats[IO]
+
   test("do a lookup in Solr to find entities matching the given phrase"):
     withSearchSolrClient().use { client =>
       val project1 = projectDocumentGen("matching", "matching description").generateOne

modules/search-provision/src/main/scala/io/renku/search/provision/Microservice.scala

@@ -49,7 +49,8 @@ object Microservice extends IOApp:
 
   private def startProvisioning: IO[Unit] =
     SearchProvisioner[IO](queueName, redisUrl, solrConfig)
-      .use(_.provisionSolr)
+      .evalMap(_.provisionSolr.start)
+      .use(_ => IO.never)
       .handleErrorWith { err =>
         Scribe[IO].error("Starting provisioning failure, retrying", err) >>
           Temporal[IO].delayBy(startProvisioning, retryOnErrorDelay)

modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisioner.scala

@@ -45,7 +45,7 @@ object SearchProvisioner:
       .flatMap(qc => SearchSolrClient[F](solrConfig).tupleLeft(qc))
       .map { case (qc, sc) => new SearchProvisionerImpl[F](queueName, qc, sc) }
 
-private class SearchProvisionerImpl[F[_]: Async](
+private class SearchProvisionerImpl[F[_]: Async: Scribe](
     queueName: QueueName,
     queueClient: QueueClient[F],
     solrClient: SearchSolrClient[F]
@@ -55,6 +55,7 @@ private class SearchProvisionerImpl[F[_]: Async](
     queueClient
       .acquireEventsStream(queueName, chunkSize = 1, maybeOffset = None)
       .map(decodeEvent)
+      .evalTap(decoded => Scribe[F].info(s"Received $decoded"))
       .flatMap(decoded => Stream.emits[F, ProjectCreated](decoded))
       .evalMap(pushToSolr)
       .compile

modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala

@@ -31,12 +31,14 @@ import io.renku.redis.client.util.RedisSpec
 import io.renku.search.solr.client.SearchSolrSpec
 import io.renku.search.solr.documents.Project
 import munit.CatsEffectSuite
+import scribe.Scribe
 
 import java.time.temporal.ChronoUnit
 import scala.concurrent.duration.*
 
 class SearchProvisionerSpec extends CatsEffectSuite with RedisSpec with SearchSolrSpec:
 
+  private given Scribe[IO] = scribe.cats[IO]
   private val avro = AvroIO(ProjectCreated.SCHEMA$)
 
   test("can fetch events and send them to Solr"):

modules/search-solr-client/src/test/scala/io/renku/search/solr/client/TestSearchSolrServer.scala

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2024 Swiss Data Science Center (SDSC)
+ * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+ * Eidgenössische Technische Hochschule Zürich (ETHZ).
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.renku.search.solr.client
+
+import cats.effect.{ExitCode, IO, IOApp}
+import io.renku.servers.SolrServer
+
+/** This is a utility to start a Solr server for manual testing */
+object TestSearchSolrServer extends IOApp:
+
+  override def run(args: List[String]): IO[ExitCode] =
+    (IO(SolrServer.start()) >> IO.never[ExitCode]).as(ExitCode.Success)