feat: a simple service exposing Search API (#8)

* feat: a simple service exposing Search API

* feat: solr client to allow specifying a search phrase

* feat: avro JSON EntityCodec for http4s

build.sbt

@@ -51,7 +51,8 @@ lazy val root = project
     redisClient,
     solrClient,
     searchSolrClient,
-    searchProvision
+    searchProvision,
+    searchApi
   )
 
 lazy val commons = project
@@ -178,6 +179,23 @@ lazy val avroCodec = project
         Dependencies.scodecBits
   )
 
+lazy val http4sAvro = project
+  .in(file("modules/http4s-avro"))
+  .enablePlugins(AutomateHeaderPlugin)
+  .disablePlugins(DbTestPlugin)
+  .withId("http4s-avro")
+  .settings(commonSettings)
+  .settings(
+    name := "http4s-avro",
+    description := "Avro codecs for http4s",
+    libraryDependencies ++=
+      Dependencies.http4sCore ++
+        Dependencies.fs2Core
+  )
+  .dependsOn(
+    avroCodec % "compile->compile;test->test"
+  )
+
 lazy val messages = project
   .in(file("modules/messages"))
   .settings(commonSettings)
@@ -206,6 +224,24 @@ lazy val searchProvision = project
   )
   .enablePlugins(AutomateHeaderPlugin)
 
+lazy val searchApi = project
+  .in(file("modules/search-api"))
+  .withId("search-api")
+  .settings(commonSettings)
+  .settings(
+    name := "search-api",
+    libraryDependencies ++=
+      Dependencies.http4sDsl ++
+        Dependencies.http4sServer
+  )
+  .dependsOn(
+    commons % "compile->compile;test->test",
+    messages % "compile->compile;test->test",
+    http4sAvro % "compile->compile;test->test",
+    searchSolrClient % "compile->compile;test->test"
+  )
+  .enablePlugins(AutomateHeaderPlugin)
+
 lazy val commonSettings = Seq(
   organization := "io.renku",
   publish / skip := true,

modules/avro-codec/src/main/scala/io/renku/avro/codec/json/AvroJsonDecoder.scala

@@ -1,5 +1,6 @@
 package io.renku.avro.codec.json
 
+import cats.syntax.all.*
 import io.renku.avro.codec.{AvroDecoder, AvroReader}
 import org.apache.avro.Schema
 import scodec.bits.ByteVector
@@ -19,8 +20,11 @@ object AvroJsonDecoder:
   def apply[A](f: ByteVector => Either[String, A]): AvroJsonDecoder[A] =
     (json: ByteVector) => f(json)
 
-  def create[A: AvroDecoder](schema: Schema): AvroJsonDecoder[A] = { json =>
-    Try(AvroReader(schema).readJson[A](json)).toEither.left
-      .map(_.getMessage)
+  def create[A: AvroDecoder](schema: Schema): AvroJsonDecoder[A] = json =>
+    Try(AvroReader(schema).readJson[A](json)).toEither
+      .leftMap(_.getMessage)
       .flatMap(_.headOption.toRight(s"Empty json"))
-  }
+
+  def decodeList[A: AvroDecoder](schema: Schema): AvroJsonDecoder[List[A]] = json =>
+    Try(AvroReader(schema).readJson[A](json)).toEither
+      .bimap(_.getMessage, _.toList)

modules/avro-codec/src/main/scala/io/renku/avro/codec/json/AvroJsonEncoder.scala

@@ -1,7 +1,7 @@
 package io.renku.avro.codec.json
 
-import io.renku.avro.codec.{AvroEncoder, AvroWriter}
 import io.renku.avro.codec.encoders.all.given
+import io.renku.avro.codec.{AvroEncoder, AvroWriter}
 import org.apache.avro.{Schema, SchemaBuilder}
 import scodec.bits.ByteVector
 
@@ -17,7 +17,10 @@ object AvroJsonEncoder:
     (a: A) => f(a)
 
   def create[A: AvroEncoder](schema: Schema): AvroJsonEncoder[A] =
-    a => AvroWriter(schema).writeJson(Seq(a))
+    encodeList(schema).contramap(List(_))
+
+  def encodeList[A: AvroEncoder](schema: Schema): AvroJsonEncoder[List[A]] =
+    AvroWriter(schema).writeJson(_)
 
   given AvroJsonEncoder[String] =
     create[String](SchemaBuilder.builder().stringType())

modules/avro-codec/src/test/scala/io/renku/avro/codec/AvroReaderTest.scala → modules/avro-codec/src/test/scala/io/renku/avro/codec/AvroReaderSpec.scala

@@ -23,7 +23,7 @@ import io.renku.avro.codec.encoders.all.given
 import munit.FunSuite
 import org.apache.avro.SchemaBuilder
 
-class AvroReaderTest extends FunSuite {
+class AvroReaderSpec extends FunSuite {
 
   case class Foo(name: String, age: Int) derives AvroDecoder, AvroEncoder
   val fooSchema = SchemaBuilder

modules/avro-codec/src/test/scala/io/renku/avro/codec/json/JsonCodecTest.scala → modules/avro-codec/src/test/scala/io/renku/avro/codec/json/JsonCodecSpec.scala

@@ -4,20 +4,36 @@ import io.renku.avro.codec.*
 import io.renku.avro.codec.all.given
 import munit.FunSuite
 import org.apache.avro.{Schema, SchemaBuilder}
+
 import scala.collection.immutable.Map
 
-class JsonCodecTest extends FunSuite {
+class JsonCodecSpec extends FunSuite {
 
-  test("encode and decode json") {
+  test("encode and decode json"):
     val person =
-      JsonCodecTest.Person("hugo", 42, Map("date" -> "1982", "children" -> "0"))
-    val json = AvroJsonEncoder[JsonCodecTest.Person].encode(person)
-    val decoded = AvroJsonDecoder[JsonCodecTest.Person].decode(json)
+      JsonCodecSpec.Person("hugo", 42, Map("date" -> "1982", "children" -> "0"))
+    val json = AvroJsonEncoder[JsonCodecSpec.Person].encode(person)
+    val decoded = AvroJsonDecoder[JsonCodecSpec.Person].decode(json)
     assertEquals(decoded, Right(person))
-  }
+
+  test("encode and decode lists"):
+    val person1 =
+      JsonCodecSpec.Person("hugo1", 41, Map("date" -> "1981", "children" -> "1"))
+    val person2 =
+      JsonCodecSpec.Person("hugo1", 42, Map("date" -> "1982", "children" -> "2"))
+    val persons = person1 :: person2 :: Nil
+
+    val json = AvroJsonEncoder
+      .encodeList[JsonCodecSpec.Person](JsonCodecSpec.Person.schema)
+      .encode(persons)
+    val decoded = AvroJsonDecoder
+      .decodeList[JsonCodecSpec.Person](JsonCodecSpec.Person.schema)
+      .decode(json)
+
+    assertEquals(decoded, Right(persons))
 }
 
-object JsonCodecTest:
+object JsonCodecSpec:
 
   case class Person(name: String, age: Int, props: Map[String, String])
       derives AvroEncoder,

modules/http4s-avro/README.md

@@ -0,0 +1,3 @@
+# http4s-avro
+
+This module contains tooling to bridge avro and http4s.

modules/http4s-avro/src/main/scala/io/renku/search/http/avro/AvroEntityCodec.scala

@@ -0,0 +1,68 @@
+/*
+ * Copyright 2024 Swiss Data Science Center (SDSC)
+ * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+ * Eidgenössische Technische Hochschule Zürich (ETHZ).
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.renku.search.http.avro
+
+import cats.data.EitherT
+import cats.effect.Async
+import cats.syntax.all.*
+import fs2.Chunk
+import io.renku.avro.codec.json.{AvroJsonDecoder, AvroJsonEncoder}
+import org.http4s.*
+import org.http4s.MediaType.application
+import org.http4s.headers.`Content-Type`
+import scodec.bits.ByteVector
+
+object AvroEntityCodec extends AvroEntityCodec:
+  export Implicits.given
+
+trait AvroEntityCodec:
+
+  def decodeEntity[F[_]: Async, A: AvroJsonDecoder]: EntityDecoder[F, A] =
+    EntityDecoder.decodeBy(MediaType.application.json)(decodeJson)
+
+  private def decodeJson[F[_]: Async, A: AvroJsonDecoder](
+      media: Media[F]
+  ): DecodeResult[F, A] =
+    EitherT {
+      media.body.compile
+        .to(ByteVector)
+        .map(decodeAvro)
+    }
+
+  private def decodeAvro[A: AvroJsonDecoder]: ByteVector => Either[DecodeFailure, A] =
+    AvroJsonDecoder[A]
+      .decode(_)
+      .leftMap(err =>
+        MalformedMessageBodyFailure(s"Cannot decode Json Avro message: $err")
+      )
+
+  def encodeEntity[F[_], A: AvroJsonEncoder]: EntityEncoder[F, A] =
+    EntityEncoder.simple(`Content-Type`(application.json))(a =>
+      Chunk.byteVector(AvroJsonEncoder[A].encode(a))
+    )
+
+  trait Implicits:
+
+    given entityDecoder[F[_]: Async, A: AvroJsonDecoder]: EntityDecoder[F, A] =
+      decodeEntity[F, A]
+
+    given entityEncoder[F[_], A: AvroJsonEncoder]: EntityEncoder[F, A] =
+      encodeEntity[F, A]
+
+  object Implicits extends Implicits

modules/messages/src/main/avro/api.avdl

@@ -0,0 +1,10 @@
+@namespace("io.renku.api")
+protocol ApiShapes {
+
+  /* An example record for a Project Entity returned from the Search API */
+  record Project {
+  	string id;
+  	string name;
+  	string description;
+  }
+}

modules/messages/src/main/avro/messages.avdl

@@ -28,16 +28,4 @@ protocol Messages {
     string name;
     timestamp_ms deletedAt;
   }
-
-
-/*  record ProjectMsg {
-    union { ProjectCreated, ProjectUpdated, ProjectDeleted } message;
-  } */
-
-  /* An example record for a Project Entity returned from the Search API */
-  record ProjectEntity {
-    string id;
-    string name;
-    string description;
-  }
 }

modules/redis-client/src/test/scala/io/renku/redis/client/util/TestSearchRedisServer.scala

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2024 Swiss Data Science Center (SDSC)
+ * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+ * Eidgenössische Technische Hochschule Zürich (ETHZ).
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.renku.redis.client.util
+
+import cats.effect.{ExitCode, IO, IOApp}
+import io.renku.servers.RedisServer
+
+/** This is a utility to start a Redis server for manual testing */
+object TestSearchRedisServer extends IOApp:
+
+  override def run(args: List[String]): IO[ExitCode] =
+    (IO(RedisServer.start()) >> IO.never[ExitCode]).as(ExitCode.Success)

modules/search-api/README.md

@@ -0,0 +1,3 @@
+# search-api
+
+This module exposes the Search API.

modules/search-api/src/main/scala/io/renku/search/api/HttpApplication.scala

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2024 Swiss Data Science Center (SDSC)
+ * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+ * Eidgenössische Technische Hochschule Zürich (ETHZ).
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.renku.search.api
+
+import cats.Monad
+import cats.effect.{Async, Resource}
+import fs2.io.net.Network
+import io.renku.solr.client.SolrConfig
+import org.http4s.dsl.Http4sDsl
+import org.http4s.server.Router
+import org.http4s.{HttpApp, HttpRoutes, Request, Response}
+
+object HttpApplication:
+  def apply[F[_]: Async: Network](
+      solrConfig: SolrConfig
+  ): Resource[F, HttpApp[F]] =
+    SearchApi[F](solrConfig).map(new HttpApplication[F](_).router)
+
+class HttpApplication[F[_]: Monad](searchApi: SearchApi[F]) extends Http4sDsl[F]:
+
+  lazy val router: HttpApp[F] =
+    Router[F]("/" -> routes).orNotFound
+
+  private lazy val routes: HttpRoutes[F] = HttpRoutes.of[F] {
+    case GET -> Root / "api" / phrase => searchApi.find(phrase)
+    case GET -> Root / "ping"         => Ok("pong")
+  }

modules/search-api/src/main/scala/io/renku/search/api/HttpServer.scala

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2024 Swiss Data Science Center (SDSC)
+ * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+ * Eidgenössische Technische Hochschule Zürich (ETHZ).
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.renku.search.api
+
+import cats.effect.{Async, Resource}
+import com.comcast.ip4s.{Port, ipv4, port}
+import fs2.io.net.Network
+import org.http4s.HttpApp
+import org.http4s.ember.server.EmberServerBuilder
+import org.http4s.server.Server
+
+object HttpServer:
+
+  val port: Port = port"8080"
+
+  def build[F[_]: Async: Network](app: HttpApp[F]): Resource[F, Server] =
+    EmberServerBuilder
+      .default[F]
+      .withHost(ipv4"0.0.0.0")
+      .withPort(port)
+      .withHttpApp(app)
+      .build