From a1c8167696ae360758552490aa90c6ea5ee83871 Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Tue, 30 Jan 2024 12:33:16 +0100 Subject: [PATCH] feat: a simple service exposing Search API (#8) * feat: a simple service exposing Search API * feat: solr client to allow specifying a search phrase * feat: avro JSON EntityCodec for http4s --- build.sbt | 38 ++++++++++- .../avro/codec/json/AvroJsonDecoder.scala | 12 ++-- .../avro/codec/json/AvroJsonEncoder.scala | 7 +- ...oReaderTest.scala => AvroReaderSpec.scala} | 2 +- ...sonCodecTest.scala => JsonCodecSpec.scala} | 30 ++++++-- modules/http4s-avro/README.md | 3 + .../search/http/avro/AvroEntityCodec.scala | 68 +++++++++++++++++++ modules/messages/src/main/avro/api.avdl | 10 +++ modules/messages/src/main/avro/messages.avdl | 12 ---- .../client/util/TestSearchRedisServer.scala | 28 ++++++++ modules/search-api/README.md | 3 + .../io/renku/search/api/HttpApplication.scala | 43 ++++++++++++ .../io/renku/search/api/HttpServer.scala | 38 +++++++++++ .../io/renku/search/api/Microservice.scala | 40 +++++++++++ .../scala/io/renku/search/api/SearchApi.scala | 34 ++++++++++ .../io/renku/search/api/SearchApiImpl.scala | 56 +++++++++++++++ .../io/renku/search/api/SearchApiSpec.scala | 54 +++++++++++++++ .../renku/search/provision/Microservice.scala | 3 +- .../search/provision/SearchProvisioner.scala | 3 +- .../provision/SearchProvisionerSpec.scala | 4 +- .../search/solr/client/SearchSolrClient.scala | 2 +- .../solr/client/SearchSolrClientImpl.scala | 8 ++- .../client/SearchSolrClientGenerators.scala | 4 +- .../solr/client/SearchSolrClientSpec.scala | 2 +- .../solr/client/TestSearchSolrServer.scala | 28 ++++++++ project/Dependencies.scala | 6 ++ 26 files changed, 500 insertions(+), 38 deletions(-) rename modules/avro-codec/src/test/scala/io/renku/avro/codec/{AvroReaderTest.scala => AvroReaderSpec.scala} (98%) rename modules/avro-codec/src/test/scala/io/renku/avro/codec/json/{JsonCodecTest.scala => JsonCodecSpec.scala} (51%) create mode 100644 modules/http4s-avro/README.md create mode 100644 modules/http4s-avro/src/main/scala/io/renku/search/http/avro/AvroEntityCodec.scala create mode 100644 modules/messages/src/main/avro/api.avdl create mode 100644 modules/redis-client/src/test/scala/io/renku/redis/client/util/TestSearchRedisServer.scala create mode 100644 modules/search-api/README.md create mode 100644 modules/search-api/src/main/scala/io/renku/search/api/HttpApplication.scala create mode 100644 modules/search-api/src/main/scala/io/renku/search/api/HttpServer.scala create mode 100644 modules/search-api/src/main/scala/io/renku/search/api/Microservice.scala create mode 100644 modules/search-api/src/main/scala/io/renku/search/api/SearchApi.scala create mode 100644 modules/search-api/src/main/scala/io/renku/search/api/SearchApiImpl.scala create mode 100644 modules/search-api/src/test/scala/io/renku/search/api/SearchApiSpec.scala create mode 100644 modules/search-solr-client/src/test/scala/io/renku/search/solr/client/TestSearchSolrServer.scala diff --git a/build.sbt b/build.sbt index f27b43f8..574bcea7 100644 --- a/build.sbt +++ b/build.sbt @@ -51,7 +51,8 @@ lazy val root = project redisClient, solrClient, searchSolrClient, - searchProvision + searchProvision, + searchApi ) lazy val commons = project @@ -178,6 +179,23 @@ lazy val avroCodec = project Dependencies.scodecBits ) +lazy val http4sAvro = project + .in(file("modules/http4s-avro")) + .enablePlugins(AutomateHeaderPlugin) + .disablePlugins(DbTestPlugin) + .withId("http4s-avro") + .settings(commonSettings) + .settings( + name := "http4s-avro", + description := "Avro codecs for http4s", + libraryDependencies ++= + Dependencies.http4sCore ++ + Dependencies.fs2Core + ) + .dependsOn( + avroCodec % "compile->compile;test->test" + ) + lazy val messages = project .in(file("modules/messages")) .settings(commonSettings) @@ -206,6 +224,24 @@ lazy val searchProvision = project ) .enablePlugins(AutomateHeaderPlugin) +lazy val searchApi = project + .in(file("modules/search-api")) + .withId("search-api") + .settings(commonSettings) + .settings( + name := "search-api", + libraryDependencies ++= + Dependencies.http4sDsl ++ + Dependencies.http4sServer + ) + .dependsOn( + commons % "compile->compile;test->test", + messages % "compile->compile;test->test", + http4sAvro % "compile->compile;test->test", + searchSolrClient % "compile->compile;test->test" + ) + .enablePlugins(AutomateHeaderPlugin) + lazy val commonSettings = Seq( organization := "io.renku", publish / skip := true, diff --git a/modules/avro-codec/src/main/scala/io/renku/avro/codec/json/AvroJsonDecoder.scala b/modules/avro-codec/src/main/scala/io/renku/avro/codec/json/AvroJsonDecoder.scala index e65222ee..02ff54e5 100644 --- a/modules/avro-codec/src/main/scala/io/renku/avro/codec/json/AvroJsonDecoder.scala +++ b/modules/avro-codec/src/main/scala/io/renku/avro/codec/json/AvroJsonDecoder.scala @@ -1,5 +1,6 @@ package io.renku.avro.codec.json +import cats.syntax.all.* import io.renku.avro.codec.{AvroDecoder, AvroReader} import org.apache.avro.Schema import scodec.bits.ByteVector @@ -19,8 +20,11 @@ object AvroJsonDecoder: def apply[A](f: ByteVector => Either[String, A]): AvroJsonDecoder[A] = (json: ByteVector) => f(json) - def create[A: AvroDecoder](schema: Schema): AvroJsonDecoder[A] = { json => - Try(AvroReader(schema).readJson[A](json)).toEither.left - .map(_.getMessage) + def create[A: AvroDecoder](schema: Schema): AvroJsonDecoder[A] = json => + Try(AvroReader(schema).readJson[A](json)).toEither + .leftMap(_.getMessage) .flatMap(_.headOption.toRight(s"Empty json")) - } + + def decodeList[A: AvroDecoder](schema: Schema): AvroJsonDecoder[List[A]] = json => + Try(AvroReader(schema).readJson[A](json)).toEither + .bimap(_.getMessage, _.toList) diff --git a/modules/avro-codec/src/main/scala/io/renku/avro/codec/json/AvroJsonEncoder.scala b/modules/avro-codec/src/main/scala/io/renku/avro/codec/json/AvroJsonEncoder.scala index 1e9a72b4..87e8ab3d 100644 --- a/modules/avro-codec/src/main/scala/io/renku/avro/codec/json/AvroJsonEncoder.scala +++ b/modules/avro-codec/src/main/scala/io/renku/avro/codec/json/AvroJsonEncoder.scala @@ -1,7 +1,7 @@ package io.renku.avro.codec.json -import io.renku.avro.codec.{AvroEncoder, AvroWriter} import io.renku.avro.codec.encoders.all.given +import io.renku.avro.codec.{AvroEncoder, AvroWriter} import org.apache.avro.{Schema, SchemaBuilder} import scodec.bits.ByteVector @@ -17,7 +17,10 @@ object AvroJsonEncoder: (a: A) => f(a) def create[A: AvroEncoder](schema: Schema): AvroJsonEncoder[A] = - a => AvroWriter(schema).writeJson(Seq(a)) + encodeList(schema).contramap(List(_)) + + def encodeList[A: AvroEncoder](schema: Schema): AvroJsonEncoder[List[A]] = + AvroWriter(schema).writeJson(_) given AvroJsonEncoder[String] = create[String](SchemaBuilder.builder().stringType()) diff --git a/modules/avro-codec/src/test/scala/io/renku/avro/codec/AvroReaderTest.scala b/modules/avro-codec/src/test/scala/io/renku/avro/codec/AvroReaderSpec.scala similarity index 98% rename from modules/avro-codec/src/test/scala/io/renku/avro/codec/AvroReaderTest.scala rename to modules/avro-codec/src/test/scala/io/renku/avro/codec/AvroReaderSpec.scala index 1cba2f3e..214d429d 100644 --- a/modules/avro-codec/src/test/scala/io/renku/avro/codec/AvroReaderTest.scala +++ b/modules/avro-codec/src/test/scala/io/renku/avro/codec/AvroReaderSpec.scala @@ -23,7 +23,7 @@ import io.renku.avro.codec.encoders.all.given import munit.FunSuite import org.apache.avro.SchemaBuilder -class AvroReaderTest extends FunSuite { +class AvroReaderSpec extends FunSuite { case class Foo(name: String, age: Int) derives AvroDecoder, AvroEncoder val fooSchema = SchemaBuilder diff --git a/modules/avro-codec/src/test/scala/io/renku/avro/codec/json/JsonCodecTest.scala b/modules/avro-codec/src/test/scala/io/renku/avro/codec/json/JsonCodecSpec.scala similarity index 51% rename from modules/avro-codec/src/test/scala/io/renku/avro/codec/json/JsonCodecTest.scala rename to modules/avro-codec/src/test/scala/io/renku/avro/codec/json/JsonCodecSpec.scala index e9f60039..cb1d521c 100644 --- a/modules/avro-codec/src/test/scala/io/renku/avro/codec/json/JsonCodecTest.scala +++ b/modules/avro-codec/src/test/scala/io/renku/avro/codec/json/JsonCodecSpec.scala @@ -4,20 +4,36 @@ import io.renku.avro.codec.* import io.renku.avro.codec.all.given import munit.FunSuite import org.apache.avro.{Schema, SchemaBuilder} + import scala.collection.immutable.Map -class JsonCodecTest extends FunSuite { +class JsonCodecSpec extends FunSuite { - test("encode and decode json") { + test("encode and decode json"): val person = - JsonCodecTest.Person("hugo", 42, Map("date" -> "1982", "children" -> "0")) - val json = AvroJsonEncoder[JsonCodecTest.Person].encode(person) - val decoded = AvroJsonDecoder[JsonCodecTest.Person].decode(json) + JsonCodecSpec.Person("hugo", 42, Map("date" -> "1982", "children" -> "0")) + val json = AvroJsonEncoder[JsonCodecSpec.Person].encode(person) + val decoded = AvroJsonDecoder[JsonCodecSpec.Person].decode(json) assertEquals(decoded, Right(person)) - } + + test("encode and decode lists"): + val person1 = + JsonCodecSpec.Person("hugo1", 41, Map("date" -> "1981", "children" -> "1")) + val person2 = + JsonCodecSpec.Person("hugo1", 42, Map("date" -> "1982", "children" -> "2")) + val persons = person1 :: person2 :: Nil + + val json = AvroJsonEncoder + .encodeList[JsonCodecSpec.Person](JsonCodecSpec.Person.schema) + .encode(persons) + val decoded = AvroJsonDecoder + .decodeList[JsonCodecSpec.Person](JsonCodecSpec.Person.schema) + .decode(json) + + assertEquals(decoded, Right(persons)) } -object JsonCodecTest: +object JsonCodecSpec: case class Person(name: String, age: Int, props: Map[String, String]) derives AvroEncoder, diff --git a/modules/http4s-avro/README.md b/modules/http4s-avro/README.md new file mode 100644 index 00000000..f981963e --- /dev/null +++ b/modules/http4s-avro/README.md @@ -0,0 +1,3 @@ +# http4s-avro + +This module contains tooling to bridge avro and http4s. diff --git a/modules/http4s-avro/src/main/scala/io/renku/search/http/avro/AvroEntityCodec.scala b/modules/http4s-avro/src/main/scala/io/renku/search/http/avro/AvroEntityCodec.scala new file mode 100644 index 00000000..ff5502a4 --- /dev/null +++ b/modules/http4s-avro/src/main/scala/io/renku/search/http/avro/AvroEntityCodec.scala @@ -0,0 +1,68 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.http.avro + +import cats.data.EitherT +import cats.effect.Async +import cats.syntax.all.* +import fs2.Chunk +import io.renku.avro.codec.json.{AvroJsonDecoder, AvroJsonEncoder} +import org.http4s.* +import org.http4s.MediaType.application +import org.http4s.headers.`Content-Type` +import scodec.bits.ByteVector + +object AvroEntityCodec extends AvroEntityCodec: + export Implicits.given + +trait AvroEntityCodec: + + def decodeEntity[F[_]: Async, A: AvroJsonDecoder]: EntityDecoder[F, A] = + EntityDecoder.decodeBy(MediaType.application.json)(decodeJson) + + private def decodeJson[F[_]: Async, A: AvroJsonDecoder]( + media: Media[F] + ): DecodeResult[F, A] = + EitherT { + media.body.compile + .to(ByteVector) + .map(decodeAvro) + } + + private def decodeAvro[A: AvroJsonDecoder]: ByteVector => Either[DecodeFailure, A] = + AvroJsonDecoder[A] + .decode(_) + .leftMap(err => + MalformedMessageBodyFailure(s"Cannot decode Json Avro message: $err") + ) + + def encodeEntity[F[_], A: AvroJsonEncoder]: EntityEncoder[F, A] = + EntityEncoder.simple(`Content-Type`(application.json))(a => + Chunk.byteVector(AvroJsonEncoder[A].encode(a)) + ) + + trait Implicits: + + given entityDecoder[F[_]: Async, A: AvroJsonDecoder]: EntityDecoder[F, A] = + decodeEntity[F, A] + + given entityEncoder[F[_], A: AvroJsonEncoder]: EntityEncoder[F, A] = + encodeEntity[F, A] + + object Implicits extends Implicits diff --git a/modules/messages/src/main/avro/api.avdl b/modules/messages/src/main/avro/api.avdl new file mode 100644 index 00000000..7c9da800 --- /dev/null +++ b/modules/messages/src/main/avro/api.avdl @@ -0,0 +1,10 @@ +@namespace("io.renku.api") +protocol ApiShapes { + + /* An example record for a Project Entity returned from the Search API */ + record Project { + string id; + string name; + string description; + } +} \ No newline at end of file diff --git a/modules/messages/src/main/avro/messages.avdl b/modules/messages/src/main/avro/messages.avdl index 2cfb19b7..04885b52 100644 --- a/modules/messages/src/main/avro/messages.avdl +++ b/modules/messages/src/main/avro/messages.avdl @@ -28,16 +28,4 @@ protocol Messages { string name; timestamp_ms deletedAt; } - - -/* record ProjectMsg { - union { ProjectCreated, ProjectUpdated, ProjectDeleted } message; - } */ - - /* An example record for a Project Entity returned from the Search API */ - record ProjectEntity { - string id; - string name; - string description; - } } \ No newline at end of file diff --git a/modules/redis-client/src/test/scala/io/renku/redis/client/util/TestSearchRedisServer.scala b/modules/redis-client/src/test/scala/io/renku/redis/client/util/TestSearchRedisServer.scala new file mode 100644 index 00000000..44011d7d --- /dev/null +++ b/modules/redis-client/src/test/scala/io/renku/redis/client/util/TestSearchRedisServer.scala @@ -0,0 +1,28 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.redis.client.util + +import cats.effect.{ExitCode, IO, IOApp} +import io.renku.servers.RedisServer + +/** This is a utility to start a Redis server for manual testing */ +object TestSearchRedisServer extends IOApp: + + override def run(args: List[String]): IO[ExitCode] = + (IO(RedisServer.start()) >> IO.never[ExitCode]).as(ExitCode.Success) diff --git a/modules/search-api/README.md b/modules/search-api/README.md new file mode 100644 index 00000000..2883b4d8 --- /dev/null +++ b/modules/search-api/README.md @@ -0,0 +1,3 @@ +# search-api + +This module exposes the Search API. diff --git a/modules/search-api/src/main/scala/io/renku/search/api/HttpApplication.scala b/modules/search-api/src/main/scala/io/renku/search/api/HttpApplication.scala new file mode 100644 index 00000000..3ba034c4 --- /dev/null +++ b/modules/search-api/src/main/scala/io/renku/search/api/HttpApplication.scala @@ -0,0 +1,43 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.api + +import cats.Monad +import cats.effect.{Async, Resource} +import fs2.io.net.Network +import io.renku.solr.client.SolrConfig +import org.http4s.dsl.Http4sDsl +import org.http4s.server.Router +import org.http4s.{HttpApp, HttpRoutes, Request, Response} + +object HttpApplication: + def apply[F[_]: Async: Network]( + solrConfig: SolrConfig + ): Resource[F, HttpApp[F]] = + SearchApi[F](solrConfig).map(new HttpApplication[F](_).router) + +class HttpApplication[F[_]: Monad](searchApi: SearchApi[F]) extends Http4sDsl[F]: + + lazy val router: HttpApp[F] = + Router[F]("/" -> routes).orNotFound + + private lazy val routes: HttpRoutes[F] = HttpRoutes.of[F] { + case GET -> Root / "api" / phrase => searchApi.find(phrase) + case GET -> Root / "ping" => Ok("pong") + } diff --git a/modules/search-api/src/main/scala/io/renku/search/api/HttpServer.scala b/modules/search-api/src/main/scala/io/renku/search/api/HttpServer.scala new file mode 100644 index 00000000..486ec551 --- /dev/null +++ b/modules/search-api/src/main/scala/io/renku/search/api/HttpServer.scala @@ -0,0 +1,38 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.api + +import cats.effect.{Async, Resource} +import com.comcast.ip4s.{Port, ipv4, port} +import fs2.io.net.Network +import org.http4s.HttpApp +import org.http4s.ember.server.EmberServerBuilder +import org.http4s.server.Server + +object HttpServer: + + val port: Port = port"8080" + + def build[F[_]: Async: Network](app: HttpApp[F]): Resource[F, Server] = + EmberServerBuilder + .default[F] + .withHost(ipv4"0.0.0.0") + .withPort(port) + .withHttpApp(app) + .build diff --git a/modules/search-api/src/main/scala/io/renku/search/api/Microservice.scala b/modules/search-api/src/main/scala/io/renku/search/api/Microservice.scala new file mode 100644 index 00000000..ef4ae132 --- /dev/null +++ b/modules/search-api/src/main/scala/io/renku/search/api/Microservice.scala @@ -0,0 +1,40 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.api + +import cats.effect.{ExitCode, IO, IOApp} +import cats.syntax.all.* +import io.renku.solr.client.SolrConfig +import org.http4s.implicits.* + +import scala.concurrent.duration.Duration + +object Microservice extends IOApp: + + private val solrConfig = SolrConfig( + baseUrl = uri"http://localhost:8983" / "solr", + core = "search-core-test", + commitWithin = Some(Duration.Zero), + logMessageBodies = true + ) + + override def run(args: List[String]): IO[ExitCode] = + (createHttpApp >>= HttpServer.build).use(_ => IO.never).as(ExitCode.Success) + + private def createHttpApp = HttpApplication[IO](solrConfig) diff --git a/modules/search-api/src/main/scala/io/renku/search/api/SearchApi.scala b/modules/search-api/src/main/scala/io/renku/search/api/SearchApi.scala new file mode 100644 index 00000000..55312832 --- /dev/null +++ b/modules/search-api/src/main/scala/io/renku/search/api/SearchApi.scala @@ -0,0 +1,34 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.api + +import cats.effect.{Async, Resource} +import fs2.io.net.Network +import io.renku.search.solr.client.SearchSolrClient +import io.renku.solr.client.SolrConfig +import org.http4s.Response + +trait SearchApi[F[_]]: + def find(phrase: String): F[Response[F]] + +object SearchApi: + def apply[F[_]: Async: Network]( + solrConfig: SolrConfig + ): Resource[F, SearchApi[F]] = + SearchSolrClient[F](solrConfig).map(new SearchApiImpl[F](_)) diff --git a/modules/search-api/src/main/scala/io/renku/search/api/SearchApiImpl.scala b/modules/search-api/src/main/scala/io/renku/search/api/SearchApiImpl.scala new file mode 100644 index 00000000..55179f4b --- /dev/null +++ b/modules/search-api/src/main/scala/io/renku/search/api/SearchApiImpl.scala @@ -0,0 +1,56 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.api + +import cats.effect.Async +import cats.syntax.all.* +import io.renku.api.Project as ApiProject +import io.renku.avro.codec.all.given +import io.renku.avro.codec.json.AvroJsonEncoder +import io.renku.search.http.avro.AvroEntityCodec.Implicits.entityEncoder +import io.renku.search.solr.client.SearchSolrClient +import io.renku.search.solr.documents.Project as SolrProject +import org.http4s.Response +import org.http4s.dsl.Http4sDsl +import scribe.Scribe + +private class SearchApiImpl[F[_]: Async](solrClient: SearchSolrClient[F]) + extends Http4sDsl[F] + with SearchApi[F]: + + private given Scribe[F] = scribe.cats[F] + + override def find(phrase: String): F[Response[F]] = + solrClient + .findProjects(phrase) + .map(toApiModel) + .flatMap(Ok(_)) + .handleErrorWith(errorResponse(phrase)) + + private given AvroJsonEncoder[List[ApiProject]] = + AvroJsonEncoder.encodeList[ApiProject](ApiProject.SCHEMA$) + + private def errorResponse(phrase: String): Throwable => F[Response[F]] = + err => + Scribe[F] + .error(s"Finding by '$phrase' phrase failed", err) + .map(_ => Response[F](InternalServerError).withEntity(err.getMessage)) + + private def toApiModel(entities: List[SolrProject]): List[ApiProject] = + entities.map(p => ApiProject(p.id, p.name, p.description)) diff --git a/modules/search-api/src/test/scala/io/renku/search/api/SearchApiSpec.scala b/modules/search-api/src/test/scala/io/renku/search/api/SearchApiSpec.scala new file mode 100644 index 00000000..25311f27 --- /dev/null +++ b/modules/search-api/src/test/scala/io/renku/search/api/SearchApiSpec.scala @@ -0,0 +1,54 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.api + +import cats.effect.IO +import cats.syntax.all.* +import io.renku.api.Project as ApiProject +import io.renku.avro.codec.AvroDecoder +import io.renku.avro.codec.all.given +import io.renku.avro.codec.json.AvroJsonDecoder +import io.renku.search.http.avro.AvroEntityCodec.given +import io.renku.search.solr.client.SearchSolrClientGenerators.* +import io.renku.search.solr.client.SearchSolrSpec +import io.renku.search.solr.documents.Project as SolrProject +import munit.CatsEffectSuite +import scribe.Scribe + +class SearchApiSpec extends CatsEffectSuite with SearchSolrSpec: + + private given Scribe[IO] = scribe.cats[IO] + + test("do a lookup in Solr to find entities matching the given phrase"): + withSearchSolrClient().use { client => + val project1 = projectDocumentGen("matching", "matching description").generateOne + val project2 = projectDocumentGen("disparate", "disparate description").generateOne + val searchApi = new SearchApiImpl[IO](client) + for { + _ <- (project1 :: project2 :: Nil).traverse_(client.insertProject) + response <- searchApi.find("matching") + results <- response.as[List[ApiProject]] + } yield assert(results contains toApiProject(project1)) + } + + private given AvroJsonDecoder[List[ApiProject]] = + AvroJsonDecoder.decodeList(ApiProject.SCHEMA$) + + private def toApiProject(project: SolrProject) = + ApiProject(project.id, project.name, project.description) diff --git a/modules/search-provision/src/main/scala/io/renku/search/provision/Microservice.scala b/modules/search-provision/src/main/scala/io/renku/search/provision/Microservice.scala index 178d4d75..b932a76e 100644 --- a/modules/search-provision/src/main/scala/io/renku/search/provision/Microservice.scala +++ b/modules/search-provision/src/main/scala/io/renku/search/provision/Microservice.scala @@ -49,7 +49,8 @@ object Microservice extends IOApp: private def startProvisioning: IO[Unit] = SearchProvisioner[IO](queueName, redisUrl, solrConfig) - .use(_.provisionSolr) + .evalMap(_.provisionSolr.start) + .use(_ => IO.never) .handleErrorWith { err => Scribe[IO].error("Starting provisioning failure, retrying", err) >> Temporal[IO].delayBy(startProvisioning, retryOnErrorDelay) diff --git a/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisioner.scala b/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisioner.scala index 845c6fca..7359c063 100644 --- a/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisioner.scala +++ b/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisioner.scala @@ -45,7 +45,7 @@ object SearchProvisioner: .flatMap(qc => SearchSolrClient[F](solrConfig).tupleLeft(qc)) .map { case (qc, sc) => new SearchProvisionerImpl[F](queueName, qc, sc) } -private class SearchProvisionerImpl[F[_]: Async]( +private class SearchProvisionerImpl[F[_]: Async: Scribe]( queueName: QueueName, queueClient: QueueClient[F], solrClient: SearchSolrClient[F] @@ -55,6 +55,7 @@ private class SearchProvisionerImpl[F[_]: Async]( queueClient .acquireEventsStream(queueName, chunkSize = 1, maybeOffset = None) .map(decodeEvent) + .evalTap(decoded => Scribe[F].info(s"Received $decoded")) .flatMap(decoded => Stream.emits[F, ProjectCreated](decoded)) .evalMap(pushToSolr) .compile diff --git a/modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala b/modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala index d559eb97..5106c43a 100644 --- a/modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala +++ b/modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala @@ -31,12 +31,14 @@ import io.renku.redis.client.util.RedisSpec import io.renku.search.solr.client.SearchSolrSpec import io.renku.search.solr.documents.Project import munit.CatsEffectSuite +import scribe.Scribe import java.time.temporal.ChronoUnit import scala.concurrent.duration.* class SearchProvisionerSpec extends CatsEffectSuite with RedisSpec with SearchSolrSpec: + private given Scribe[IO] = scribe.cats[IO] private val avro = AvroIO(ProjectCreated.SCHEMA$) test("can fetch events and send them to Solr"): @@ -56,7 +58,7 @@ class SearchProvisionerSpec extends CatsEffectSuite with RedisSpec with SearchSo docsCollectorFiber <- Stream .awakeEvery[IO](500 millis) - .evalMap(_ => solrClient.findAllProjects) + .evalMap(_ => solrClient.findProjects("*")) .flatMap(Stream.emits(_)) .evalTap(IO.println) .evalMap(d => solrDocs.update(_ + d)) diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClient.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClient.scala index 2e849c64..dd173b82 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClient.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClient.scala @@ -27,7 +27,7 @@ trait SearchSolrClient[F[_]]: def insertProject(project: Project): F[Unit] - def findAllProjects: F[List[Project]] + def findProjects(phrase: String): F[List[Project]] object SearchSolrClient: def apply[F[_]: Async: Network]( diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala index 987ef0dc..e8990898 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala @@ -24,15 +24,17 @@ import io.renku.search.solr.documents.Project import io.renku.search.solr.schema.EntityDocumentSchema import io.renku.solr.client.{QueryString, SolrClient} -class SearchSolrClientImpl[F[_]: Async](solrClient: SolrClient[F]) +private class SearchSolrClientImpl[F[_]: Async](solrClient: SolrClient[F]) extends SearchSolrClient[F]: override def insertProject(project: Project): F[Unit] = solrClient.insert(Seq(project)).void - override def findAllProjects: F[List[Project]] = + override def findProjects(phrase: String): F[List[Project]] = solrClient .query[Project]( - QueryString(s"${EntityDocumentSchema.Fields.entityType}:${Project.entityType}") + QueryString( + s"${EntityDocumentSchema.Fields.entityType}:${Project.entityType} AND (name:$phrase OR description:$phrase)" + ) ) .map(_.responseBody.docs.toList) diff --git a/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientGenerators.scala b/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientGenerators.scala index 04828969..d1f98c05 100644 --- a/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientGenerators.scala +++ b/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientGenerators.scala @@ -24,8 +24,6 @@ import org.scalacheck.Gen object SearchSolrClientGenerators: def projectDocumentGen(name: String, desc: String): Gen[Project] = - Gen.uuid.map(uuid => - Project(uuid.toString, "solr-project", "solr project description") - ) + Gen.uuid.map(uuid => Project(uuid.toString, name, desc)) extension [V](gen: Gen[V]) def generateOne: V = gen.sample.getOrElse(generateOne) diff --git a/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientSpec.scala b/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientSpec.scala index 88318eb8..e38723bd 100644 --- a/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientSpec.scala +++ b/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientSpec.scala @@ -30,7 +30,7 @@ class SearchSolrClientSpec extends CatsEffectSuite with SearchSolrSpec: projectDocumentGen("solr-project", "solr project description").generateOne for { _ <- client.insertProject(project) - r <- client.findAllProjects + r <- client.findProjects("solr") _ = assert(r contains project) } yield () } diff --git a/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/TestSearchSolrServer.scala b/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/TestSearchSolrServer.scala new file mode 100644 index 00000000..36e0fddb --- /dev/null +++ b/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/TestSearchSolrServer.scala @@ -0,0 +1,28 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.solr.client + +import cats.effect.{ExitCode, IO, IOApp} +import io.renku.servers.SolrServer + +/** This is a utility to start a Solr server for manual testing */ +object TestSearchSolrServer extends IOApp: + + override def run(args: List[String]): IO[ExitCode] = + (IO(SolrServer.start()) >> IO.never[ExitCode]).as(ExitCode.Success) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index d696e8c6..f0efc7da 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -66,9 +66,15 @@ object Dependencies { val http4sCore = Seq( "org.http4s" %% "http4s-core" % V.http4s ) + val http4sDsl = Seq( + "org.http4s" %% "http4s-dsl" % V.http4s + ) val http4sClient = Seq( "org.http4s" %% "http4s-ember-client" % V.http4s ) + val http4sServer = Seq( + "org.http4s" %% "http4s-ember-server" % V.http4s + ) val scribe = Seq( "com.outr" %% "scribe" % V.scribe,