From 5cf9fffc927f162bd8754541d76188c2bec6d2fb Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Mon, 29 Jan 2024 18:53:11 +0100 Subject: [PATCH] feat: Search API Microservice and manual testing facilities --- .../client/util/TestSearchRedisServer.scala | 28 ++++++++++++ .../io/renku/search/api/HttpApplication.scala | 44 +++++++++++++++++++ .../io/renku/search/api/HttpServer.scala | 38 ++++++++++++++++ .../io/renku/search/api/Microservice.scala | 43 ++++++++++++++++++ .../scala/io/renku/search/api/SearchApi.scala | 5 ++- .../io/renku/search/api/SearchApiImpl.scala | 15 ++++++- .../io/renku/search/api/SearchApiSpec.scala | 3 ++ .../renku/search/provision/Microservice.scala | 3 +- .../search/provision/SearchProvisioner.scala | 3 +- .../provision/SearchProvisionerSpec.scala | 2 + .../solr/client/TestSearchSolrServer.scala | 28 ++++++++++++ 11 files changed, 207 insertions(+), 5 deletions(-) create mode 100644 modules/redis-client/src/test/scala/io/renku/redis/client/util/TestSearchRedisServer.scala create mode 100644 modules/search-api/src/main/scala/io/renku/search/api/HttpApplication.scala create mode 100644 modules/search-api/src/main/scala/io/renku/search/api/HttpServer.scala create mode 100644 modules/search-api/src/main/scala/io/renku/search/api/Microservice.scala create mode 100644 modules/search-solr-client/src/test/scala/io/renku/search/solr/client/TestSearchSolrServer.scala diff --git a/modules/redis-client/src/test/scala/io/renku/redis/client/util/TestSearchRedisServer.scala b/modules/redis-client/src/test/scala/io/renku/redis/client/util/TestSearchRedisServer.scala new file mode 100644 index 00000000..44011d7d --- /dev/null +++ b/modules/redis-client/src/test/scala/io/renku/redis/client/util/TestSearchRedisServer.scala @@ -0,0 +1,28 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.redis.client.util + +import cats.effect.{ExitCode, IO, IOApp} +import io.renku.servers.RedisServer + +/** This is a utility to start a Redis server for manual testing */ +object TestSearchRedisServer extends IOApp: + + override def run(args: List[String]): IO[ExitCode] = + (IO(RedisServer.start()) >> IO.never[ExitCode]).as(ExitCode.Success) diff --git a/modules/search-api/src/main/scala/io/renku/search/api/HttpApplication.scala b/modules/search-api/src/main/scala/io/renku/search/api/HttpApplication.scala new file mode 100644 index 00000000..615456ab --- /dev/null +++ b/modules/search-api/src/main/scala/io/renku/search/api/HttpApplication.scala @@ -0,0 +1,44 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.api + +import cats.Monad +import cats.effect.{Async, Resource} +import fs2.io.net.Network +import io.renku.solr.client.SolrConfig +import org.http4s.dsl.Http4sDsl +import org.http4s.server.Router +import org.http4s.{HttpApp, HttpRoutes, Request, Response} +import scribe.Scribe + +object HttpApplication: + def apply[F[_]: Async: Network: Scribe]( + solrConfig: SolrConfig + ): Resource[F, HttpApp[F]] = + SearchApi[F](solrConfig).map(new HttpApplication[F](_).router) + +class HttpApplication[F[_]: Monad](searchApi: SearchApi[F]) extends Http4sDsl[F]: + + lazy val router: HttpApp[F] = + Router[F]("/" -> routes).orNotFound + + private lazy val routes: HttpRoutes[F] = HttpRoutes.of[F] { + case GET -> Root / "api" / phrase => searchApi.find(phrase) + case GET -> Root / "ping" => Ok("pong") + } diff --git a/modules/search-api/src/main/scala/io/renku/search/api/HttpServer.scala b/modules/search-api/src/main/scala/io/renku/search/api/HttpServer.scala new file mode 100644 index 00000000..486ec551 --- /dev/null +++ b/modules/search-api/src/main/scala/io/renku/search/api/HttpServer.scala @@ -0,0 +1,38 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.api + +import cats.effect.{Async, Resource} +import com.comcast.ip4s.{Port, ipv4, port} +import fs2.io.net.Network +import org.http4s.HttpApp +import org.http4s.ember.server.EmberServerBuilder +import org.http4s.server.Server + +object HttpServer: + + val port: Port = port"8080" + + def build[F[_]: Async: Network](app: HttpApp[F]): Resource[F, Server] = + EmberServerBuilder + .default[F] + .withHost(ipv4"0.0.0.0") + .withPort(port) + .withHttpApp(app) + .build diff --git a/modules/search-api/src/main/scala/io/renku/search/api/Microservice.scala b/modules/search-api/src/main/scala/io/renku/search/api/Microservice.scala new file mode 100644 index 00000000..0f76eba2 --- /dev/null +++ b/modules/search-api/src/main/scala/io/renku/search/api/Microservice.scala @@ -0,0 +1,43 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.api + +import cats.effect.{ExitCode, IO, IOApp} +import cats.syntax.all.* +import io.renku.solr.client.SolrConfig +import org.http4s.implicits.* +import scribe.Scribe + +import scala.concurrent.duration.Duration + +object Microservice extends IOApp: + + private given Scribe[IO] = scribe.cats[IO] + + private val solrConfig = SolrConfig( + baseUrl = uri"http://localhost:8983" / "solr", + core = "search-core-test", + commitWithin = Some(Duration.Zero), + logMessageBodies = true + ) + + override def run(args: List[String]): IO[ExitCode] = + (createHttpApp >>= HttpServer.build).use(_ => IO.never).as(ExitCode.Success) + + private def createHttpApp = HttpApplication[IO](solrConfig) diff --git a/modules/search-api/src/main/scala/io/renku/search/api/SearchApi.scala b/modules/search-api/src/main/scala/io/renku/search/api/SearchApi.scala index 97f65ddd..16217424 100644 --- a/modules/search-api/src/main/scala/io/renku/search/api/SearchApi.scala +++ b/modules/search-api/src/main/scala/io/renku/search/api/SearchApi.scala @@ -23,10 +23,13 @@ import fs2.io.net.Network import io.renku.search.solr.client.SearchSolrClient import io.renku.solr.client.SolrConfig import org.http4s.Response +import scribe.Scribe trait SearchApi[F[_]]: def find(phrase: String): F[Response[F]] object SearchApi: - def apply[F[_]: Async: Network](solrConfig: SolrConfig): Resource[F, SearchApi[F]] = + def apply[F[_]: Async: Network: Scribe]( + solrConfig: SolrConfig + ): Resource[F, SearchApi[F]] = SearchSolrClient[F](solrConfig).map(new SearchApiImpl[F](_)) diff --git a/modules/search-api/src/main/scala/io/renku/search/api/SearchApiImpl.scala b/modules/search-api/src/main/scala/io/renku/search/api/SearchApiImpl.scala index 027e6506..3431a908 100644 --- a/modules/search-api/src/main/scala/io/renku/search/api/SearchApiImpl.scala +++ b/modules/search-api/src/main/scala/io/renku/search/api/SearchApiImpl.scala @@ -28,13 +28,18 @@ import io.renku.search.solr.client.SearchSolrClient import io.renku.search.solr.documents.Project as SolrProject import org.http4s.Response import org.http4s.dsl.Http4sDsl +import scribe.Scribe -private class SearchApiImpl[F[_]: Async](solrClient: SearchSolrClient[F]) +private class SearchApiImpl[F[_]: Async: Scribe](solrClient: SearchSolrClient[F]) extends Http4sDsl[F] with SearchApi[F]: override def find(phrase: String): F[Response[F]] = - solrClient.findProjects(phrase).map(toApiModel).map(toAvroResponse) + solrClient + .findProjects(phrase) + .map(toApiModel) + .map(toAvroResponse) + .handleErrorWith(errorResponse(phrase)) private given AvroJsonEncoder[List[ApiProject]] = AvroJsonEncoder.encodeList[ApiProject](ApiProject.SCHEMA$) @@ -43,5 +48,11 @@ private class SearchApiImpl[F[_]: Async](solrClient: SearchSolrClient[F]) Response[F](Ok) .withEntity(entities) + private def errorResponse(phrase: String): Throwable => F[Response[F]] = + err => + Scribe[F] + .error(s"Finding by '$phrase' phrase failed", err) + .map(_ => Response[F](InternalServerError).withEntity(err.getMessage)) + private def toApiModel(entities: List[SolrProject]): List[ApiProject] = entities.map(p => ApiProject(p.id, p.name, p.description)) diff --git a/modules/search-api/src/test/scala/io/renku/search/api/SearchApiSpec.scala b/modules/search-api/src/test/scala/io/renku/search/api/SearchApiSpec.scala index 05638f3b..a326f836 100644 --- a/modules/search-api/src/test/scala/io/renku/search/api/SearchApiSpec.scala +++ b/modules/search-api/src/test/scala/io/renku/search/api/SearchApiSpec.scala @@ -29,9 +29,12 @@ import io.renku.search.solr.client.SearchSolrClientGenerators.* import io.renku.search.solr.client.SearchSolrSpec import io.renku.search.solr.documents.Project as SolrProject import munit.CatsEffectSuite +import scribe.Scribe class SearchApiSpec extends CatsEffectSuite with SearchSolrSpec: + private given Scribe[IO] = scribe.cats[IO] + test("do a lookup in Solr to find entities matching the given phrase"): withSearchSolrClient().use { client => val project1 = projectDocumentGen("matching", "matching description").generateOne diff --git a/modules/search-provision/src/main/scala/io/renku/search/provision/Microservice.scala b/modules/search-provision/src/main/scala/io/renku/search/provision/Microservice.scala index 178d4d75..b932a76e 100644 --- a/modules/search-provision/src/main/scala/io/renku/search/provision/Microservice.scala +++ b/modules/search-provision/src/main/scala/io/renku/search/provision/Microservice.scala @@ -49,7 +49,8 @@ object Microservice extends IOApp: private def startProvisioning: IO[Unit] = SearchProvisioner[IO](queueName, redisUrl, solrConfig) - .use(_.provisionSolr) + .evalMap(_.provisionSolr.start) + .use(_ => IO.never) .handleErrorWith { err => Scribe[IO].error("Starting provisioning failure, retrying", err) >> Temporal[IO].delayBy(startProvisioning, retryOnErrorDelay) diff --git a/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisioner.scala b/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisioner.scala index 845c6fca..7359c063 100644 --- a/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisioner.scala +++ b/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisioner.scala @@ -45,7 +45,7 @@ object SearchProvisioner: .flatMap(qc => SearchSolrClient[F](solrConfig).tupleLeft(qc)) .map { case (qc, sc) => new SearchProvisionerImpl[F](queueName, qc, sc) } -private class SearchProvisionerImpl[F[_]: Async]( +private class SearchProvisionerImpl[F[_]: Async: Scribe]( queueName: QueueName, queueClient: QueueClient[F], solrClient: SearchSolrClient[F] @@ -55,6 +55,7 @@ private class SearchProvisionerImpl[F[_]: Async]( queueClient .acquireEventsStream(queueName, chunkSize = 1, maybeOffset = None) .map(decodeEvent) + .evalTap(decoded => Scribe[F].info(s"Received $decoded")) .flatMap(decoded => Stream.emits[F, ProjectCreated](decoded)) .evalMap(pushToSolr) .compile diff --git a/modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala b/modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala index 755e39b0..5106c43a 100644 --- a/modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala +++ b/modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala @@ -31,12 +31,14 @@ import io.renku.redis.client.util.RedisSpec import io.renku.search.solr.client.SearchSolrSpec import io.renku.search.solr.documents.Project import munit.CatsEffectSuite +import scribe.Scribe import java.time.temporal.ChronoUnit import scala.concurrent.duration.* class SearchProvisionerSpec extends CatsEffectSuite with RedisSpec with SearchSolrSpec: + private given Scribe[IO] = scribe.cats[IO] private val avro = AvroIO(ProjectCreated.SCHEMA$) test("can fetch events and send them to Solr"): diff --git a/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/TestSearchSolrServer.scala b/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/TestSearchSolrServer.scala new file mode 100644 index 00000000..36e0fddb --- /dev/null +++ b/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/TestSearchSolrServer.scala @@ -0,0 +1,28 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.solr.client + +import cats.effect.{ExitCode, IO, IOApp} +import io.renku.servers.SolrServer + +/** This is a utility to start a Solr server for manual testing */ +object TestSearchSolrServer extends IOApp: + + override def run(args: List[String]): IO[ExitCode] = + (IO(SolrServer.start()) >> IO.never[ExitCode]).as(ExitCode.Success)