diff --git a/modules/commons/src/main/scala/io/renku/search/model/users.scala b/modules/commons/src/main/scala/io/renku/search/model/users.scala index 07369624..b21deb05 100644 --- a/modules/commons/src/main/scala/io/renku/search/model/users.scala +++ b/modules/commons/src/main/scala/io/renku/search/model/users.scala @@ -29,3 +29,24 @@ object users: extension (self: Id) def value: String = self given Transformer[String, Id] = apply given Codec[Id] = Codec.bimap[String, Id](_.value, Id.apply) + + opaque type FirstName = String + object FirstName: + def apply(v: String): FirstName = v + extension (self: FirstName) def value: String = self + given Transformer[String, FirstName] = apply + given Codec[FirstName] = Codec.bimap[String, FirstName](_.value, FirstName.apply) + + opaque type LastName = String + object LastName: + def apply(v: String): LastName = v + extension (self: LastName) def value: String = self + given Transformer[String, LastName] = apply + given Codec[LastName] = Codec.bimap[String, LastName](_.value, LastName.apply) + + opaque type Email = String + object Email: + def apply(v: String): Email = v + extension (self: Email) def value: String = self + given Transformer[String, Email] = apply + given Codec[Email] = Codec.bimap[String, Email](_.value, LastName.apply) diff --git a/modules/commons/src/test/scala/io/renku/search/model/ModelGenerators.scala b/modules/commons/src/test/scala/io/renku/search/model/ModelGenerators.scala index aa0fd87a..44692d89 100644 --- a/modules/commons/src/test/scala/io/renku/search/model/ModelGenerators.scala +++ b/modules/commons/src/test/scala/io/renku/search/model/ModelGenerators.scala @@ -18,16 +18,19 @@ package io.renku.search.model -import io.renku.search.model.projects.* +import cats.syntax.all.* import org.scalacheck.Gen +import org.scalacheck.cats.implicits.* import java.time.Instant import java.time.temporal.ChronoUnit -object ModelGenerators { +object ModelGenerators: - val visibilityGen: Gen[Visibility] = Gen.oneOf(Visibility.values.toList) - val creationDateGen: Gen[CreationDate] = instantGen().map(CreationDate.apply) + val projectVisibilityGen: Gen[projects.Visibility] = + Gen.oneOf(projects.Visibility.values.toList) + val projectCreationDateGen: Gen[projects.CreationDate] = + instantGen().map(projects.CreationDate.apply) private def instantGen( min: Instant = Instant.EPOCH, @@ -37,4 +40,19 @@ object ModelGenerators { .chooseNum(min.toEpochMilli, max.toEpochMilli) .map(Instant.ofEpochMilli(_).truncatedTo(ChronoUnit.MILLIS)) -} + val userIdGen: Gen[users.Id] = Gen.uuid.map(uuid => users.Id(uuid.toString)) + val userFirstNameGen: Gen[users.FirstName] = Gen + .oneOf("Eike", "Kuba", "Ralf", "Lorenzo", "Jean-Pierre", "Alfonso") + .map(users.FirstName.apply) + val userLastNameGen: Gen[users.LastName] = Gen + .oneOf("Kowalski", "Doe", "Tourist", "Milkman", "Da Silva", "Bar") + .map(users.LastName.apply) + def userEmailGen(first: users.FirstName, last: users.LastName): Gen[users.Email] = Gen + .oneOf("mail.com", "hotmail.com", "epfl.ch", "ethz.ch") + .map(v => users.Email(s"$first.$last@$v")) + val userEmailGen: Gen[users.Email] = + ( + Gen.oneOf("mail.com", "hotmail.com", "epfl.ch", "ethz.ch"), + userFirstNameGen, + userLastNameGen + ).mapN((f, l, p) => users.Email(s"$f.$l@$p")) diff --git a/modules/events/src/test/scala/io/renku/events/EventsGenerators.scala b/modules/events/src/test/scala/io/renku/events/EventsGenerators.scala index abc7bce5..a3961154 100644 --- a/modules/events/src/test/scala/io/renku/events/EventsGenerators.scala +++ b/modules/events/src/test/scala/io/renku/events/EventsGenerators.scala @@ -18,7 +18,7 @@ package io.renku.events -import io.renku.events.v1.{ProjectCreated, Visibility} +import io.renku.events.v1.{ProjectCreated, UserAdded, Visibility} import org.scalacheck.Gen import org.scalacheck.Gen.alphaNumChar @@ -46,6 +46,19 @@ object EventsGenerators: Instant.now().truncatedTo(ChronoUnit.MILLIS) ) + def userAddedGen(prefix: String): Gen[UserAdded] = + for + id <- Gen.uuid.map(_.toString) + firstName <- Gen.option(stringGen(max = 5).map(v => s"$prefix-$v")) + lastName <- stringGen(max = 5).map(v => s"$prefix-$v") + email <- Gen.option(stringGen(max = 5).map(host => s"$lastName@$host.com")) + yield UserAdded( + id, + firstName, + Some(lastName), + email + ) + def stringGen(max: Int): Gen[String] = Gen .chooseNum(3, max) diff --git a/modules/search-api/src/main/scala/io/renku/search/api/SearchApiImpl.scala b/modules/search-api/src/main/scala/io/renku/search/api/SearchApiImpl.scala index feb280fb..b95890e2 100644 --- a/modules/search-api/src/main/scala/io/renku/search/api/SearchApiImpl.scala +++ b/modules/search-api/src/main/scala/io/renku/search/api/SearchApiImpl.scala @@ -24,7 +24,7 @@ import io.github.arainko.ducktape.* import io.renku.search.api.data.* import io.renku.search.model.users import io.renku.search.solr.client.SearchSolrClient -import io.renku.search.solr.documents.Project as SolrProject +import io.renku.search.solr.documents.Entity as SolrEntity import io.renku.solr.client.QueryResponse import org.http4s.dsl.Http4sDsl import scribe.Scribe @@ -37,7 +37,7 @@ private class SearchApiImpl[F[_]: Async](solrClient: SearchSolrClient[F]) override def query(query: QueryInput): F[Either[String, SearchResult]] = solrClient - .queryProjects(query.query, query.page.limit + 1, query.page.offset) + .queryEntity(query.query, query.page.limit + 1, query.page.offset) .map(toApiResult(query.page)) .map(_.asRight[String]) .handleErrorWith(errorResponse(query.query.render)) @@ -54,14 +54,14 @@ private class SearchApiImpl[F[_]: Async](solrClient: SearchSolrClient[F]) .map(_.asLeft[SearchResult]) private def toApiResult(currentPage: PageDef)( - solrResult: QueryResponse[SolrProject] + solrResult: QueryResponse[SolrEntity] ): SearchResult = val hasMore = solrResult.responseBody.docs.size > currentPage.limit val pageInfo = PageWithTotals(currentPage, solrResult.responseBody.numFound, hasMore) - val items = solrResult.responseBody.docs.map(toApiProject) + val items = solrResult.responseBody.docs.map(toApiEntity) if (hasMore) SearchResult(items.init, pageInfo) else SearchResult(items, pageInfo) - private lazy val toApiProject: SolrProject => Project = - given Transformer[users.Id, User] = (id: users.Id) => User(id) - _.to[Project] + private lazy val toApiEntity: SolrEntity => SearchEntity = + given Transformer[users.Id, UserId] = (id: users.Id) => UserId(id) + _.to[SearchEntity] diff --git a/modules/search-api/src/main/scala/io/renku/search/api/data/SearchEntity.scala b/modules/search-api/src/main/scala/io/renku/search/api/data/SearchEntity.scala index 997dda03..a1a2f918 100644 --- a/modules/search-api/src/main/scala/io/renku/search/api/data/SearchEntity.scala +++ b/modules/search-api/src/main/scala/io/renku/search/api/data/SearchEntity.scala @@ -18,12 +18,12 @@ package io.renku.search.api.data +import io.bullet.borer.NullOptions.given import io.bullet.borer.derivation.MapBasedCodecs.{deriveAllCodecs, deriveCodec} import io.bullet.borer.{AdtEncodingStrategy, Codec, Decoder, Encoder} -import io.bullet.borer.NullOptions.given import io.renku.search.model.* import sttp.tapir.Schema.SName -import sttp.tapir.SchemaType.{SDateTime, SProductField} +import sttp.tapir.SchemaType.{SCoproduct, SDateTime, SProductField, SRef} import sttp.tapir.generic.Configuration import sttp.tapir.{FieldName, Schema, SchemaType} @@ -36,7 +36,7 @@ final case class Project( repositories: Seq[projects.Repository], visibility: projects.Visibility, description: Option[projects.Description] = None, - createdBy: User, + createdBy: UserId, creationDate: projects.CreationDate, score: Option[Double] = None ) extends SearchEntity @@ -52,14 +52,26 @@ object Project: private given Schema[projects.CreationDate] = Schema(SDateTime()) given Schema[Project] = Schema.derived[Project] +final case class UserId(id: users.Id) +object UserId: + given Codec[UserId] = deriveCodec[UserId] + + private given Schema[users.Id] = Schema.string[users.Id] + given Schema[UserId] = Schema.derived[UserId] + final case class User( - id: users.Id -) + id: users.Id, + firstName: Option[users.FirstName] = None, + lastName: Option[users.LastName] = None, + email: Option[users.Email] = None, + score: Option[Double] = None +) extends SearchEntity object User: - given Codec[User] = deriveCodec[User] - private given Schema[users.Id] = Schema.string[users.Id] + private given Schema[users.FirstName] = Schema.string[users.FirstName] + private given Schema[users.LastName] = Schema.string[users.LastName] + private given Schema[users.Email] = Schema.string[users.Email] given Schema[User] = Schema.derived[User] object SearchEntity: @@ -71,13 +83,14 @@ object SearchEntity: given Schema[SearchEntity] = { val derived = Schema.derived[SearchEntity] derived.schemaType match { - case s: SchemaType.SCoproduct[_] => + case s: SCoproduct[_] => derived.copy(schemaType = s.addDiscriminatorField( FieldName(discriminatorField), Schema.string, List( - implicitly[Schema[Project]].name.map(SchemaType.SRef(_)).map("Project" -> _) + summon[Schema[Project]].name.map(SRef(_)).map("Project" -> _), + summon[Schema[User]].name.map(SRef(_)).map("User" -> _) ).flatten.toMap ) ) diff --git a/modules/search-api/src/test/scala/io/renku/search/api/SearchApiSpec.scala b/modules/search-api/src/test/scala/io/renku/search/api/SearchApiSpec.scala index b4e85d71..3e60c674 100644 --- a/modules/search-api/src/test/scala/io/renku/search/api/SearchApiSpec.scala +++ b/modules/search-api/src/test/scala/io/renku/search/api/SearchApiSpec.scala @@ -19,14 +19,19 @@ package io.renku.search.api import cats.effect.IO +import cats.syntax.all.* import io.github.arainko.ducktape.* import io.renku.search.api.data.* import io.renku.search.model.users import io.renku.search.query.Query import io.renku.search.solr.client.SearchSolrClientGenerators.* import io.renku.search.solr.client.SearchSolrSpec -import io.renku.search.solr.documents.Project as SolrProject import io.renku.search.solr.documents.Project.given +import io.renku.search.solr.documents.{ + Entity as SolrEntity, + Project as SolrProject, + User as SolrUser +} import munit.CatsEffectSuite import scribe.Scribe @@ -44,15 +49,36 @@ class SearchApiSpec extends CatsEffectSuite with SearchSolrSpec: results <- searchApi .query(mkQuery("matching")) .map(_.fold(err => fail(s"Calling Search API failed with $err"), identity)) - } yield assert(results.items.map(scoreToNone) contains toApiProject(project1)) + } yield assert { + results.items.map(scoreToNone) contains toApiEntity(project1) + } + } + + test("return Project and User entities"): + withSearchSolrClient().use { client => + val project = projectDocumentGen("exclusive", "exclusive description").generateOne + val user = SolrUser(project.createdBy, users.FirstName("exclusive").some) + val searchApi = new SearchApiImpl[IO](client) + for { + _ <- client.insert(project :: Nil) + _ <- client.insert(user :: Nil) + results <- searchApi + .query(mkQuery("exclusive")) + .map(_.fold(err => fail(s"Calling Search API failed with $err"), identity)) + } yield assert { + toApiEntities(project, user).diff(results.items.map(scoreToNone)).isEmpty + } } private def scoreToNone(e: SearchEntity): SearchEntity = e match - case p: Project => p.copy(score = None) + case e: Project => e.copy(score = None) + case e: User => e.copy(score = None) private def mkQuery(phrase: String): QueryInput = - QueryInput.pageOne(Query.parse(phrase).fold(sys.error, identity)) + QueryInput.pageOne(Query.parse(s"Fields $phrase").fold(sys.error, identity)) + + private def toApiEntities(e: SolrEntity*) = e.map(toApiEntity) - private def toApiProject(p: SolrProject) = - given Transformer[users.Id, User] = (id: users.Id) => User(id) - p.to[Project] + private def toApiEntity(e: SolrEntity) = + given Transformer[users.Id, UserId] = (id: users.Id) => UserId(id) + e.to[SearchEntity] diff --git a/modules/search-provision/src/main/scala/io/renku/search/provision/Microservice.scala b/modules/search-provision/src/main/scala/io/renku/search/provision/Microservice.scala index 013d33d2..43104b52 100644 --- a/modules/search-provision/src/main/scala/io/renku/search/provision/Microservice.scala +++ b/modules/search-provision/src/main/scala/io/renku/search/provision/Microservice.scala @@ -18,9 +18,11 @@ package io.renku.search.provision -import cats.effect.{ExitCode, IO, IOApp, Temporal} +import cats.effect.* +import cats.syntax.all.* import io.renku.logging.LoggingSetup import io.renku.search.provision.project.ProjectCreatedProvisioning +import io.renku.search.provision.user.UserAddedProvisioning import io.renku.search.solr.schema.Migrations import io.renku.solr.client.migration.SchemaMigrator import scribe.Scribe @@ -36,18 +38,31 @@ object Microservice extends IOApp: config <- loadConfig _ <- IO(LoggingSetup.doConfigure(config.verbosity)) _ <- runSolrMigrations(config) - _ <- startProvisioning(config) + _ <- startProvisioners(config) } yield ExitCode.Success - private def startProvisioning(cfg: SearchProvisionConfig): IO[Unit] = - ProjectCreatedProvisioning - .make[IO](cfg.queuesConfig.projectCreated, cfg.redisConfig, cfg.solrConfig) - .evalMap(_.provisioningProcess.start) - .use(_ => IO.never) - .handleErrorWith { err => - Scribe[IO].error("Starting provisioning failure, retrying", err) >> - Temporal[IO].delayBy(startProvisioning(cfg), cfg.retryOnErrorDelay) - } + private def startProvisioners(cfg: SearchProvisionConfig): IO[Unit] = + List( + "ProjectCreated" -> ProjectCreatedProvisioning + .make[IO](cfg.queuesConfig.projectCreated, cfg.redisConfig, cfg.solrConfig), + "UserAdded" -> UserAddedProvisioning + .make[IO](cfg.queuesConfig.userAdded, cfg.redisConfig, cfg.solrConfig) + ).traverse_(startProcess(cfg)) + + private def startProcess( + cfg: SearchProvisionConfig + ): ((String, Resource[IO, SolrProvisioningProcess[IO]])) => IO[Unit] = { + case t @ (name, resource) => + resource + .evalMap(_.provisioningProcess.start) + .use(_ => IO.never) + .handleErrorWith { err => + Scribe[IO].error( + s"Starting provisioning process for '$name' failed, retrying", + err + ) >> Temporal[IO].delayBy(startProcess(cfg)(t), cfg.retryOnErrorDelay) + } + } private def runSolrMigrations(cfg: SearchProvisionConfig): IO[Unit] = SchemaMigrator[IO](cfg.solrConfig) diff --git a/modules/search-provision/src/main/scala/io/renku/search/provision/QueuesConfig.scala b/modules/search-provision/src/main/scala/io/renku/search/provision/QueuesConfig.scala index 98ac0ef7..e9791428 100644 --- a/modules/search-provision/src/main/scala/io/renku/search/provision/QueuesConfig.scala +++ b/modules/search-provision/src/main/scala/io/renku/search/provision/QueuesConfig.scala @@ -18,16 +18,19 @@ package io.renku.search.provision +import cats.syntax.all.* import ciris.{ConfigValue, Effect} import io.renku.redis.client.QueueName import io.renku.search.config.ConfigValues final case class QueuesConfig( - projectCreated: QueueName + projectCreated: QueueName, + userAdded: QueueName ) object QueuesConfig: val config: ConfigValue[Effect, QueuesConfig] = - ConfigValues - .eventQueue("projectCreated") - .map(QueuesConfig.apply) + ( + ConfigValues.eventQueue("projectCreated"), + ConfigValues.eventQueue("userAdded") + ).mapN(QueuesConfig.apply) diff --git a/modules/search-provision/src/main/scala/io/renku/search/provision/user/UserAddedProvisioning.scala b/modules/search-provision/src/main/scala/io/renku/search/provision/user/UserAddedProvisioning.scala new file mode 100644 index 00000000..95aebe95 --- /dev/null +++ b/modules/search-provision/src/main/scala/io/renku/search/provision/user/UserAddedProvisioning.scala @@ -0,0 +1,58 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.provision.user + +import cats.Show +import cats.effect.{Async, Resource} +import cats.syntax.all.* +import fs2.io.net.Network +import io.github.arainko.ducktape.* +import io.renku.avro.codec.decoders.all.given +import io.renku.events.v1 +import io.renku.events.v1.UserAdded +import io.renku.redis.client.{QueueName, RedisConfig} +import io.renku.search.provision.SolrProvisioningProcess +import io.renku.search.solr.documents +import io.renku.solr.client.SolrConfig +import scribe.Scribe + +trait UserAddedProvisioning[F[_]] extends SolrProvisioningProcess[F] + +object UserAddedProvisioning: + + def make[F[_]: Async: Network]( + queueName: QueueName, + redisConfig: RedisConfig, + solrConfig: SolrConfig + ): Resource[F, SolrProvisioningProcess[F]] = + given Scribe[F] = scribe.cats[F] + SolrProvisioningProcess.make[F, UserAdded, documents.User]( + queueName, + UserAdded.SCHEMA$, + redisConfig, + solrConfig + ) + + private given Show[UserAdded] = + Show.show[UserAdded](u => + u.lastName.map(v => s"lastName '$v'").getOrElse(s"id '${u.id}'") + ) + + private given Transformer[UserAdded, documents.User] = + _.into[documents.User].transform(Field.default(_.score)) diff --git a/modules/search-provision/src/test/scala/io/renku/search/provision/project/ProjectCreatedProvisionerSpec.scala b/modules/search-provision/src/test/scala/io/renku/search/provision/project/ProjectCreatedProvisionerSpec.scala index beb71626..a5d53511 100644 --- a/modules/search-provision/src/test/scala/io/renku/search/provision/project/ProjectCreatedProvisionerSpec.scala +++ b/modules/search-provision/src/test/scala/io/renku/search/provision/project/ProjectCreatedProvisionerSpec.scala @@ -31,9 +31,13 @@ import io.renku.queue.client.Generators.messageHeaderGen import io.renku.queue.client.{DataContentType, QueueSpec} import io.renku.redis.client.RedisClientGenerators.* import io.renku.redis.client.{QueueName, RedisClientGenerators} -import io.renku.search.model.{projects, users} +import io.renku.search.model.{EntityType, projects, users} +import io.renku.search.query.Query +import io.renku.search.query.Query.Segment +import io.renku.search.query.Query.Segment.typeIs import io.renku.search.solr.client.SearchSolrSpec -import io.renku.search.solr.documents.Project +import io.renku.search.solr.documents.EntityOps.* +import io.renku.search.solr.documents.{Entity, Project} import munit.CatsEffectSuite import scala.concurrent.duration.* @@ -50,7 +54,7 @@ class ProjectCreatedProvisionerSpec clientsAndProvisioning(queue).use { case (queueClient, solrClient, provisioner) => for - solrDocs <- SignallingRef.of[IO, Set[Project]](Set.empty) + solrDocs <- SignallingRef.of[IO, Set[Entity]](Set.empty) provisioningFiber <- provisioner.provisioningProcess.start @@ -64,9 +68,9 @@ class ProjectCreatedProvisionerSpec docsCollectorFiber <- Stream .awakeEvery[IO](500 millis) - .evalMap(_ => solrClient.findProjects("*")) - .flatMap(Stream.emits(_)) - .evalMap(d => solrDocs.update(_ + d.copy(score = None))) + .evalMap(_ => solrClient.queryEntity(queryProjects, 10, 0)) + .flatMap(qr => Stream.emits(qr.responseBody.docs)) + .evalMap(e => solrDocs.update(_ + e.noneScore)) .compile .drain .start @@ -83,7 +87,7 @@ class ProjectCreatedProvisionerSpec clientsAndProvisioning(queue).use { case (queueClient, solrClient, provisioner) => for - solrDocs <- SignallingRef.of[IO, Set[Project]](Set.empty) + solrDocs <- SignallingRef.of[IO, Set[Entity]](Set.empty) provisioningFiber <- provisioner.provisioningProcess.start @@ -97,10 +101,10 @@ class ProjectCreatedProvisionerSpec docsCollectorFiber <- Stream .awakeEvery[IO](500 millis) - .evalMap(_ => solrClient.findProjects("*")) - .flatMap(Stream.emits(_)) + .evalMap(_ => solrClient.queryEntity(queryProjects, 10, 0)) + .flatMap(qr => Stream.emits(qr.responseBody.docs)) .evalTap(IO.println) - .evalMap(d => solrDocs.update(_ + d.copy(score = None))) + .evalMap(e => solrDocs.update(_ + e.noneScore)) .compile .drain .start @@ -112,6 +116,8 @@ class ProjectCreatedProvisionerSpec yield () } + private lazy val queryProjects = Query(typeIs(EntityType.Project)) + private def clientsAndProvisioning(queueName: QueueName) = (withQueueClient() >>= withSearchSolrClient().tupleLeft) .flatMap { case (rc, sc) => diff --git a/modules/search-provision/src/test/scala/io/renku/search/provision/user/UserAddedProvisionerSpec.scala b/modules/search-provision/src/test/scala/io/renku/search/provision/user/UserAddedProvisionerSpec.scala new file mode 100644 index 00000000..cd170a65 --- /dev/null +++ b/modules/search-provision/src/test/scala/io/renku/search/provision/user/UserAddedProvisionerSpec.scala @@ -0,0 +1,134 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.provision.user + +import cats.effect.{IO, Resource} +import cats.syntax.all.* +import fs2.Stream +import fs2.concurrent.SignallingRef +import io.github.arainko.ducktape.* +import io.renku.avro.codec.AvroIO +import io.renku.avro.codec.encoders.all.given +import io.renku.events.EventsGenerators.userAddedGen +import io.renku.events.v1.UserAdded +import io.renku.queue.client.Generators.messageHeaderGen +import io.renku.queue.client.{DataContentType, QueueSpec} +import io.renku.redis.client.RedisClientGenerators.* +import io.renku.redis.client.{QueueName, RedisClientGenerators} +import io.renku.search.model.{EntityType, users} +import io.renku.search.query.Query +import io.renku.search.query.Query.Segment +import io.renku.search.query.Query.Segment.typeIs +import io.renku.search.solr.client.SearchSolrSpec +import io.renku.search.solr.documents.EntityOps.* +import io.renku.search.solr.documents.{Entity, User} +import munit.CatsEffectSuite + +import scala.concurrent.duration.* + +class UserAddedProvisionerSpec extends CatsEffectSuite with QueueSpec with SearchSolrSpec: + + private val avro = AvroIO(UserAdded.SCHEMA$) + + test("can fetch events binary encoded, decode them, and send them to Solr"): + val queue = RedisClientGenerators.queueNameGen.generateOne + + clientsAndProvisioning(queue).use { case (queueClient, solrClient, provisioner) => + for + solrDocs <- SignallingRef.of[IO, Set[Entity]](Set.empty) + + provisioningFiber <- provisioner.provisioningProcess.start + + message1 = userAddedGen(prefix = "binary").generateOne + _ <- queueClient.enqueue( + queue, + messageHeaderGen(UserAdded.SCHEMA$, DataContentType.Binary).generateOne, + message1 + ) + + docsCollectorFiber <- + Stream + .awakeEvery[IO](500 millis) + .evalMap(_ => solrClient.queryEntity(queryUsers, 10, 0)) + .flatMap(qr => Stream.emits(qr.responseBody.docs)) + .evalMap(e => solrDocs.update(_ + e.noneScore)) + .compile + .drain + .start + + _ <- solrDocs.waitUntil(_ contains toSolrDocument(message1)) + + _ <- provisioningFiber.cancel + _ <- docsCollectorFiber.cancel + yield () + } + + test("can fetch events JSON encoded, decode them, and send them to Solr"): + val queue = RedisClientGenerators.queueNameGen.generateOne + + clientsAndProvisioning(queue).use { case (queueClient, solrClient, provisioner) => + for + solrDocs <- SignallingRef.of[IO, Set[Entity]](Set.empty) + + provisioningFiber <- provisioner.provisioningProcess.start + + message1 = userAddedGen(prefix = "json").generateOne + _ <- queueClient.enqueue( + queue, + messageHeaderGen(UserAdded.SCHEMA$, DataContentType.Json).generateOne, + message1 + ) + + docsCollectorFiber <- + Stream + .awakeEvery[IO](500 millis) + .evalMap(_ => solrClient.queryEntity(queryUsers, 10, 0)) + .flatMap(qr => Stream.emits(qr.responseBody.docs)) + .evalTap(IO.println) + .evalMap(e => solrDocs.update(_ + e.noneScore)) + .compile + .drain + .start + + _ <- solrDocs.waitUntil(_ contains toSolrDocument(message1)) + + _ <- provisioningFiber.cancel + _ <- docsCollectorFiber.cancel + yield () + } + + private lazy val queryUsers = Query(typeIs(EntityType.User)) + + private def clientsAndProvisioning(queueName: QueueName) = + (withQueueClient() >>= withSearchSolrClient().tupleLeft) + .flatMap { case (rc, sc) => + UserAddedProvisioning + .make[IO]( + queueName, + withRedisClient.redisConfig, + withSearchSolrClient.solrConfig + ) + .map((rc, sc, _)) + } + + private def toSolrDocument(added: UserAdded): User = + added.into[User].transform(Field.default(_.score)) + + override def munitFixtures: Seq[Fixture[_]] = + List(withRedisClient, withQueueClient, withSearchSolrClient) diff --git a/modules/search-query/src/main/scala/io/renku/search/query/Query.scala b/modules/search-query/src/main/scala/io/renku/search/query/Query.scala index 3de80a66..58bc4611 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/Query.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/Query.scala @@ -19,14 +19,15 @@ package io.renku.search.query import cats.data.NonEmptyList +import cats.kernel.Monoid import cats.syntax.all.* import io.bullet.borer.{Decoder, Encoder} +import io.renku.search.model.EntityType import io.renku.search.model.projects.Visibility import io.renku.search.query.FieldTerm.Created import io.renku.search.query.Query.Segment import io.renku.search.query.json.QueryJsonCodec import io.renku.search.query.parse.{QueryParser, QueryUtil} -import cats.kernel.Monoid final case class Query( segments: List[Query.Segment] @@ -76,6 +77,9 @@ object Query: def text(phrase: String): Segment = Segment.Text(phrase) + def typeIs(value: EntityType, more: EntityType*): Segment = + Segment.Field(FieldTerm.TypeIs(NonEmptyList(value, more.toList))) + def projectIdIs(value: String, more: String*): Segment = Segment.Field(FieldTerm.ProjectIdIs(NonEmptyList(value, more.toList))) diff --git a/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala b/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala index 9a2401d1..ae936a0a 100644 --- a/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala +++ b/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala @@ -128,8 +128,8 @@ object QueryGenerators: val visibilityTerm: Gen[FieldTerm] = Gen .frequency( - 10 -> ModelGenerators.visibilityGen.map(NonEmptyList.one), - 1 -> CommonGenerators.nelOfN(2, ModelGenerators.visibilityGen) + 10 -> ModelGenerators.projectVisibilityGen.map(NonEmptyList.one), + 1 -> CommonGenerators.nelOfN(2, ModelGenerators.projectVisibilityGen) ) .map(vs => FieldTerm.VisibilityIs(vs.distinct)) diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClient.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClient.scala index 59c324d3..20c7accc 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClient.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClient.scala @@ -21,17 +21,14 @@ package io.renku.search.solr.client import cats.effect.{Async, Resource} import fs2.io.net.Network import io.bullet.borer.Encoder -import io.renku.search.solr.documents.Project +import io.renku.search.solr.documents.Entity import io.renku.solr.client.{SolrClient, SolrConfig} import io.renku.search.query.Query import io.renku.solr.client.QueryResponse trait SearchSolrClient[F[_]]: - def insert[D: Encoder](documents: Seq[D]): F[Unit] - - def findProjects(phrase: String): F[List[Project]] - def queryProjects(query: Query, limit: Int, offset: Int): F[QueryResponse[Project]] + def queryEntity(query: Query, limit: Int, offset: Int): F[QueryResponse[Entity]] object SearchSolrClient: def make[F[_]: Async: Network]( diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala index f061a22a..04e7fd24 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala @@ -21,12 +21,10 @@ package io.renku.search.solr.client import cats.effect.Async import cats.syntax.all.* import io.bullet.borer.Encoder -import io.renku.search.solr.documents.Project -import io.renku.search.solr.query.LuceneQueryInterpreter -import io.renku.search.solr.schema.EntityDocumentSchema -import io.renku.solr.client.{QueryData, QueryString, SolrClient} import io.renku.search.query.Query -import io.renku.solr.client.QueryResponse +import io.renku.search.solr.documents.Entity +import io.renku.search.solr.query.LuceneQueryInterpreter +import io.renku.solr.client.{QueryData, QueryResponse, QueryString, SolrClient} private class SearchSolrClientImpl[F[_]: Async](solrClient: SolrClient[F]) extends SearchSolrClient[F]: @@ -37,29 +35,18 @@ private class SearchSolrClientImpl[F[_]: Async](solrClient: SolrClient[F]) override def insert[D: Encoder](documents: Seq[D]): F[Unit] = solrClient.insert(documents).void - override def queryProjects( + override def queryEntity( query: Query, limit: Int, offset: Int - ): F[QueryResponse[Project]] = + ): F[QueryResponse[Entity]] = for { solrQuery <- interpreter.run(query) _ <- logger.debug(s"Query: ${query.render} ->Solr: $solrQuery") res <- solrClient - .query[Project]( + .query[Entity]( QueryData(QueryString(solrQuery.query.value, limit, offset)) .withSort(solrQuery.sort) .withScore ) } yield res - - override def findProjects(phrase: String): F[List[Project]] = - solrClient - .query[Project]( - QueryData( - QueryString( - s"${EntityDocumentSchema.Fields.entityType}:${Project.entityType} AND (name:$phrase OR description:$phrase)" - ) - ) - ) - .map(_.responseBody.docs.toList) diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/documents/Project.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/documents/Entity.scala similarity index 58% rename from modules/search-solr-client/src/main/scala/io/renku/search/solr/documents/Project.scala rename to modules/search-solr-client/src/main/scala/io/renku/search/solr/documents/Entity.scala index f7d39d4a..0d7f0253 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/documents/Project.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/documents/Entity.scala @@ -18,11 +18,23 @@ package io.renku.search.solr.documents -import io.bullet.borer.NullOptions.given -import io.bullet.borer.derivation.MapBasedCodecs.deriveDecoder -import io.bullet.borer.{Decoder, Encoder} -import io.renku.search.model.* -import io.renku.solr.client.EncoderSupport.deriveWithDiscriminator +import io.bullet.borer.derivation.MapBasedCodecs.* +import io.bullet.borer.{AdtEncodingStrategy, Decoder, Encoder} +import io.renku.search.model.{projects, users} +import io.renku.solr.client.EncoderSupport.* + +sealed trait Entity: + val score: Option[Double] + +object Entity: + + val allTypes: Set[String] = Set(Project.entityType, User.entityType) + + given AdtEncodingStrategy = + AdtEncodingStrategy.flat(typeMemberName = discriminatorField) + + given Encoder[Entity] = deriveEncoder[Entity] + given Decoder[Entity] = deriveAllDecoders[Entity] final case class Project( id: projects.Id, @@ -34,15 +46,20 @@ final case class Project( createdBy: users.Id, creationDate: projects.CreationDate, score: Option[Double] = None -) +) extends Entity object Project: val entityType: String = "Project" - given Encoder[Project] = deriveWithDiscriminator - given Decoder[Seq[User]] = - Decoder[Seq[User]] { reader => - if reader.hasArrayStart then Decoder.forArray[User].map(_.toSeq).read(reader) - else Decoder[User].map(Seq(_)).read(reader) - } - given Decoder[Project] = deriveDecoder + +final case class User( + id: users.Id, + firstName: Option[users.FirstName] = None, + lastName: Option[users.LastName] = None, + email: Option[users.Email] = None, + score: Option[Double] = None +) extends Entity + +object User: + val entityType: String = "User" + given Encoder[User] = deriveWithDiscriminator diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrToken.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrToken.scala index fde8bc15..e5d5b5b1 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrToken.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrToken.scala @@ -18,17 +18,17 @@ package io.renku.search.solr.query -import cats.syntax.all.* -import io.renku.search.query.Field -import cats.data.NonEmptyList import cats.Monoid -import java.time.Instant +import cats.data.NonEmptyList +import cats.syntax.all.* +import io.renku.search.model.EntityType +import io.renku.search.model.projects.Visibility +import io.renku.search.query.{Comparison, Field} import io.renku.search.solr.documents.{Project as SolrProject, User as SolrUser} import io.renku.search.solr.schema.EntityDocumentSchema.Fields as SolrField import io.renku.solr.client.schema.FieldName -import io.renku.search.query.Comparison -import io.renku.search.model.EntityType -import io.renku.search.model.projects.Visibility + +import java.time.Instant opaque type SolrToken = String @@ -73,8 +73,7 @@ object SolrToken: def dateLt(field: Field, date: Instant): SolrToken = fieldIs(field, s"[* TO ${fromInstant(date)}]") - // TODO: currently only projects work, user can't be decoded - val allTypes: SolrToken = fieldIs(Field.Type, "Project") + val allTypes: SolrToken = fieldIs(Field.Type, "*") private def fieldOp(field: Field, op: Comparison, value: SolrToken): SolrToken = val cmp = fromComparison(op) diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/schema/EntityDocumentSchema.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/schema/EntityDocumentSchema.scala index 5f64c246..4698bf93 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/schema/EntityDocumentSchema.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/schema/EntityDocumentSchema.scala @@ -23,16 +23,20 @@ import io.renku.solr.client.schema.* object EntityDocumentSchema: object Fields: - val id: FieldName = FieldName("id") + val createdBy: FieldName = FieldName("createdBy") + val creationDate: FieldName = FieldName("creationDate") + val description: FieldName = FieldName("description") + val email: FieldName = FieldName("email") val entityType: FieldName = FieldName("_type") + val firstName: FieldName = FieldName("firstName") + val id: FieldName = FieldName("id") + val lastName: FieldName = FieldName("lastName") val name: FieldName = FieldName("name") - val slug: FieldName = FieldName("slug") + val nestPath: FieldName = FieldName("_nest_path_") val repositories: FieldName = FieldName("repositories") + val slug: FieldName = FieldName("slug") val visibility: FieldName = FieldName("visibility") - val description: FieldName = FieldName("description") - val createdBy: FieldName = FieldName("createdBy") - val creationDate: FieldName = FieldName("creationDate") - val nestPath: FieldName = FieldName("_nest_path_") + val root: FieldName = FieldName("_root_") val nestParent: FieldName = FieldName("_nest_parent_") // catch-all field @@ -72,3 +76,11 @@ object EntityDocumentSchema: SchemaCommand.Add(CopyFieldRule(Fields.slug, Fields.contentAll)), SchemaCommand.Add(CopyFieldRule(Fields.repositories, Fields.contentAll)) ) + + val userFields: Seq[SchemaCommand] = Seq( + SchemaCommand.Add(Field(Fields.firstName, FieldTypes.string)), + SchemaCommand.Add(Field(Fields.lastName, FieldTypes.string)), + SchemaCommand.Add(Field(Fields.email, FieldTypes.string)), + SchemaCommand.Add(CopyFieldRule(Fields.firstName, Fields.contentAll)), + SchemaCommand.Add(CopyFieldRule(Fields.lastName, Fields.contentAll)) + ) diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/schema/Migrations.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/schema/Migrations.scala index 380ec44f..4dc4da2a 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/schema/Migrations.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/schema/Migrations.scala @@ -24,6 +24,7 @@ object Migrations { val all: Seq[SchemaMigration] = Seq( SchemaMigration(version = 1L, EntityDocumentSchema.initialEntityDocumentAdd), - SchemaMigration(version = 2L, EntityDocumentSchema.copyContentField) + SchemaMigration(version = 2L, EntityDocumentSchema.copyContentField), + SchemaMigration(version = 3L, EntityDocumentSchema.userFields) ) } diff --git a/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientGenerators.scala b/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientGenerators.scala index 536b63b5..f824edc7 100644 --- a/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientGenerators.scala +++ b/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientGenerators.scala @@ -31,7 +31,7 @@ object SearchSolrClientGenerators: Gen.uuid.map(uuid => projects.Id(uuid.toString)) def projectDocumentGen(name: String, desc: String): Gen[Project] = - (projectIdGen, userIdGen, visibilityGen, creationDateGen) + (projectIdGen, userIdGen, projectVisibilityGen, projectCreationDateGen) .mapN((projectId, creatorId, visibility, creationDate) => Project( projectId, @@ -46,10 +46,13 @@ object SearchSolrClientGenerators: ) def userDocumentGen: Gen[User] = - userIdGen.map(id => User(id)) - - private def userIdGen: Gen[users.Id] = Gen.uuid.map(uuid => users.Id(uuid.toString)) + (userIdGen, Gen.option(userFirstNameGen), Gen.option(userLastNameGen)) + .flatMapN { case (id, f, l) => + val e = (f, l).flatMapN(userEmailGen(_, _).generateOption) + User(id, f, l, e) + } extension [V](gen: Gen[V]) def generateOne: V = gen.sample.getOrElse(generateOne) + def generateOption: Option[V] = Gen.option(gen).sample.getOrElse(generateOption) def generateAs[D](f: V => D): D = f(generateOne) diff --git a/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientSpec.scala b/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientSpec.scala index f36d4250..4b81a855 100644 --- a/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientSpec.scala +++ b/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientSpec.scala @@ -19,19 +19,33 @@ package io.renku.search.solr.client import cats.effect.IO +import cats.syntax.all.* +import io.renku.search.model.users import io.renku.search.query.Query import io.renku.search.solr.client.SearchSolrClientGenerators.* +import io.renku.search.solr.documents.EntityOps.* import munit.CatsEffectSuite class SearchSolrClientSpec extends CatsEffectSuite with SearchSolrSpec: - test("be able to insert and fetch a project document"): + test("be able to insert and fetch a Project document"): withSearchSolrClient().use { client => val project = projectDocumentGen("solr-project", "solr project description").generateOne for { _ <- client.insert(Seq(project)) - r <- client.queryProjects(Query.parse("solr").toOption.get, 10, 0) - _ = assert(r.responseBody.docs.map(_.copy(score = None)) contains project) + r <- client.queryEntity(Query.parse("solr").toOption.get, 10, 0) + _ = assert(r.responseBody.docs.map(_.noneScore) contains project) + } yield () + } + + test("be able to insert and fetch a User document"): + withSearchSolrClient().use { client => + val firstName = users.FirstName("Johnny") + val user = userDocumentGen.generateOne.copy(firstName = firstName.some) + for { + _ <- client.insert(Seq(user)) + r <- client.queryEntity(Query.parse(firstName.value).toOption.get, 10, 0) + _ = assert(r.responseBody.docs.map(_.noneScore) contains user) } yield () } diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/documents/User.scala b/modules/search-solr-client/src/test/scala/io/renku/search/solr/documents/EntityOps.scala similarity index 67% rename from modules/search-solr-client/src/main/scala/io/renku/search/solr/documents/User.scala rename to modules/search-solr-client/src/test/scala/io/renku/search/solr/documents/EntityOps.scala index 75d8ddec..d392d007 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/documents/User.scala +++ b/modules/search-solr-client/src/test/scala/io/renku/search/solr/documents/EntityOps.scala @@ -18,15 +18,11 @@ package io.renku.search.solr.documents -import io.bullet.borer.derivation.MapBasedCodecs.deriveDecoder -import io.bullet.borer.{Decoder, Encoder} -import io.renku.search.model.users -import io.renku.solr.client.EncoderSupport.deriveWithDiscriminator +object EntityOps extends EntityOps +trait EntityOps: -final case class User(id: users.Id) - -object User: - val entityType: String = "User" - - given Encoder[User] = deriveWithDiscriminator - given Decoder[User] = deriveDecoder + extension (entity: Entity) + def noneScore: Entity = entity match { + case e: Project => e.copy(score = None) + case e: User => e.copy(score = None) + } diff --git a/modules/solr-client/src/main/scala/io/renku/solr/client/EncoderSupport.scala b/modules/solr-client/src/main/scala/io/renku/solr/client/EncoderSupport.scala index 5aaf6b90..df7765cc 100644 --- a/modules/solr-client/src/main/scala/io/renku/solr/client/EncoderSupport.scala +++ b/modules/solr-client/src/main/scala/io/renku/solr/client/EncoderSupport.scala @@ -26,10 +26,12 @@ import scala.deriving.* object EncoderSupport { + val discriminatorField: String = "_type" + inline def deriveWithDiscriminator[A <: Product](using Mirror.ProductOf[A] ): Encoder[A] = - Macros.createEncoder[String, A]("_type") + Macros.createEncoder[String, A](discriminatorField) private object Macros {