diff --git a/build.sbt b/build.sbt index 287f638a..b9260f01 100644 --- a/build.sbt +++ b/build.sbt @@ -68,7 +68,8 @@ lazy val commons = project .settings( name := "commons", libraryDependencies ++= - Dependencies.catsCore ++ + Dependencies.borer ++ + Dependencies.catsCore ++ Dependencies.catsEffect ++ Dependencies.fs2Core ++ Dependencies.scodecBits ++ @@ -173,7 +174,7 @@ lazy val searchSolrClient = project .dependsOn( avroCodec % "compile->compile;test->test", solrClient % "compile->compile;test->test", - commons % "test->test" + commons % "compile->compile;test->test" ) lazy val avroCodec = project @@ -296,9 +297,8 @@ lazy val commonSettings = Seq( ), Compile / console / scalacOptions := (Compile / scalacOptions).value.filterNot(_ == "-Xfatal-warnings"), Test / console / scalacOptions := (Compile / console / scalacOptions).value, - libraryDependencies ++= ( - Dependencies.scribe - ), + libraryDependencies ++= + Dependencies.scribe, libraryDependencies ++= ( Dependencies.catsEffectMunit ++ Dependencies.scalacheckEffectMunit diff --git a/modules/commons/src/main/scala/io/renku/search/borer/codecs/DateTimeDecoders.scala b/modules/commons/src/main/scala/io/renku/search/borer/codecs/DateTimeDecoders.scala new file mode 100644 index 00000000..f38eaf64 --- /dev/null +++ b/modules/commons/src/main/scala/io/renku/search/borer/codecs/DateTimeDecoders.scala @@ -0,0 +1,38 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.borer.codecs + +import cats.syntax.all.* +import io.bullet.borer.Decoder +import io.bullet.borer.Decoder.* + +import java.time.Instant +import java.time.format.DateTimeParseException + +trait DateTimeDecoders: + given Decoder[Instant] = DateTimeDecoders.forInstant + +object DateTimeDecoders: + + val forInstant: Decoder[Instant] = + Decoder.forString.mapEither { v => + Either + .catchOnly[DateTimeParseException](Instant.parse(v)) + .leftMap(_.getMessage) + } diff --git a/modules/commons/src/main/scala/io/renku/search/borer/codecs/DateTimeEncoders.scala b/modules/commons/src/main/scala/io/renku/search/borer/codecs/DateTimeEncoders.scala new file mode 100644 index 00000000..2b6b4884 --- /dev/null +++ b/modules/commons/src/main/scala/io/renku/search/borer/codecs/DateTimeEncoders.scala @@ -0,0 +1,29 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.borer.codecs + +import io.bullet.borer.Encoder + +import java.time.Instant + +trait DateTimeEncoders: + given Encoder[Instant] = DateTimeEncoders.forInstant + +object DateTimeEncoders: + val forInstant: Encoder[Instant] = Encoder.forString.contramap[Instant](_.toString) diff --git a/modules/commons/src/main/scala/io/renku/search/borer/codecs/all.scala b/modules/commons/src/main/scala/io/renku/search/borer/codecs/all.scala new file mode 100644 index 00000000..ae13cf7c --- /dev/null +++ b/modules/commons/src/main/scala/io/renku/search/borer/codecs/all.scala @@ -0,0 +1,23 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.borer.codecs + +trait all extends DateTimeEncoders, DateTimeDecoders + +object all extends all diff --git a/modules/commons/src/main/scala/io/renku/search/model/projects.scala b/modules/commons/src/main/scala/io/renku/search/model/projects.scala new file mode 100644 index 00000000..61cca498 --- /dev/null +++ b/modules/commons/src/main/scala/io/renku/search/model/projects.scala @@ -0,0 +1,78 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.model + +import io.bullet.borer.derivation.MapBasedCodecs.* +import io.bullet.borer.{Codec, Decoder, Encoder} +import io.renku.search.borer.codecs.all.given + +import java.time.Instant + +object projects: + + opaque type Id = String + object Id: + def apply(v: String): Id = v + extension (self: Id) def value: String = self + given Codec[Id] = Codec.of[String] + + opaque type Name = String + object Name: + def apply(v: String): Name = v + extension (self: Name) def value: String = self + given Codec[Name] = Codec.of[String] + + opaque type Slug = String + object Slug: + def apply(v: String): Slug = v + extension (self: Slug) def value: String = self + given Codec[Slug] = Codec.of[String] + + opaque type Repository = String + object Repository: + def apply(v: String): Repository = v + extension (self: Repository) def value: String = self + given Codec[Repository] = Codec.of[String] + + opaque type Description = String + object Description: + def apply(v: String): Description = v + def from(v: Option[String]): Option[Description] = + v.flatMap { + _.trim match { + case "" => Option.empty[Description] + case o => Option(o) + } + } + extension (self: Description) def value: String = self + given Codec[Description] = Codec.of[String] + + opaque type CreationDate = Instant + object CreationDate: + def apply(v: Instant): CreationDate = v + extension (self: CreationDate) def value: Instant = self + given Codec[CreationDate] = Codec.of[Instant] + + enum Visibility derives Codec: + lazy val name: String = productPrefix + case Public, Private + + object Visibility: + def fromCaseInsensitive(v: String): Visibility = + valueOf(v.toLowerCase.capitalize) diff --git a/modules/commons/src/main/scala/io/renku/search/model/users.scala b/modules/commons/src/main/scala/io/renku/search/model/users.scala new file mode 100644 index 00000000..21b2d750 --- /dev/null +++ b/modules/commons/src/main/scala/io/renku/search/model/users.scala @@ -0,0 +1,29 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.model + +import io.bullet.borer.Codec + +object users: + + opaque type Id = String + object Id: + def apply(v: String): Id = v + extension (self: Id) def value: String = self + given Codec[Id] = Codec.bimap[String, Id](_.value, Id.apply) diff --git a/modules/events/src/test/scala/io/renku/events/SerializeDeserializeTest.scala b/modules/events/src/test/scala/io/renku/events/SerializeDeserializeSpec.scala similarity index 96% rename from modules/events/src/test/scala/io/renku/events/SerializeDeserializeTest.scala rename to modules/events/src/test/scala/io/renku/events/SerializeDeserializeSpec.scala index 538b6609..7f35a8ac 100644 --- a/modules/events/src/test/scala/io/renku/events/SerializeDeserializeTest.scala +++ b/modules/events/src/test/scala/io/renku/events/SerializeDeserializeSpec.scala @@ -28,7 +28,7 @@ import java.time.Instant import java.time.temporal.ChronoUnit import java.util.UUID -class SerializeDeserializeTest extends FunSuite { +class SerializeDeserializeSpec extends FunSuite { test("serialize and deserialize ProjectCreated") { val data = ProjectCreated( diff --git a/modules/http4s-borer/src/main/scala/io/renku/search/http/borer/BorerEntities.scala b/modules/http4s-borer/src/main/scala/io/renku/search/http/borer/BorerEntities.scala index 0e16fcce..8ead5019 100644 --- a/modules/http4s-borer/src/main/scala/io/renku/search/http/borer/BorerEntities.scala +++ b/modules/http4s-borer/src/main/scala/io/renku/search/http/borer/BorerEntities.scala @@ -22,12 +22,12 @@ import cats.data.EitherT import cats.effect.* import cats.syntax.all.* import fs2.Chunk - import io.bullet.borer.* import org.http4s.* import org.http4s.headers.* object BorerEntities: + def decodeEntityJson[F[_]: Async, A: Decoder]: EntityDecoder[F, A] = EntityDecoder.decodeBy(MediaType.application.json)(decodeJson) diff --git a/modules/search-api/src/main/scala/io/renku/search/api/HttpApplication.scala b/modules/search-api/src/main/scala/io/renku/search/api/HttpApplication.scala index e41477c1..e165431f 100644 --- a/modules/search-api/src/main/scala/io/renku/search/api/HttpApplication.scala +++ b/modules/search-api/src/main/scala/io/renku/search/api/HttpApplication.scala @@ -28,6 +28,7 @@ import org.http4s.dsl.Http4sDsl import org.http4s.server.Router import org.http4s.{HttpApp, HttpRoutes, Response} import sttp.tapir.* +import sttp.tapir.docs.openapi.OpenAPIDocsOptions import sttp.tapir.server.ServerEndpoint import sttp.tapir.server.http4s.Http4sServerInterpreter import sttp.tapir.swagger.SwaggerUIOptions diff --git a/modules/search-api/src/main/scala/io/renku/search/api/Project.scala b/modules/search-api/src/main/scala/io/renku/search/api/Project.scala index 72510a14..a0a16f74 100644 --- a/modules/search-api/src/main/scala/io/renku/search/api/Project.scala +++ b/modules/search-api/src/main/scala/io/renku/search/api/Project.scala @@ -20,12 +20,42 @@ package io.renku.search.api import io.bullet.borer.derivation.MapBasedCodecs.{deriveDecoder, deriveEncoder} import io.bullet.borer.{Decoder, Encoder} +import io.renku.search.model.* import sttp.tapir.Schema -import sttp.tapir.generic.auto.schemaForCaseClass +import sttp.tapir.Schema.SName +import sttp.tapir.SchemaType.SDateTime -final case class Project(id: String, name: String, description: String) +final case class Project( + id: projects.Id, + name: projects.Name, + slug: projects.Slug, + repositories: Seq[projects.Repository], + visibility: projects.Visibility, + description: Option[projects.Description] = None, + createdBy: User, + creationDate: projects.CreationDate, + members: Seq[User] +) + +final case class User( + id: users.Id +) object Project: + given Encoder[User] = deriveEncoder + given Decoder[User] = deriveDecoder given Encoder[Project] = deriveEncoder given Decoder[Project] = deriveDecoder - given Schema[Project] = Schema.derivedSchema[Project] + private given Schema[projects.Id] = Schema.string[projects.Id] + private given Schema[projects.Name] = Schema.string[projects.Name] + private given Schema[projects.Slug] = Schema.string[projects.Slug] + private given Schema[projects.Repository] = Schema.string[projects.Repository] + private given Schema[projects.Visibility] = + Schema.derivedEnumeration[projects.Visibility].defaultStringBased + private given Schema[projects.Description] = Schema.string[projects.Description] + private given Schema[projects.CreationDate] = Schema(SDateTime()) + given Schema[User] = { + given Schema[users.Id] = Schema.string[users.Id] + Schema.derived[User] + } + given Schema[Project] = Schema.derived[Project] diff --git a/modules/search-api/src/main/scala/io/renku/search/api/SearchApiImpl.scala b/modules/search-api/src/main/scala/io/renku/search/api/SearchApiImpl.scala index 7e7b4449..4b4bf845 100644 --- a/modules/search-api/src/main/scala/io/renku/search/api/SearchApiImpl.scala +++ b/modules/search-api/src/main/scala/io/renku/search/api/SearchApiImpl.scala @@ -21,7 +21,7 @@ package io.renku.search.api import cats.effect.Async import cats.syntax.all.* import io.renku.search.solr.client.SearchSolrClient -import io.renku.search.solr.documents.Project as SolrProject +import io.renku.search.solr.documents.{Project as SolrProject, User as SolrUser} import org.http4s.dsl.Http4sDsl import scribe.Scribe @@ -49,4 +49,17 @@ private class SearchApiImpl[F[_]: Async](solrClient: SearchSolrClient[F]) .map(_.asLeft[List[Project]]) private def toApiModel(entities: List[SolrProject]): List[Project] = - entities.map(p => Project(p.id, p.name, p.description)) + entities.map { p => + def toUser(user: SolrUser): User = User(user.id) + Project( + p.id, + p.name, + p.slug, + p.repositories, + p.visibility, + p.description, + toUser(p.createdBy), + p.creationDate, + p.members.map(toUser) + ) + } diff --git a/modules/search-api/src/test/scala/io/renku/search/api/SearchApiSpec.scala b/modules/search-api/src/test/scala/io/renku/search/api/SearchApiSpec.scala index 13c3aefd..b0226c43 100644 --- a/modules/search-api/src/test/scala/io/renku/search/api/SearchApiSpec.scala +++ b/modules/search-api/src/test/scala/io/renku/search/api/SearchApiSpec.scala @@ -21,7 +21,7 @@ package io.renku.search.api import cats.effect.IO import io.renku.search.solr.client.SearchSolrClientGenerators.* import io.renku.search.solr.client.SearchSolrSpec -import io.renku.search.solr.documents.Project as SolrProject +import io.renku.search.solr.documents.{Project as SolrProject, User as SolrUser} import munit.CatsEffectSuite import scribe.Scribe @@ -42,5 +42,16 @@ class SearchApiSpec extends CatsEffectSuite with SearchSolrSpec: } yield assert(results contains toApiProject(project1)) } - private def toApiProject(project: SolrProject) = - Project(project.id, project.name, project.description) + private def toApiProject(p: SolrProject) = + def toUser(user: SolrUser): User = User(user.id) + Project( + p.id, + p.name, + p.slug, + p.repositories, + p.visibility, + p.description, + toUser(p.createdBy), + p.creationDate, + p.members.map(toUser) + ) diff --git a/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisioner.scala b/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisioner.scala index 7924b45e..2904f1ee 100644 --- a/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisioner.scala +++ b/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisioner.scala @@ -28,8 +28,9 @@ import io.renku.avro.codec.decoders.all.given import io.renku.events.v1.ProjectCreated import io.renku.queue.client.* import io.renku.redis.client.RedisUrl +import io.renku.search.model.* import io.renku.search.solr.client.SearchSolrClient -import io.renku.search.solr.documents.Project +import io.renku.search.solr.documents.* import io.renku.solr.client.SolrConfig import scribe.Scribe @@ -102,9 +103,22 @@ private class SearchProvisionerImpl[F[_]: Async]( } private lazy val toSolrDocuments: Seq[ProjectCreated] => Seq[Project] = - _.map(pc => - Project(id = pc.id, name = pc.name, description = pc.description.getOrElse("")) - ) + _.map { pc => + + def toUser(id: String): User = User(users.Id(id)) + + Project( + projects.Id(pc.id), + projects.Name(pc.name), + projects.Slug(pc.slug), + pc.repositories.map(projects.Repository(_)), + projects.Visibility.fromCaseInsensitive(pc.visibility.name()), + pc.description.map(projects.Description(_)), + toUser(pc.createdBy), + projects.CreationDate(pc.creationDate), + pc.members.map(toUser) + ) + } private def markProcessedOnFailure( message: Message diff --git a/modules/search-provision/src/test/scala/io/renku/search/provision/Generators.scala b/modules/search-provision/src/test/scala/io/renku/search/provision/Generators.scala new file mode 100644 index 00000000..18038463 --- /dev/null +++ b/modules/search-provision/src/test/scala/io/renku/search/provision/Generators.scala @@ -0,0 +1,55 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.provision + +import io.renku.events.v1.{ProjectCreated, Visibility} +import org.scalacheck.Gen +import org.scalacheck.Gen.alphaNumChar + +import java.time.Instant +import java.time.temporal.ChronoUnit + +object Generators: + + def projectCreatedGen(prefix: String): Gen[ProjectCreated] = + for + id <- Gen.uuid.map(_.toString) + name <- stringGen(max = 5).map(v => s"$prefix-$v") + repositories <- Gen.listOfN(Gen.choose(1, 3).generateOne, stringGen(10)) + visibility <- Gen.oneOf(Visibility.values().toList) + maybeDesc <- Gen.option(stringGen(20)) + creator <- Gen.uuid.map(_.toString) + yield ProjectCreated( + id, + name, + name, + repositories, + visibility, + maybeDesc, + creator, + Instant.now().truncatedTo(ChronoUnit.MILLIS), + Seq(creator) + ) + + def stringGen(max: Int): Gen[String] = + Gen + .chooseNum(3, max) + .flatMap(Gen.stringOfN(_, alphaNumChar)) + + extension [V](gen: Gen[V]) def generateOne: V = gen.sample.getOrElse(generateOne) diff --git a/modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala b/modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala index a459c93d..9856ce36 100644 --- a/modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala +++ b/modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala @@ -18,7 +18,7 @@ package io.renku.search.provision -import cats.effect.{Clock, IO, Resource} +import cats.effect.{IO, Resource} import cats.syntax.all.* import fs2.Stream import fs2.concurrent.SignallingRef @@ -29,14 +29,12 @@ import io.renku.queue.client.Encoding import io.renku.redis.client.RedisClientGenerators import io.renku.redis.client.RedisClientGenerators.* import io.renku.redis.client.util.RedisSpec +import io.renku.search.model.{projects, users} +import io.renku.search.provision.Generators.projectCreatedGen import io.renku.search.solr.client.SearchSolrSpec -import io.renku.search.solr.documents.Project +import io.renku.search.solr.documents.{Project, User} import munit.CatsEffectSuite -import org.scalacheck.Gen -import org.scalacheck.Gen.alphaNumChar -import java.time.temporal.ChronoUnit -import java.time.temporal.ChronoUnit.MILLIS import scala.concurrent.duration.* class SearchProvisionerSpec extends CatsEffectSuite with RedisSpec with SearchSolrSpec: @@ -55,7 +53,7 @@ class SearchProvisionerSpec extends CatsEffectSuite with RedisSpec with SearchSo provisioningFiber <- provisioner.provisionSolr.start - message1 <- generateProjectCreated(prefix = "binary") + message1 = projectCreatedGen(prefix = "binary").generateOne _ <- queueClient.enqueue( queue, avro.write[ProjectCreated](Seq(message1)), @@ -91,7 +89,7 @@ class SearchProvisionerSpec extends CatsEffectSuite with RedisSpec with SearchSo provisioningFiber <- provisioner.provisionSolr.start - message1 <- generateProjectCreated(prefix = "json") + message1 = projectCreatedGen(prefix = "json").generateOne _ <- queueClient.enqueue( queue, avro.writeJson[ProjectCreated](Seq(message1)), @@ -119,32 +117,19 @@ class SearchProvisionerSpec extends CatsEffectSuite with RedisSpec with SearchSo private def redisAndSolrClients = withRedisClient.asQueueClient() >>= withSearchSolrClient().tupleLeft - private def generateProjectCreated(prefix: String): IO[ProjectCreated] = - def generateString(max: Int): Gen[String] = - Gen - .chooseNum(3, max) - .flatMap(Gen.stringOfN(_, alphaNumChar)) - - for - now <- Clock[IO].realTimeInstant.map(_.truncatedTo(MILLIS)) - uuid <- IO.randomUUID - name = s"$prefix-${generateString(max = 5).sample.get}" - desc = s"$prefix ${generateString(max = 10).sample.get}" - ownerGen = generateString(max = 5).map(prefix + _) - yield ProjectCreated( - uuid.toString, - name, - "slug", - Seq.empty, - Visibility.PUBLIC, - Some(desc), - ownerGen.sample.get, - now, - Seq.empty - ) - private def toSolrDocument(created: ProjectCreated): Project = - Project(created.id, created.name, created.description.getOrElse("")) + def toUser(id: String): User = User(users.Id(id)) + Project( + projects.Id(created.id), + projects.Name(created.name), + projects.Slug(created.slug), + created.repositories.map(projects.Repository(_)), + projects.Visibility.fromCaseInsensitive(created.visibility.name()), + created.description.map(projects.Description(_)), + toUser(created.createdBy), + projects.CreationDate(created.creationDate), + created.members.map(toUser) + ) override def munitFixtures: Seq[Fixture[_]] = List(withRedisClient, withSearchSolrClient) diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala index 6f88bea6..a1b80aab 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala @@ -22,7 +22,7 @@ import cats.effect.Async import cats.syntax.all.* import io.renku.search.solr.documents.Project import io.renku.search.solr.schema.EntityDocumentSchema -import io.renku.solr.client.{QueryString, SolrClient} +import io.renku.solr.client.{QueryData, QueryString, SolrClient} private class SearchSolrClientImpl[F[_]: Async](solrClient: SolrClient[F]) extends SearchSolrClient[F]: @@ -33,8 +33,10 @@ private class SearchSolrClientImpl[F[_]: Async](solrClient: SolrClient[F]) override def findProjects(phrase: String): F[List[Project]] = solrClient .query[Project]( - QueryString( - s"${EntityDocumentSchema.Fields.entityType}:${Project.entityType} AND (name:$phrase OR description:$phrase)" + QueryData.withChildren( + QueryString( + s"${EntityDocumentSchema.Fields.entityType}:${Project.entityType} AND (name:$phrase OR description:$phrase)" + ) ) ) .map(_.responseBody.docs.toList) diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/documents/Project.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/documents/Project.scala index 26981c35..9f145a5d 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/documents/Project.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/documents/Project.scala @@ -18,14 +18,31 @@ package io.renku.search.solr.documents +import io.bullet.borer.NullOptions.given import io.bullet.borer.derivation.MapBasedCodecs.deriveDecoder import io.bullet.borer.{Decoder, Encoder} +import io.renku.search.model.* import io.renku.solr.client.EncoderSupport.deriveWithDiscriminator -final case class Project(id: String, name: String, description: String) +final case class Project( + id: projects.Id, + name: projects.Name, + slug: projects.Slug, + repositories: Seq[projects.Repository], + visibility: projects.Visibility, + description: Option[projects.Description] = None, + createdBy: User, + creationDate: projects.CreationDate, + members: Seq[User] +) object Project: val entityType: String = "Project" given Encoder[Project] = deriveWithDiscriminator + given Decoder[Seq[User]] = + Decoder[Seq[User]] { reader => + if reader.hasArrayStart then Decoder.forArray[User].map(_.toSeq).read(reader) + else Decoder[User].map(Seq(_)).read(reader) + } given Decoder[Project] = deriveDecoder diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/documents/User.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/documents/User.scala new file mode 100644 index 00000000..75d8ddec --- /dev/null +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/documents/User.scala @@ -0,0 +1,32 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.solr.documents + +import io.bullet.borer.derivation.MapBasedCodecs.deriveDecoder +import io.bullet.borer.{Decoder, Encoder} +import io.renku.search.model.users +import io.renku.solr.client.EncoderSupport.deriveWithDiscriminator + +final case class User(id: users.Id) + +object User: + val entityType: String = "User" + + given Encoder[User] = deriveWithDiscriminator + given Decoder[User] = deriveDecoder diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/schema/EntityDocumentSchema.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/schema/EntityDocumentSchema.scala index 0c6e1aff..be39fa20 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/schema/EntityDocumentSchema.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/schema/EntityDocumentSchema.scala @@ -26,13 +26,30 @@ object EntityDocumentSchema: val id: FieldName = FieldName("id") val entityType: FieldName = FieldName("_type") val name: FieldName = FieldName("name") + val slug: FieldName = FieldName("slug") + val repositories: FieldName = FieldName("repositories") + val visibility: FieldName = FieldName("visibility") val description: FieldName = FieldName("description") + val createdBy: FieldName = FieldName("createdBy") + val creationDate: FieldName = FieldName("creationDate") + val members: FieldName = FieldName("members") + + object FieldTypes: + val string: FieldType = FieldType.str(TypeName("SearchString")).makeDocValue + val text: FieldType = FieldType.text(TypeName("SearchText"), Analyzer.classic) + val dateTime: FieldType = FieldType.dateTimePoint(TypeName("SearchDateTime")) val initialEntityDocumentAdd: Seq[SchemaCommand] = Seq( - SchemaCommand.Add(FieldType.str(TypeName("entityType"))), - SchemaCommand.Add(FieldType.str(TypeName("name"))), - SchemaCommand.Add(FieldType.text(TypeName("description"), Analyzer.classic)), - SchemaCommand.Add(Field(Fields.entityType, TypeName("entityType"))), - SchemaCommand.Add(Field(Fields.name, TypeName("name"))), - SchemaCommand.Add(Field(Fields.description, TypeName("description"))) + SchemaCommand.Add(FieldTypes.string), + SchemaCommand.Add(FieldTypes.text), + SchemaCommand.Add(FieldTypes.dateTime), + SchemaCommand.Add(Field(Fields.entityType, FieldTypes.string)), + SchemaCommand.Add(Field(Fields.name, FieldTypes.string)), + SchemaCommand.Add(Field(Fields.slug, FieldTypes.string)), + SchemaCommand.Add(Field(Fields.repositories, FieldTypes.string).makeMultiValued), + SchemaCommand.Add(Field(Fields.visibility, FieldTypes.string)), + SchemaCommand.Add(Field(Fields.description, FieldTypes.text)), + SchemaCommand.Add(Field(Fields.createdBy, FieldType.nestedPath)), + SchemaCommand.Add(Field(Fields.creationDate, FieldTypes.dateTime)), + SchemaCommand.Add(Field(Fields.members, FieldType.nestedPath).makeMultiValued) ) diff --git a/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientGenerators.scala b/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientGenerators.scala index d1f98c05..04e25582 100644 --- a/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientGenerators.scala +++ b/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientGenerators.scala @@ -18,12 +18,47 @@ package io.renku.search.solr.client -import io.renku.search.solr.documents.Project +import io.renku.search.model.* +import io.renku.search.solr.documents.* import org.scalacheck.Gen +import java.time.Instant +import java.time.temporal.ChronoUnit + object SearchSolrClientGenerators: + private def projectIdGen: Gen[projects.Id] = + Gen.uuid.map(uuid => projects.Id(uuid.toString)) + def projectDocumentGen(name: String, desc: String): Gen[Project] = - Gen.uuid.map(uuid => Project(uuid.toString, name, desc)) + projectIdGen.map(projectId => + val creator = userDocumentGen.generateOne + Project( + projectId, + projects.Name(name), + projects.Slug(name), + Seq(projects.Repository(s"http://github.com/$name")), + Gen.oneOf(projects.Visibility.values.toList).generateOne, + Option(projects.Description(desc)), + creator, + instantGen().generateAs(projects.CreationDate.apply), + Seq(creator) + ) + ) + + def userDocumentGen: Gen[User] = + userIdGen.map(id => User(id)) + + private def userIdGen: Gen[users.Id] = Gen.uuid.map(uuid => users.Id(uuid.toString)) + + private def instantGen( + min: Instant = Instant.EPOCH, + max: Instant = Instant.now() + ): Gen[Instant] = + Gen + .chooseNum(min.toEpochMilli, max.toEpochMilli) + .map(Instant.ofEpochMilli(_).truncatedTo(ChronoUnit.MILLIS)) - extension [V](gen: Gen[V]) def generateOne: V = gen.sample.getOrElse(generateOne) + extension [V](gen: Gen[V]) + def generateOne: V = gen.sample.getOrElse(generateOne) + def generateAs[D](f: V => D): D = f(generateOne) diff --git a/modules/solr-client/src/main/scala/io/renku/solr/client/EncoderSupport.scala b/modules/solr-client/src/main/scala/io/renku/solr/client/EncoderSupport.scala index 832a599d..5aaf6b90 100644 --- a/modules/solr-client/src/main/scala/io/renku/solr/client/EncoderSupport.scala +++ b/modules/solr-client/src/main/scala/io/renku/solr/client/EncoderSupport.scala @@ -18,9 +18,11 @@ package io.renku.solr.client +import io.bullet.borer.NullOptions.* import io.bullet.borer.{Encoder, Writer} -import scala.deriving.* + import scala.compiletime.* +import scala.deriving.* object EncoderSupport { @@ -58,5 +60,4 @@ object EncoderSupport { case _: EmptyTuple => Nil case _: (t *: ts) => constValue[t].toString :: summonLabels[ts] } - } diff --git a/modules/solr-client/src/main/scala/io/renku/solr/client/QueryData.scala b/modules/solr-client/src/main/scala/io/renku/solr/client/QueryData.scala index 73cb1ae7..aee7010a 100644 --- a/modules/solr-client/src/main/scala/io/renku/solr/client/QueryData.scala +++ b/modules/solr-client/src/main/scala/io/renku/solr/client/QueryData.scala @@ -45,4 +45,11 @@ final case class QueryData( ) object QueryData: + + def apply(query: QueryString): QueryData = + QueryData(query.q, Nil, query.limit, query.offset, Nil, Map.empty) + + def withChildren(query: QueryString): QueryData = + QueryData(query.q, Nil, query.limit, query.offset, Nil, Map("fl" -> "*,[child]")) + given Encoder[QueryData] = deriveEncoder diff --git a/modules/solr-client/src/main/scala/io/renku/solr/client/SolrClient.scala b/modules/solr-client/src/main/scala/io/renku/solr/client/SolrClient.scala index 33a7faea..5c43ad15 100644 --- a/modules/solr-client/src/main/scala/io/renku/solr/client/SolrClient.scala +++ b/modules/solr-client/src/main/scala/io/renku/solr/client/SolrClient.scala @@ -34,6 +34,8 @@ trait SolrClient[F[_]]: def query[A: Decoder](q: QueryString): F[QueryResponse[A]] + def query[A: Decoder](q: QueryData): F[QueryResponse[A]] + def delete(q: QueryString): F[Unit] def insert[A: Encoder](docs: Seq[A]): F[InsertResponse] diff --git a/modules/solr-client/src/main/scala/io/renku/solr/client/SolrClientImpl.scala b/modules/solr-client/src/main/scala/io/renku/solr/client/SolrClientImpl.scala index d0bac305..0e435394 100644 --- a/modules/solr-client/src/main/scala/io/renku/solr/client/SolrClientImpl.scala +++ b/modules/solr-client/src/main/scala/io/renku/solr/client/SolrClientImpl.scala @@ -43,10 +43,10 @@ private class SolrClientImpl[F[_]: Async](config: SolrConfig, underlying: Client underlying.expectOr[String](req)(onErrorLog(logger, req)).void def query[A: Decoder](q: QueryString): F[QueryResponse[A]] = - val req = Method.POST( - io.renku.solr.client.QueryData(q.q, Nil, q.limit, q.offset, Nil, Map.empty), - solrUrl / "query" - ) + query[A](QueryData(q)) + + def query[A: Decoder](query: QueryData): F[QueryResponse[A]] = + val req = Method.POST(query, solrUrl / "query") underlying .expectOr[QueryResponse[A]](req)(ResponseLogging.Error(logger, req)) .flatTap(r => logger.trace(s"Query response: $r")) diff --git a/modules/solr-client/src/main/scala/io/renku/solr/client/schema/Field.scala b/modules/solr-client/src/main/scala/io/renku/solr/client/schema/Field.scala index 105b166e..92098ffb 100644 --- a/modules/solr-client/src/main/scala/io/renku/solr/client/schema/Field.scala +++ b/modules/solr-client/src/main/scala/io/renku/solr/client/schema/Field.scala @@ -31,9 +31,11 @@ final case class Field( multiValued: Boolean, uninvertible: Boolean, docValues: Boolean -) +): + def makeMultiValued: Field = copy(multiValued = true) object Field: + def apply(name: FieldName, typeName: TypeName): Field = Field( name = name, @@ -46,4 +48,7 @@ object Field: docValues = false ) + def apply(name: FieldName, fieldType: FieldType): Field = + apply(name, fieldType.name) + given Encoder[Field] = deriveEncoder diff --git a/modules/solr-client/src/main/scala/io/renku/solr/client/schema/FieldType.scala b/modules/solr-client/src/main/scala/io/renku/solr/client/schema/FieldType.scala index 3ea13eec..82fa0ea9 100644 --- a/modules/solr-client/src/main/scala/io/renku/solr/client/schema/FieldType.scala +++ b/modules/solr-client/src/main/scala/io/renku/solr/client/schema/FieldType.scala @@ -29,7 +29,8 @@ final case class FieldType( uninvertible: Boolean = false, docValues: Boolean = false, sortMissingLast: Boolean = true -) +): + lazy val makeDocValue: FieldType = copy(docValues = true) object FieldType: @@ -48,5 +49,11 @@ object FieldType: def double(name: TypeName): FieldType = FieldType(name, FieldTypeClass.Defaults.doublePointField) - def dateTime(name: TypeName): FieldType = + def dateTimeRange(name: TypeName): FieldType = FieldType(name, FieldTypeClass.Defaults.dateRangeField) + + def dateTimePoint(name: TypeName): FieldType = + FieldType(name, FieldTypeClass.Defaults.datePointField) + + lazy val nestedPath: FieldType = + FieldType(TypeName("_nest_path_"), FieldTypeClass.Defaults.nestedPath) diff --git a/modules/solr-client/src/main/scala/io/renku/solr/client/schema/FieldTypeClass.scala b/modules/solr-client/src/main/scala/io/renku/solr/client/schema/FieldTypeClass.scala index 62b12dac..5598f483 100644 --- a/modules/solr-client/src/main/scala/io/renku/solr/client/schema/FieldTypeClass.scala +++ b/modules/solr-client/src/main/scala/io/renku/solr/client/schema/FieldTypeClass.scala @@ -36,8 +36,10 @@ object FieldTypeClass: val strField: FieldTypeClass = "StrField" val uuidField: FieldTypeClass = "UUIDField" val rankField: FieldTypeClass = "RankField" + val datePointField: FieldTypeClass = "DatePointField" val dateRangeField: FieldTypeClass = "DateRangeField" val boolField: FieldTypeClass = "BoolField" val binaryField: FieldTypeClass = "BinaryField" + val nestedPath: FieldTypeClass = "solr.NestPathField" given Encoder[FieldTypeClass] = Encoder.forString diff --git a/modules/solr-client/src/test/scala/io/renku/solr/client/SolrClientSpec.scala b/modules/solr-client/src/test/scala/io/renku/solr/client/SolrClientSpec.scala index dee7bcac..9a3c1be4 100644 --- a/modules/solr-client/src/test/scala/io/renku/solr/client/SolrClientSpec.scala +++ b/modules/solr-client/src/test/scala/io/renku/solr/client/SolrClientSpec.scala @@ -19,8 +19,8 @@ package io.renku.solr.client import cats.effect.IO -import io.bullet.borer.{Decoder, Encoder} import io.bullet.borer.derivation.MapBasedCodecs.deriveDecoder +import io.bullet.borer.{Decoder, Encoder} import io.renku.solr.client.SolrClientSpec.Room import io.renku.solr.client.schema.* import io.renku.solr.client.util.{SolrSpec, SolrTruncate} @@ -41,7 +41,7 @@ class SolrClientSpec extends CatsEffectSuite with SolrSpec with SolrTruncate: _ <- client.modifySchema(cmds) _ <- client .insert[Room](Seq(Room("meeting room", "room for meetings", 56))) - r <- client.query[Room](QueryString("_type:Room")) + r <- client.query[Room](QueryData(QueryString("_type:Room"))) _ <- IO.println(r) } yield () }