diff --git a/modules/http4s-borer/src/main/scala/io/renku/search/http/borer/BorerEntities.scala b/modules/http4s-borer/src/main/scala/io/renku/search/http/borer/BorerEntities.scala index 8ead5019..b9e62f76 100644 --- a/modules/http4s-borer/src/main/scala/io/renku/search/http/borer/BorerEntities.scala +++ b/modules/http4s-borer/src/main/scala/io/renku/search/http/borer/BorerEntities.scala @@ -38,7 +38,8 @@ object BorerEntities: EitherT(StreamProvider(media.body).flatMap { implicit input => for { res <- Async[F].delay(Json.decode(input).to[A].valueEither) - } yield res.left.map(BorerDecodeFailure("", _)) + txt <- if (res.isLeft) media.bodyText.compile.string else Async[F].pure("") + } yield res.left.map(BorerDecodeFailure(txt, _)) }) def decodeCbor[F[_]: Async, A: Decoder](media: Media[F]): DecodeResult[F, A] = diff --git a/modules/search-api/src/main/scala/io/renku/search/api/SearchApiImpl.scala b/modules/search-api/src/main/scala/io/renku/search/api/SearchApiImpl.scala index b95890e2..bfcab975 100644 --- a/modules/search-api/src/main/scala/io/renku/search/api/SearchApiImpl.scala +++ b/modules/search-api/src/main/scala/io/renku/search/api/SearchApiImpl.scala @@ -25,9 +25,12 @@ import io.renku.search.api.data.* import io.renku.search.model.users import io.renku.search.solr.client.SearchSolrClient import io.renku.search.solr.documents.Entity as SolrEntity +import io.renku.search.solr.schema.EntityDocumentSchema.Fields import io.renku.solr.client.QueryResponse import org.http4s.dsl.Http4sDsl import scribe.Scribe +import io.renku.search.model.EntityType +import io.renku.solr.client.facet.FacetResponse private class SearchApiImpl[F[_]: Async](solrClient: SearchSolrClient[F]) extends Http4sDsl[F] @@ -58,9 +61,19 @@ private class SearchApiImpl[F[_]: Async](solrClient: SearchSolrClient[F]) ): SearchResult = val hasMore = solrResult.responseBody.docs.size > currentPage.limit val pageInfo = PageWithTotals(currentPage, solrResult.responseBody.numFound, hasMore) + val facets = solrResult.facetResponse + .flatMap(_.buckets.get(Fields.entityType)) + .map { counts => + val all = + counts.buckets.flatMap { case FacetResponse.Bucket(key, count) => + EntityType.fromString(key).toOption.map(et => et -> count) + }.toMap + FacetData(all) + } + .getOrElse(FacetData.empty) val items = solrResult.responseBody.docs.map(toApiEntity) - if (hasMore) SearchResult(items.init, pageInfo) - else SearchResult(items, pageInfo) + if (hasMore) SearchResult(items.init, facets, pageInfo) + else SearchResult(items, facets, pageInfo) private lazy val toApiEntity: SolrEntity => SearchEntity = given Transformer[users.Id, UserId] = (id: users.Id) => UserId(id) diff --git a/modules/search-api/src/main/scala/io/renku/search/api/data/FacetData.scala b/modules/search-api/src/main/scala/io/renku/search/api/data/FacetData.scala new file mode 100644 index 00000000..4abb8785 --- /dev/null +++ b/modules/search-api/src/main/scala/io/renku/search/api/data/FacetData.scala @@ -0,0 +1,49 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.api.data + +import io.renku.search.model.EntityType +import io.bullet.borer.Decoder +import io.bullet.borer.derivation.MapBasedCodecs +import sttp.tapir.Schema +import io.bullet.borer.Encoder +import io.renku.search.api.tapir.SchemaSyntax.* + +final case class FacetData( + entityType: Map[EntityType, Int] +) + +object FacetData: + val empty: FacetData = FacetData(Map.empty) + + given Decoder[FacetData] = MapBasedCodecs.deriveDecoder + given Encoder[FacetData] = MapBasedCodecs.deriveEncoder + given Schema[FacetData] = { + given Schema[Map[EntityType, Int]] = Schema.schemaForMap(_.name) + Schema + .derived[FacetData] + .jsonExample( + FacetData( + Map( + EntityType.Project -> 15, + EntityType.User -> 3 + ) + ) + ) + } diff --git a/modules/search-api/src/main/scala/io/renku/search/api/data/SearchEntity.scala b/modules/search-api/src/main/scala/io/renku/search/api/data/SearchEntity.scala index a77786b7..7d9067a4 100644 --- a/modules/search-api/src/main/scala/io/renku/search/api/data/SearchEntity.scala +++ b/modules/search-api/src/main/scala/io/renku/search/api/data/SearchEntity.scala @@ -22,6 +22,7 @@ import io.bullet.borer.* import io.bullet.borer.NullOptions.given import io.bullet.borer.derivation.MapBasedCodecs.{deriveAllCodecs, deriveCodec} import io.renku.search.model.* +import io.renku.search.api.tapir.SchemaSyntax.* import sttp.tapir.Schema.SName import sttp.tapir.SchemaType.{SCoproduct, SDateTime, SProductField, SRef} import sttp.tapir.generic.Configuration @@ -54,20 +55,18 @@ object Project: private given Schema[projects.CreationDate] = Schema(SDateTime()) given Schema[Project] = Schema .derived[Project] - .encodedExample( - Json.encode { - Project( - projects.Id("01HRA7AZ2Q234CDQWGA052F8MK"), - projects.Name("renku"), - projects.Slug("renku"), - Seq(projects.Repository("https://github.com/renku")), - projects.Visibility.Public, - Some(projects.Description("Renku project")), - UserId(users.Id("1CAF4C73F50D4514A041C9EDDB025A36")), - projects.CreationDate(Instant.now), - Some(1.0) - ).asInstanceOf[SearchEntity] - }.toUtf8String + .jsonExample( + Project( + projects.Id("01HRA7AZ2Q234CDQWGA052F8MK"), + projects.Name("renku"), + projects.Slug("renku"), + Seq(projects.Repository("https://github.com/renku")), + projects.Visibility.Public, + Some(projects.Description("Renku project")), + UserId(users.Id("1CAF4C73F50D4514A041C9EDDB025A36")), + projects.CreationDate(Instant.now), + Some(1.0) + ): SearchEntity ) final case class UserId(id: users.Id) @@ -77,9 +76,7 @@ object UserId: private given Schema[users.Id] = Schema.string[users.Id] given Schema[UserId] = Schema .derived[UserId] - .encodedExample( - Json.encode(UserId(users.Id("01HRA7AZ2Q234CDQWGA052F8MK"))).toUtf8String - ) + .jsonExample(UserId(users.Id("01HRA7AZ2Q234CDQWGA052F8MK"))) final case class User( id: users.Id, @@ -96,16 +93,14 @@ object User: private given Schema[users.Email] = Schema.string[users.Email] given Schema[User] = Schema .derived[User] - .encodedExample( - Json.encode { - User( - users.Id("1CAF4C73F50D4514A041C9EDDB025A36"), - Some(users.FirstName("Albert")), - Some(users.LastName("Einstein")), - Some(users.Email("albert.einstein@mail.com")), - Some(2.1) - ).asInstanceOf[SearchEntity] - }.toUtf8String + .jsonExample( + User( + users.Id("1CAF4C73F50D4514A041C9EDDB025A36"), + Some(users.FirstName("Albert")), + Some(users.LastName("Einstein")), + Some(users.Email("albert.einstein@mail.com")), + Some(2.1) + ): SearchEntity ) object SearchEntity: diff --git a/modules/search-api/src/main/scala/io/renku/search/api/data/SearchResult.scala b/modules/search-api/src/main/scala/io/renku/search/api/data/SearchResult.scala index 06b71ae0..065687cc 100644 --- a/modules/search-api/src/main/scala/io/renku/search/api/data/SearchResult.scala +++ b/modules/search-api/src/main/scala/io/renku/search/api/data/SearchResult.scala @@ -25,6 +25,7 @@ import sttp.tapir.Schema final case class SearchResult( items: Seq[SearchEntity], + facets: FacetData, pagingInfo: PageWithTotals ) diff --git a/modules/search-api/src/main/scala/io/renku/search/api/tapir/SchemaSyntax.scala b/modules/search-api/src/main/scala/io/renku/search/api/tapir/SchemaSyntax.scala new file mode 100644 index 00000000..c665385b --- /dev/null +++ b/modules/search-api/src/main/scala/io/renku/search/api/tapir/SchemaSyntax.scala @@ -0,0 +1,31 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.api.tapir + +import io.bullet.borer.Encoder +import io.bullet.borer.Json +import sttp.tapir.Schema + +trait SchemaSyntax: + + extension [T](self: Schema[T]) + def jsonExample[TT >: T](value: TT)(using Encoder[TT]): Schema[T] = + self.encodedExample(Json.encode(value).toUtf8String) + +object SchemaSyntax extends SchemaSyntax diff --git a/modules/search-api/src/main/scala/io/renku/search/api/tapir/TapirCodecs.scala b/modules/search-api/src/main/scala/io/renku/search/api/tapir/TapirCodecs.scala index ff058ce8..f4693064 100644 --- a/modules/search-api/src/main/scala/io/renku/search/api/tapir/TapirCodecs.scala +++ b/modules/search-api/src/main/scala/io/renku/search/api/tapir/TapirCodecs.scala @@ -21,6 +21,7 @@ package io.renku.search.api.tapir import sttp.tapir.* import io.renku.search.api.data.* import io.renku.search.query.Query +import io.renku.search.model.EntityType trait TapirCodecs: given Codec[String, Query, CodecFormat.TextPlain] = @@ -28,3 +29,10 @@ trait TapirCodecs: given Schema[Query] = Schema.anyObject[Query] given Schema[QueryInput] = Schema.derived + + given Codec[String, EntityType, CodecFormat.TextPlain] = + Codec.string.mapEither(EntityType.fromString(_))(_.name) + + given Schema[EntityType] = Schema.derivedEnumeration.defaultStringBased + +object TapirCodecs extends TapirCodecs diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala index 04e7fd24..c9731bd2 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala @@ -25,6 +25,9 @@ import io.renku.search.query.Query import io.renku.search.solr.documents.Entity import io.renku.search.solr.query.LuceneQueryInterpreter import io.renku.solr.client.{QueryData, QueryResponse, QueryString, SolrClient} +import io.renku.solr.client.schema.FieldName +import io.renku.solr.client.facet.{Facet, Facets} +import io.renku.search.solr.schema.EntityDocumentSchema private class SearchSolrClientImpl[F[_]: Async](solrClient: SolrClient[F]) extends SearchSolrClient[F]: @@ -32,6 +35,11 @@ private class SearchSolrClientImpl[F[_]: Async](solrClient: SolrClient[F]) private[this] val logger = scribe.cats.effect[F] private[this] val interpreter = LuceneQueryInterpreter.forSync[F] + private val typeTerms = Facet.Terms( + EntityDocumentSchema.Fields.entityType, + EntityDocumentSchema.Fields.entityType + ) + override def insert[D: Encoder](documents: Seq[D]): F[Unit] = solrClient.insert(documents).void @@ -47,6 +55,7 @@ private class SearchSolrClientImpl[F[_]: Async](solrClient: SolrClient[F]) .query[Entity]( QueryData(QueryString(solrQuery.query.value, limit, offset)) .withSort(solrQuery.sort) - .withScore + .withFacet(Facets(typeTerms)) + .withFields(FieldName.all, FieldName.score) ) } yield res diff --git a/modules/search-solr-client/src/test/scala/io/renku/search/solr/query/LuceneQueryInterpreterSpec.scala b/modules/search-solr-client/src/test/scala/io/renku/search/solr/query/LuceneQueryInterpreterSpec.scala index 779af249..b8bd7a6e 100644 --- a/modules/search-solr-client/src/test/scala/io/renku/search/solr/query/LuceneQueryInterpreterSpec.scala +++ b/modules/search-solr-client/src/test/scala/io/renku/search/solr/query/LuceneQueryInterpreterSpec.scala @@ -43,7 +43,9 @@ class LuceneQueryInterpreterSpec override protected lazy val coreName: String = server.testCoreName2 given Decoder[Unit] = new Decoder { - def read(r: Reader) = () + def read(r: Reader) = + r.skipElement() + () } def query(s: String | Query): QueryData = diff --git a/modules/solr-client/src/main/scala/io/renku/solr/client/QueryData.scala b/modules/solr-client/src/main/scala/io/renku/solr/client/QueryData.scala index f306833a..41cb354d 100644 --- a/modules/solr-client/src/main/scala/io/renku/solr/client/QueryData.scala +++ b/modules/solr-client/src/main/scala/io/renku/solr/client/QueryData.scala @@ -21,38 +21,38 @@ package io.renku.solr.client import io.bullet.borer.Encoder import io.bullet.borer.derivation.MapBasedCodecs.deriveEncoder import io.renku.solr.client.schema.FieldName +import io.renku.solr.client.facet.Facets final case class QueryData( query: String, filter: Seq[String], limit: Int, offset: Int, - fields: Seq[FieldName], - sort: SolrSort, - params: Map[String, String] + fields: Seq[FieldName] = Seq.empty, + sort: SolrSort = SolrSort.empty, + params: Map[String, String] = Map.empty, + facet: Facets = Facets.empty ): def nextPage: QueryData = copy(offset = offset + limit) - def withHighLight(fields: List[FieldName], pre: String, post: String): QueryData = - copy(params = - params ++ Map( - "hl" -> "on", - "hl.requireFieldMatch" -> "true", - "hl.fl" -> fields.map(_.name).mkString(","), - "hl.simple.pre" -> pre, - "hl.simple.post" -> post - ) - ) - def withSort(sort: SolrSort): QueryData = copy(sort = sort) - def withFieldList(fl: String): QueryData = copy(params = params.updated("fl", fl)) - def withScore: QueryData = withFieldList("* score") - def withScoreAndChildren: QueryData = withFieldList("* score,[child]") + def withFields(field: FieldName*) = copy(fields = field) + def addFilter(q: String): QueryData = copy(filter = filter :+ q) + def withFacet(facet: Facets): QueryData = copy(facet = facet) object QueryData: def apply(query: QueryString): QueryData = - QueryData(query.q, Nil, query.limit, query.offset, Nil, SolrSort.empty, Map.empty) + QueryData( + query.q, + Nil, + query.limit, + query.offset, + Nil, + SolrSort.empty, + Map.empty, + Facets.empty + ) given Encoder[QueryData] = deriveEncoder diff --git a/modules/solr-client/src/main/scala/io/renku/solr/client/QueryResponse.scala b/modules/solr-client/src/main/scala/io/renku/solr/client/QueryResponse.scala index d735b8d0..6ea476be 100644 --- a/modules/solr-client/src/main/scala/io/renku/solr/client/QueryResponse.scala +++ b/modules/solr-client/src/main/scala/io/renku/solr/client/QueryResponse.scala @@ -19,16 +19,19 @@ package io.renku.solr.client import io.bullet.borer.Decoder -import io.bullet.borer.derivation.MapBasedCodecs.deriveDecoder +import io.bullet.borer.NullOptions.given +import io.bullet.borer.derivation.MapBasedCodecs import io.bullet.borer.derivation.key +import io.renku.solr.client.facet.FacetResponse final case class QueryResponse[A]( responseHeader: ResponseHeader, - @key("response") responseBody: ResponseBody[A] + @key("response") responseBody: ResponseBody[A], + @key("facets") facetResponse: Option[FacetResponse] = None ): def map[B](f: A => B): QueryResponse[B] = copy(responseBody = responseBody.map(f)) object QueryResponse: given [A](using Decoder[A]): Decoder[QueryResponse[A]] = - deriveDecoder + MapBasedCodecs.deriveDecoder diff --git a/modules/solr-client/src/main/scala/io/renku/solr/client/facet/Facet.scala b/modules/solr-client/src/main/scala/io/renku/solr/client/facet/Facet.scala new file mode 100644 index 00000000..9a560576 --- /dev/null +++ b/modules/solr-client/src/main/scala/io/renku/solr/client/facet/Facet.scala @@ -0,0 +1,42 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.solr.client.facet + +import io.renku.solr.client.schema.FieldName +import cats.data.NonEmptyList + +enum Facet: + // https://solr.apache.org/guide/solr/latest/query-guide/json-facet-api.html#terms-facet + case Terms( + name: FieldName, + field: FieldName, + limit: Option[Int] = None, + minCount: Option[Int] = None, + method: Option[FacetAlgorithm] = None, + missing: Boolean = false, + numBuckets: Boolean = false, + allBuckets: Boolean = false + ) + + // https://solr.apache.org/guide/solr/latest/query-guide/json-facet-api.html#range-facet + case ArbitraryRange( + name: FieldName, + field: FieldName, + ranges: NonEmptyList[FacetRange] + ) diff --git a/modules/solr-client/src/main/scala/io/renku/solr/client/facet/FacetAlgorithm.scala b/modules/solr-client/src/main/scala/io/renku/solr/client/facet/FacetAlgorithm.scala new file mode 100644 index 00000000..91ac68aa --- /dev/null +++ b/modules/solr-client/src/main/scala/io/renku/solr/client/facet/FacetAlgorithm.scala @@ -0,0 +1,40 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.solr.client.facet + +import io.bullet.borer.Encoder + +enum FacetAlgorithm: + case DocValues + case UnInvertedField + case DocValuesHash + case Enum + case Stream + case Smart + + private[client] def name: String = this match + case DocValues => "dv" + case UnInvertedField => "uif" + case DocValuesHash => "dvhash" + case Enum => "enum" + case Stream => "stream" + case Smart => "smart" + +object FacetAlgorithm: + given Encoder[FacetAlgorithm] = Encoder.forString.contramap(_.name) diff --git a/modules/solr-client/src/main/scala/io/renku/solr/client/facet/FacetRange.scala b/modules/solr-client/src/main/scala/io/renku/solr/client/facet/FacetRange.scala new file mode 100644 index 00000000..df9d57ef --- /dev/null +++ b/modules/solr-client/src/main/scala/io/renku/solr/client/facet/FacetRange.scala @@ -0,0 +1,41 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.solr.client.facet + +import io.bullet.borer.Encoder +import io.bullet.borer.Writer +import io.bullet.borer.derivation.MapBasedCodecs +import io.bullet.borer.derivation.key + +final case class FacetRange( + from: FacetRange.Value, + to: FacetRange.Value, + @key("inclusive_from") inclusiveFrom: Boolean = true, + @key("inclusive_to") inclusiveTo: Boolean = false +) +object FacetRange: + case object All + type Value = Int | All.type + + given Encoder[Value] = new Encoder[Value]: + override def write(w: Writer, v: Value) = v match + case n: Int => w.write(n) + case All => w.write("*") + + given Encoder[FacetRange] = MapBasedCodecs.deriveEncoder diff --git a/modules/solr-client/src/main/scala/io/renku/solr/client/facet/FacetResponse.scala b/modules/solr-client/src/main/scala/io/renku/solr/client/facet/FacetResponse.scala new file mode 100644 index 00000000..01cf26aa --- /dev/null +++ b/modules/solr-client/src/main/scala/io/renku/solr/client/facet/FacetResponse.scala @@ -0,0 +1,59 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.solr.client.facet + +import io.renku.solr.client.schema.FieldName +import io.bullet.borer.derivation.key +import io.bullet.borer.Decoder +import io.bullet.borer.derivation.MapBasedCodecs +import io.bullet.borer.Reader + +import FacetResponse.Values + +final case class FacetResponse( + count: Int, + buckets: Map[FieldName, Values] +): + def isEmpty: Boolean = count == 0 && buckets.isEmpty + + private def withBuckets(field: FieldName, values: Values): FacetResponse = + copy(buckets = buckets.updated(field, values)) + +object FacetResponse: + val empty: FacetResponse = FacetResponse(0, Map.empty) + + final case class Bucket(@key("val") value: String, count: Int) + object Bucket: + given Decoder[Bucket] = MapBasedCodecs.deriveDecoder + + final case class Values(buckets: Buckets) + object Values: + given Decoder[Values] = MapBasedCodecs.deriveDecoder + + type Buckets = Seq[Bucket] + + given Decoder[FacetResponse] = new Decoder[FacetResponse] { + def read(r: Reader): FacetResponse = + r.readMapStart() + r.readUntilBreak(empty) { fr => + val nextKey = r.readString() + if (nextKey == "count") fr.copy(count = r.readInt()) + else fr.withBuckets(FieldName(nextKey), r.read[Values]()) + } + } diff --git a/modules/solr-client/src/main/scala/io/renku/solr/client/facet/Facets.scala b/modules/solr-client/src/main/scala/io/renku/solr/client/facet/Facets.scala new file mode 100644 index 00000000..b75daee1 --- /dev/null +++ b/modules/solr-client/src/main/scala/io/renku/solr/client/facet/Facets.scala @@ -0,0 +1,80 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.solr.client.facet + +import io.bullet.borer.Encoder +import io.bullet.borer.Writer +import cats.Monoid + +opaque type Facets = Seq[Facet] + +object Facets: + val empty: Facets = Seq.empty + def apply(f: Facet*): Facets = f + + extension (self: Facets) + def isEmpty: Boolean = self.isEmpty + def size: Int = self.size + + given Monoid[Facets] = Monoid.instance(empty, _ ++ _) + given Encoder[Facets] = JsonEncoder.encoder + + private object JsonEncoder { + given Encoder[Facet.ArbitraryRange] = + new Encoder[Facet.ArbitraryRange]: + def write(w: Writer, v: Facet.ArbitraryRange) = + w.write(v.name) + w.writeMapOpen(3) + w.writeMapMember("type", "range") + w.writeMapMember("field", v.field) + w.writeMapMember("ranges", v.ranges.toList) + w.writeMapClose() + + given Encoder[Facet.Terms] = + new Encoder[Facet.Terms] { + def write(w: Writer, t: Facet.Terms) = + w.write(t.name) + val size = t.productIterator.map { + case None => 0 + case _ => 1 + }.sum + w.writeMapOpen(size + 1) + // facet type + w.writeMapMember("type", "terms") + // configuration + w.writeMapMember("field", t.field) + t.limit.foreach(l => w.writeMapMember("limit", l)) + t.minCount.foreach(c => w.writeMapMember("mincount", c)) + t.method.foreach(a => w.writeMapMember("method", a.name)) + w.writeMapMember("missing", t.missing) + w.writeMapMember("numBuckets", t.numBuckets) + w.writeMapMember("allBuckets", t.allBuckets) + w.writeMapClose() + } + + def encoder: Encoder[Facets] = + new Encoder[Facets]: + def write(w: Writer, v: Facets) = + w.writeMapOpen(v.size) + v.foreach { + case f: Facet.Terms => Encoder[Facet.Terms].write(w, f) + case f: Facet.ArbitraryRange => Encoder[Facet.ArbitraryRange].write(w, f) + } + w.writeMapClose() + } diff --git a/modules/solr-client/src/main/scala/io/renku/solr/client/schema/FieldName.scala b/modules/solr-client/src/main/scala/io/renku/solr/client/schema/FieldName.scala index 6cb8f02a..26ffc4ee 100644 --- a/modules/solr-client/src/main/scala/io/renku/solr/client/schema/FieldName.scala +++ b/modules/solr-client/src/main/scala/io/renku/solr/client/schema/FieldName.scala @@ -22,6 +22,9 @@ import io.bullet.borer.Encoder opaque type FieldName = String object FieldName: + val all: FieldName = "*" + val score: FieldName = "score" + def apply(name: String): FieldName = name extension (self: FieldName) def name: String = self diff --git a/modules/solr-client/src/test/scala/io/renku/solr/client/SolrClientGenerator.scala b/modules/solr-client/src/test/scala/io/renku/solr/client/SolrClientGenerator.scala index 6cce503a..e69c34c9 100644 --- a/modules/solr-client/src/test/scala/io/renku/solr/client/SolrClientGenerator.scala +++ b/modules/solr-client/src/test/scala/io/renku/solr/client/SolrClientGenerator.scala @@ -19,7 +19,56 @@ package io.renku.solr.client import org.scalacheck.Gen +import io.renku.solr.client.facet.* +import io.renku.solr.client.schema.FieldName +import io.renku.search.model.CommonGenerators object SolrClientGenerator: extension [V](gen: Gen[V]) def generateOne: V = gen.sample.getOrElse(generateOne) + + private val fieldNameString: Gen[String] = + Gen.choose(4, 12).flatMap(n => Gen.listOfN(n, Gen.alphaLowerChar)).map(_.mkString) + + val fieldNameTypeStr: Gen[FieldName] = + fieldNameString.map(n => s"${n}_s").map(FieldName.apply) + + val fieldNameTypeInt: Gen[FieldName] = + fieldNameString.map(n => s"${n}_i").map(FieldName.apply) + + val fieldNameLiteral: Gen[FieldName] = + fieldNameString.map(FieldName.apply) + + val facetTerms: Gen[Facet.Terms] = + for { + name <- fieldNameLiteral + field <- Gen.oneOf(fieldNameTypeStr, fieldNameTypeInt) + limit <- Gen.choose(1, 10) + } yield Facet.Terms(name, field, Some(limit)) + + val facetRangeValue: Gen[FacetRange.Value] = + Gen.oneOf(Gen.const(FacetRange.All), Gen.choose(0, 1000)) + + val facetRange: Gen[FacetRange] = + for { + from <- facetRangeValue + to <- from match + case FacetRange.All => Gen.choose(0, 500) + case n: Int => Gen.oneOf(Gen.const(FacetRange.All), Gen.choose(0, 500).map(_ + n)) + } yield FacetRange(from, to) + + val facetArbitraryRange: Gen[Facet.ArbitraryRange] = + for { + name <- fieldNameLiteral + field <- fieldNameTypeInt + numRanges <- Gen.choose(1, 5) + ranges <- CommonGenerators.nelOfN(numRanges, facetRange) + } yield Facet.ArbitraryRange(name, field, ranges) + + val facet: Gen[Facet] = Gen.oneOf(facetTerms, facetArbitraryRange) + + val facets: Gen[Facets] = + Gen.choose(0, 5).flatMap(n => Gen.listOfN(n, facet)).map { n => + if (n.isEmpty) Facets.empty + else Facets(n: _*) + } diff --git a/modules/solr-client/src/test/scala/io/renku/solr/client/SolrClientSpec.scala b/modules/solr-client/src/test/scala/io/renku/solr/client/SolrClientSpec.scala index 8913a1dc..c2923395 100644 --- a/modules/solr-client/src/test/scala/io/renku/solr/client/SolrClientSpec.scala +++ b/modules/solr-client/src/test/scala/io/renku/solr/client/SolrClientSpec.scala @@ -20,14 +20,24 @@ package io.renku.solr.client import cats.effect.IO import cats.syntax.all.* -import io.bullet.borer.derivation.MapBasedCodecs.deriveDecoder +import io.bullet.borer.derivation.MapBasedCodecs import io.bullet.borer.{Decoder, Encoder} import io.renku.solr.client.SolrClientSpec.Room import io.renku.solr.client.schema.* import io.renku.solr.client.util.{SolrSpec, SolrTruncate} import munit.CatsEffectSuite +import munit.ScalaCheckEffectSuite +import org.scalacheck.effect.PropF +import io.bullet.borer.Reader +import org.scalacheck.Gen +import io.renku.solr.client.facet.{Facet, Facets} +import io.bullet.borer.derivation.key -class SolrClientSpec extends CatsEffectSuite with SolrSpec with SolrTruncate: +class SolrClientSpec + extends CatsEffectSuite + with ScalaCheckEffectSuite + with SolrSpec + with SolrTruncate: test("use schema for inserting and querying"): val cmds = Seq( @@ -38,17 +48,82 @@ class SolrClientSpec extends CatsEffectSuite with SolrSpec with SolrTruncate: SchemaCommand.Add(Field(FieldName("roomSeats"), TypeName("roomInt"))) ) withSolrClient().use { client => + val rooms = Seq(Room("meeting room", "room for meetings", 56)) for { + _ <- truncateAll(client)( + List("roomName", "roomDescription", "roomSeats").map(FieldName.apply), + List("roomText", "roomInt").map(TypeName.apply) + ) _ <- client.modifySchema(cmds) _ <- client - .insert[Room](Seq(Room("meeting room", "room for meetings", 56))) + .insert[Room](rooms) r <- client.query[Room](QueryData(QueryString("_type:Room"))) - _ <- IO.println(r) + _ = assertEquals(r.responseBody.docs, rooms) } yield () } + test("correct facet queries"): + val decoder: Decoder[Unit] = new Decoder { + def read(r: Reader): Unit = + r.skipElement() + () + } + PropF.forAllF(SolrClientGenerator.facets) { facets => + val q = QueryData(QueryString("*:*")).withFacet(facets) + withSolrClient().use { client => + client.query(q)(using decoder).void + } + } + + test("decoding facet response"): + val rooms = Gen.listOfN(15, Room.gen).sample.get + val facets = + Facets(Facet.Terms(FieldName("by_name"), FieldName("roomName"), limit = Some(6))) + withSolrClient().use { client => + for { + _ <- client.delete(QueryString("*:*")) + _ <- client.insert(rooms) + r <- client.query[Room](QueryData(QueryString("*:*")).withFacet(facets)) + _ = assert(r.facetResponse.nonEmpty) + _ = assertEquals(r.facetResponse.get.count, 15) + _ = assertEquals( + r.facetResponse.get.buckets(FieldName("by_name")).buckets.size, + 6 + ) + } yield () + } + + // test("delete by id"): + // withSolrClient().use { client => + // for { + // _ <- client.delete(QueryString("*:*")) + // _ <- client.insert(Seq(SolrClientSpec.Person("p1", "John"))) + // r <- client.query[SolrClientSpec.Person](QueryData(QueryString("*:*"))) + // p = r.responseBody.docs.head + // _ = assertEquals(p.id, "p1") + // _ <- client.deleteById("p1", "p2") + // r2 <- client.query[SolrClientSpec.Person](QueryData(QueryString("*:*"))) + // _ <- IO.sleep(50.millis) // seems to be necessary on ci + // _ = assert(r2.responseBody.docs.isEmpty) + // } yield () + // } + object SolrClientSpec: case class Room(roomName: String, roomDescription: String, roomSeats: Int) object Room: - given Decoder[Room] = deriveDecoder + val gen: Gen[Room] = for { + name <- Gen + .choose(4, 12) + .flatMap(n => Gen.listOfN(n, Gen.alphaChar)) + .map(_.mkString) + descr = s"Room description for $name" + seats <- Gen.choose(15, 350) + } yield Room(name, descr, seats) + + given Decoder[Room] = MapBasedCodecs.deriveDecoder given Encoder[Room] = EncoderSupport.deriveWithDiscriminator[Room] + + case class Person(id: String, @key("name_s") name: String) + object Person: + given Decoder[Person] = MapBasedCodecs.deriveDecoder + given Encoder[Person] = MapBasedCodecs.deriveEncoder diff --git a/modules/solr-client/src/test/scala/io/renku/solr/client/facet/FacetsSpec.scala b/modules/solr-client/src/test/scala/io/renku/solr/client/facet/FacetsSpec.scala new file mode 100644 index 00000000..ffa2b12b --- /dev/null +++ b/modules/solr-client/src/test/scala/io/renku/solr/client/facet/FacetsSpec.scala @@ -0,0 +1,51 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.solr.client.facet + +import munit.FunSuite +import io.bullet.borer.Json +import io.renku.solr.client.schema.FieldName +import cats.data.NonEmptyList + +class FacetsSpec extends FunSuite: + + test("encode multiple facets into object"): + val terms1 = Facet.Terms(FieldName("type"), FieldName("_type")) + val terms2 = Facet.Terms(FieldName("cat"), FieldName("category")) + val json = Json.encode(Facets(terms1, terms2)).toUtf8String + assertEquals( + json, + """{"type":{"type":"terms","field":"_type","missing":false,"numBuckets":false,"allBuckets":false},"cat":{"type":"terms","field":"category","missing":false,"numBuckets":false,"allBuckets":false}}""" + ) + + test("encode arbitrary range"): + val range = Facet.ArbitraryRange( + FieldName("stars"), + FieldName("stars"), + NonEmptyList.of( + FacetRange(FacetRange.All, 100), + FacetRange(100, 200), + FacetRange(200, FacetRange.All) + ) + ) + val json = Json.encode(Facets(range)).toUtf8String + assertEquals( + json, + """{"stars":{"type":"range","field":"stars","ranges":[{"from":"*","to":100},{"from":100,"to":200},{"from":200,"to":"*"}]}}""" + ) diff --git a/modules/solr-client/src/test/scala/io/renku/solr/client/migration/SolrMigratorSpec.scala b/modules/solr-client/src/test/scala/io/renku/solr/client/migration/SolrMigratorSpec.scala index bf96c3fb..bef962d9 100644 --- a/modules/solr-client/src/test/scala/io/renku/solr/client/migration/SolrMigratorSpec.scala +++ b/modules/solr-client/src/test/scala/io/renku/solr/client/migration/SolrMigratorSpec.scala @@ -27,6 +27,7 @@ import munit.CatsEffectSuite class SolrMigratorSpec extends CatsEffectSuite with SolrSpec with SolrTruncate: private val logger = scribe.cats.io + override protected lazy val coreName: String = server.testCoreName3 private val migrations = Seq( SchemaMigration(-5, Add(FieldType.text(TypeName("testText"), Analyzer.classic))), SchemaMigration(-4, Add(FieldType.int(TypeName("testInt")))), diff --git a/nix/dev-scripts.nix b/nix/dev-scripts.nix index 51433c50..9fedaec4 100644 --- a/nix/dev-scripts.nix +++ b/nix/dev-scripts.nix @@ -77,9 +77,11 @@ solr-recreate-dbtests-cores = writeShellScriptBin "solr-recreate-dbtests-cores" '' solr-delete-core core-test1 solr-delete-core core-test2 + solr-delete-core core-test3 solr-delete-core search-core-test solr-create-core core-test1 solr-create-core core-test2 + solr-create-core core-test3 solr-create-core search-core-test ''; diff --git a/nix/openapi-doc.nix b/nix/openapi-doc.nix index e26b7bfe..c0312830 100644 --- a/nix/openapi-doc.nix +++ b/nix/openapi-doc.nix @@ -20,7 +20,7 @@