From 3db0bc71618cea9f635eeedc69e9de867ef3f837 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Fri, 23 Feb 2024 15:17:12 +0100 Subject: [PATCH 01/17] WIP: Convert renku query to solr query --- build.sbt | 3 +- .../io/renku/search/model/EntityType.scala | 41 ++++++ .../io/renku/search/api/HttpApplication.scala | 4 +- modules/search-query-docs/docs/manual.md | 17 +++ .../io/renku/search/query/DateTimeRef.scala | 26 ++++ .../io/renku/search/query/EntityType.scala | 0 .../scala/io/renku/search/query/Field.scala | 1 + .../io/renku/search/query/FieldTerm.scala | 5 + .../search/query/json/QueryJsonCodec.scala | 8 + .../search/query/parse/QueryParser.scala | 18 ++- .../search/solr/client/QueryInterpreter.scala | 65 --------- .../solr/client/SearchSolrClientImpl.scala | 10 +- .../io/renku/search/solr/query/Context.scala | 33 +++++ .../search/solr/query/QueryInterpreter.scala | 34 +++++ .../solr/query/QueryTokenEncoders.scala | 138 ++++++++++++++++++ .../renku/search/solr/query/SolrToken.scala | 103 +++++++++++++ .../search/solr/query/SolrTokenEncoder.scala | 103 +++++++++++++ project/Dependencies.scala | 5 - 18 files changed, 536 insertions(+), 78 deletions(-) create mode 100644 modules/commons/src/main/scala/io/renku/search/model/EntityType.scala create mode 100644 modules/search-query/src/main/scala/io/renku/search/query/EntityType.scala delete mode 100644 modules/search-solr-client/src/main/scala/io/renku/search/solr/client/QueryInterpreter.scala create mode 100644 modules/search-solr-client/src/main/scala/io/renku/search/solr/query/Context.scala create mode 100644 modules/search-solr-client/src/main/scala/io/renku/search/solr/query/QueryInterpreter.scala create mode 100644 modules/search-solr-client/src/main/scala/io/renku/search/solr/query/QueryTokenEncoders.scala create mode 100644 modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrToken.scala create mode 100644 modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrTokenEncoder.scala diff --git a/build.sbt b/build.sbt index 36bbc356..9dbcd069 100644 --- a/build.sbt +++ b/build.sbt @@ -197,8 +197,7 @@ lazy val searchSolrClient = project name := "search-solr-client", libraryDependencies ++= Dependencies.catsCore ++ - Dependencies.catsEffect ++ - Dependencies.luceneQueryParser + Dependencies.catsEffect ) .dependsOn( avroCodec % "compile->compile;test->test", diff --git a/modules/commons/src/main/scala/io/renku/search/model/EntityType.scala b/modules/commons/src/main/scala/io/renku/search/model/EntityType.scala new file mode 100644 index 00000000..42fe5e91 --- /dev/null +++ b/modules/commons/src/main/scala/io/renku/search/model/EntityType.scala @@ -0,0 +1,41 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.model + +import io.bullet.borer.Encoder +import io.bullet.borer.Decoder + +enum EntityType: + case Project + case User + + def name: String = productPrefix.toLowerCase + +object EntityType: + def fromString(str: String): Either[String, EntityType] = + EntityType.values + .find(_.name.equalsIgnoreCase(str)) + .toRight(s"Invalid entity type: $str") + + def unsafeFromString(str: String): EntityType = + fromString(str).fold(sys.error, identity) + + + given Encoder[EntityType] = Encoder.forString.contramap(_.name) + given Decoder[EntityType] = Decoder.forString.mapEither(fromString) diff --git a/modules/search-api/src/main/scala/io/renku/search/api/HttpApplication.scala b/modules/search-api/src/main/scala/io/renku/search/api/HttpApplication.scala index 877e97e1..b7474d87 100644 --- a/modules/search-api/src/main/scala/io/renku/search/api/HttpApplication.scala +++ b/modules/search-api/src/main/scala/io/renku/search/api/HttpApplication.scala @@ -37,8 +37,8 @@ final class HttpApplication[F[_]: Async](searchApi: SearchApi[F]) extends Http4s private val prefix = "/search" - private val search = new SearchRoutes[F](searchApi) - private val openapi = new OpenApiRoute[F](prefix, "Renku Search API", search.endpoints) + private val search = SearchRoutes[F](searchApi) + private val openapi = OpenApiRoute[F](prefix, "Renku Search API", search.endpoints) lazy val router: HttpApp[F] = Router[F]( diff --git a/modules/search-query-docs/docs/manual.md b/modules/search-query-docs/docs/manual.md index 5606a5e4..e71c3cf6 100644 --- a/modules/search-query-docs/docs/manual.md +++ b/modules/search-query-docs/docs/manual.md @@ -73,6 +73,7 @@ Multiple alternative values can be given as a comma separated list. The following fields are available: ```scala mdoc:passthrough +import io.renku.search.model.EntityType import io.renku.search.query.* println(Field.values.map(e => s"`${e.name}`").mkString("- ", "\n- ", "")) ``` @@ -81,6 +82,22 @@ Each field allows to specify one or more values, separated by comma. The value must be separated by a `:`. For date fields, additional `<` and `>` is supported. +### EntityTypes + +The field `type` allows to search for specific entity types. If it is +missing, all entity types are included in the result. Entity types are: + +```scala mdoc:passthrough +println( + EntityType.values.map(e => s"`${e.name}`").mkString("- ", "\n- ", "") +) +``` + +Example: +```scala mdoc:passthrough +println(s" `${Field.Type.name}:${EntityType.Project.name}`") +``` + ### Dates Date fields, like diff --git a/modules/search-query/src/main/scala/io/renku/search/query/DateTimeRef.scala b/modules/search-query/src/main/scala/io/renku/search/query/DateTimeRef.scala index 3b9d9387..a3927dc4 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/DateTimeRef.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/DateTimeRef.scala @@ -21,6 +21,8 @@ package io.renku.search.query import cats.syntax.all.* import io.bullet.borer.{Decoder, Encoder} import io.renku.search.query.parse.DateTimeParser +import java.time.Instant +import java.time.ZoneId enum DateTimeRef: case Literal(ref: PartialDateTime) @@ -32,6 +34,30 @@ enum DateTimeRef: case Relative(ref) => ref.name case Calc(ref) => ref.asString + /** Resolves the date-time reference to a concrete instant using the given reference + * date. It either returns a single instant or a time range. + */ + def resolve(ref: Instant, zoneId: ZoneId): (Instant, Option[Instant]) = this match + case Relative(RelativeDate.Today) => (ref, None) + case Relative(RelativeDate.Yesterday) => (ref.atZone(zoneId).minusDays(1).toInstant, None) + case Literal(pdate) => + val min = pdate.instantMin(zoneId) + val max = pdate.instantMax(zoneId) + (min, Some(max).filter(_ != min)) + case Calc(cdate) => + val ts = cdate.ref match + case pd: PartialDateTime => + pd.instantMin(zoneId).atZone(zoneId) + + case rd: RelativeDate => + Relative(rd).resolve(ref, zoneId)._1.atZone(zoneId) + + if (cdate.range) (ts.minus(cdate.amount).toInstant, Some(ts.plus(cdate.amount).toInstant)) + else (ts.plus(cdate.amount).toInstant, None) + + + + object DateTimeRef: given Encoder[DateTimeRef] = Encoder.forString.contramap(_.asString) given Decoder[DateTimeRef] = Decoder.forString.mapEither { str => diff --git a/modules/search-query/src/main/scala/io/renku/search/query/EntityType.scala b/modules/search-query/src/main/scala/io/renku/search/query/EntityType.scala new file mode 100644 index 00000000..e69de29b diff --git a/modules/search-query/src/main/scala/io/renku/search/query/Field.scala b/modules/search-query/src/main/scala/io/renku/search/query/Field.scala index 9679d3ae..9b44a5de 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/Field.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/Field.scala @@ -27,6 +27,7 @@ enum Field: case Visibility case Created case CreatedBy + case Type val name: String = Strings.lowerFirst(productPrefix) diff --git a/modules/search-query/src/main/scala/io/renku/search/query/FieldTerm.scala b/modules/search-query/src/main/scala/io/renku/search/query/FieldTerm.scala index aee0cc5f..771d0d52 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/FieldTerm.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/FieldTerm.scala @@ -19,9 +19,11 @@ package io.renku.search.query import cats.data.NonEmptyList +import io.renku.search.model.EntityType import io.renku.search.model.projects.Visibility enum FieldTerm(val field: Field, val cmp: Comparison): + case TypeIs(values: NonEmptyList[EntityType]) extends FieldTerm(Field.Type, Comparison.Is) case ProjectIdIs(values: NonEmptyList[String]) extends FieldTerm(Field.ProjectId, Comparison.Is) case NameIs(values: NonEmptyList[String]) extends FieldTerm(Field.Name, Comparison.Is) @@ -35,6 +37,9 @@ enum FieldTerm(val field: Field, val cmp: Comparison): private[query] def asString = val value = this match + case TypeIs(values) => + val ts = values.toList.distinct.map(_.name) + ts.mkString(",") case ProjectIdIs(values) => FieldTerm.nelToString(values) case NameIs(values) => FieldTerm.nelToString(values) case SlugIs(values) => FieldTerm.nelToString(values) diff --git a/modules/search-query/src/main/scala/io/renku/search/query/json/QueryJsonCodec.scala b/modules/search-query/src/main/scala/io/renku/search/query/json/QueryJsonCodec.scala index 21239fb2..ddce9ffa 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/json/QueryJsonCodec.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/json/QueryJsonCodec.scala @@ -21,6 +21,7 @@ package io.renku.search.query.json import cats.data.NonEmptyList import io.bullet.borer.compat.cats.* import io.bullet.borer.{Decoder, Encoder, Reader, Writer} +import io.renku.search.model.EntityType import io.renku.search.model.projects.Visibility import io.renku.search.query.* import io.renku.search.query.FieldTerm.* @@ -62,6 +63,9 @@ private[query] object QueryJsonCodec: private def writeFieldTermValue(w: Writer, term: FieldTerm): Writer = term match + case FieldTerm.TypeIs(values) => + writeNelValue(w, values) + case FieldTerm.ProjectIdIs(values) => writeNelValue(w, values) @@ -103,6 +107,10 @@ private[query] object QueryJsonCodec: case Name.TextName => Segment.Text(r.readString()) + case Name.FieldName(Field.Type) => + val values = readNel[EntityType](r) + Segment.Field(TypeIs(values)) + case Name.FieldName(Field.ProjectId) => val values = readNel[String](r) Segment.Field(ProjectIdIs(values)) diff --git a/modules/search-query/src/main/scala/io/renku/search/query/parse/QueryParser.scala b/modules/search-query/src/main/scala/io/renku/search/query/parse/QueryParser.scala index d7f54a42..56e5a0b1 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/parse/QueryParser.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/parse/QueryParser.scala @@ -20,6 +20,7 @@ package io.renku.search.query.parse import cats.data.NonEmptyList import cats.parse.{Parser as P, Parser0 as P0} +import io.renku.search.model.EntityType import io.renku.search.model.projects.Visibility import io.renku.search.query.* @@ -64,6 +65,16 @@ private[query] object QueryParser { val visibilities: P[NonEmptyList[Visibility]] = nelOf(visibility, commaSep) + val entityType: P[EntityType] = + P.stringIn( + EntityType.values + .map(_.name.toLowerCase) + .toSet ++ EntityType.values.map(_.name).toSet + ).map(EntityType.unsafeFromString) + + val entityTypes: P[NonEmptyList[EntityType]] = + nelOf(entityType, commaSep) + val termIs: P[FieldTerm] = { val field = fieldNameFrom(Field.values.toSet - Field.Created - Field.Visibility) ((field <* is) ~ values).map { case (f, v) => @@ -74,9 +85,14 @@ private[query] object QueryParser { case Field.CreatedBy => FieldTerm.CreatedByIs(v) case Field.Visibility => sys.error("visibility not allowed") case Field.Created => sys.error("created not allowed") + case Field.Type => sys.error("type not allowed") } } + val typeIs: P[FieldTerm] = + val field = fieldNameFrom(Set(Field.Type)) + ((field ~ is).void *> entityTypes).map(v => FieldTerm.TypeIs(v)) + val visibilityIs: P[FieldTerm] = { val field = fieldNameFrom(Set(Field.Visibility)) ((field ~ is).void *> visibilities).map(v => FieldTerm.VisibilityIs(v)) @@ -90,7 +106,7 @@ private[query] object QueryParser { } } - val fieldTerm: P[FieldTerm] = termIs | visibilityIs | created + val fieldTerm: P[FieldTerm] = termIs | visibilityIs | typeIs | created val freeText: P[String] = P.charsWhile(c => !c.isWhitespace) diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/QueryInterpreter.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/QueryInterpreter.scala deleted file mode 100644 index f2561956..00000000 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/QueryInterpreter.scala +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2024 Swiss Data Science Center (SDSC) - * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and - * Eidgenössische Technische Hochschule Zürich (ETHZ). - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.renku.search.solr.client - -import io.renku.search.query.Query -import io.renku.search.query.Query.Segment -import io.renku.search.query.FieldTerm -import io.renku.search.solr.schema.EntityDocumentSchema.Fields -import org.apache.lucene.queryparser.flexible.standard.QueryParserUtil - -private object QueryInterpreter { - - def apply(query: Query): String = - if (query.isEmpty) "_type:Project" // User not yet supported to decode - else - query.segments - .map { - case Segment.Field(FieldTerm.ProjectIdIs(ids)) => - ids.toList - .map(escape) - .map(id => s"${Fields.id.name}:$id") - .mkString("(", " OR ", ")") - - case Segment.Field(FieldTerm.SlugIs(slugs)) => - slugs.toList - .map(escape) - .map(slug => s"${Fields.slug.name}:$slug") - .mkString("(", " OR ", ")") - - case Segment.Field(FieldTerm.NameIs(names)) => - names.toList - .map(escape) - .map(name => s"${Fields.name.name}:$name") - .mkString("(", " OR ", ")") - - case Segment.Text(txt) => - s"${Fields.contentAll.name}:${escape(txt)}" - - case _ => - "" - } - .mkString(" AND ") - - private def escape(s: String): String = - val escaped = QueryParserUtil.escape(s) - if (escaped.exists(_.isWhitespace)) s"($escaped)" - else escaped - -} diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala index 5056354e..18cc6ab2 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala @@ -21,6 +21,7 @@ package io.renku.search.solr.client import cats.effect.Async import cats.syntax.all.* import io.renku.search.solr.documents.Project +import io.renku.search.solr.query.QueryInterpreter import io.renku.search.solr.schema.EntityDocumentSchema import io.renku.solr.client.{QueryData, QueryString, SolrClient} import io.renku.search.query.Query @@ -30,6 +31,7 @@ private class SearchSolrClientImpl[F[_]: Async](solrClient: SolrClient[F]) extends SearchSolrClient[F]: private[this] val logger = scribe.cats.effect[F] + private[this] val interpreter = QueryInterpreter[F] override def insertProjects(projects: Seq[Project]): F[Unit] = solrClient.insert(projects).void @@ -39,10 +41,12 @@ private class SearchSolrClientImpl[F[_]: Async](solrClient: SolrClient[F]) limit: Int, offset: Int ): F[QueryResponse[Project]] = - val solrQuery = QueryInterpreter(query) - logger.debug(s"Query: ${query.render} ->Solr: $solrQuery") >> - solrClient + for { + solrQuery <- interpreter.solrQuery(query) + _ <- logger.debug(s"Query: ${query.render} ->Solr: $solrQuery") + res <- solrClient .query[Project](QueryData.withChildren(QueryString(solrQuery, limit, offset))) + } yield res override def findProjects(phrase: String): F[List[Project]] = solrClient diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/Context.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/Context.scala new file mode 100644 index 00000000..7cd7fcfb --- /dev/null +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/Context.scala @@ -0,0 +1,33 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.solr.query + +import java.time.Instant +import cats.effect.{Clock, Sync} +import java.time.ZoneId + +trait Context[F[_]]: + def currentTime: F[Instant] + def zoneId: F[ZoneId] + +object Context: + def forSync[F[_]: Sync]: Context[F] = + new Context[F]: + def currentTime: F[Instant] = Clock[F].realTimeInstant + def zoneId: F[ZoneId] = Sync[F].delay(ZoneId.systemDefault()) diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/QueryInterpreter.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/QueryInterpreter.scala new file mode 100644 index 00000000..4d68a8e2 --- /dev/null +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/QueryInterpreter.scala @@ -0,0 +1,34 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.solr.query + +import cats.effect.Sync +import cats.syntax.all.* +import io.renku.search.query.Query + +final class QueryInterpreter[F[_]: Sync](ctx: Context[F]) extends QueryTokenEncoders: + private val encoder = SolrTokenEncoder[F, Query] + + def solrQuery(query: Query): F[String] = + if (query.isEmpty) SolrToken.allTypes.value.pure[F] + else encoder.encode(ctx, query).map(t => List(SolrToken.allTypes, t).foldAnd.value) + +object QueryInterpreter: + def apply[F[_]: Sync]: QueryInterpreter[F] = + new QueryInterpreter[F](Context.forSync[F]) diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/QueryTokenEncoders.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/QueryTokenEncoders.scala new file mode 100644 index 00000000..66ed11ab --- /dev/null +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/QueryTokenEncoders.scala @@ -0,0 +1,138 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.solr.query + +import cats.syntax.all.* +import io.renku.search.query.Query.Segment +import io.renku.search.query.FieldTerm +import io.renku.search.query.Field +import io.renku.search.query.Query + +import cats.Monad +import cats.Applicative +import cats.effect.kernel.Sync +import io.renku.search.query.Comparison + +trait QueryTokenEncoders: + + given projectIdIs[F[_]: Applicative]: SolrTokenEncoder[F, FieldTerm.ProjectIdIs] = + SolrTokenEncoder.basic { case FieldTerm.ProjectIdIs(ids) => + SolrToken.orFieldIs(Field.ProjectId, ids) + } + + given nameIs[F[_]: Applicative]: SolrTokenEncoder[F, FieldTerm.NameIs] = + SolrTokenEncoder.basic { case FieldTerm.NameIs(names) => + SolrToken.orFieldIs(Field.Name, names) + } + + given typeIs[F[_]: Applicative]: SolrTokenEncoder[F, FieldTerm.TypeIs] = + SolrTokenEncoder.basic { case FieldTerm.TypeIs(values) => + SolrToken.orFieldIs(Field.Type, values.map(_.name)) + } + + given slugIs[F[_]: Applicative]: SolrTokenEncoder[F, FieldTerm.SlugIs] = + SolrTokenEncoder.basic { case FieldTerm.SlugIs(names) => + SolrToken.orFieldIs(Field.Slug, names) + } + + given createdByIs[F[_]: Applicative]: SolrTokenEncoder[F, FieldTerm.CreatedByIs] = + SolrTokenEncoder.basic { case FieldTerm.CreatedByIs(names) => + SolrToken.orFieldIs(Field.CreatedBy, names) + } + + given visibilityIs[F[_]: Applicative]: SolrTokenEncoder[F, FieldTerm.VisibilityIs] = + SolrTokenEncoder.basic { case FieldTerm.VisibilityIs(values) => + SolrToken.orFieldIs(Field.Visibility, values.map(_.name)) + } + + given created[F[_]: Monad]: SolrTokenEncoder[F, FieldTerm.Created] = + SolrTokenEncoder.create[F, FieldTerm.Created] { + case (ctx, FieldTerm.Created(Comparison.Is, values)) => + (ctx.currentTime, ctx.zoneId).mapN { (ref, zone) => + values + .map(_.resolve(ref, zone)) + .map { case (min, maxOpt) => + maxOpt + .map(max => SolrToken.dateRange(Field.Created, min, max)) + .getOrElse(SolrToken.dateIs(Field.Created, min)) + } + .toList + .foldOr + } + + case (ctx, FieldTerm.Created(Comparison.GreaterThan, values)) => + (ctx.currentTime, ctx.zoneId).mapN { (ref, zone) => + values + .map(_.resolve(ref, zone)) + .map { case (min, maxOpt) => + SolrToken.dateGt(Field.Created, maxOpt.getOrElse(min)) + } + .toList + .foldOr + } + + case (ctx, FieldTerm.Created(Comparison.LowerThan, values)) => + (ctx.currentTime, ctx.zoneId).mapN { (ref, zone) => + values + .map(_.resolve(ref, zone)) + .map { case (min, _) => + SolrToken.dateLt(Field.Created, min) + } + .toList + .foldOr + } + } + + given fieldTerm[F[_]: Sync]: SolrTokenEncoder[F, FieldTerm] = + SolrTokenEncoder.derived[F, FieldTerm] + + given fieldSegment[F[_]: Applicative](using + fe: SolrTokenEncoder[F, FieldTerm] + ): SolrTokenEncoder[F, Segment.Field] = + SolrTokenEncoder.curried[F, Segment.Field] { ctx => + { case Segment.Field(f) => + fe.encode(ctx, f) + } + } + + given textSegment[F[_]: Applicative]: SolrTokenEncoder[F, Segment.Text] = + SolrTokenEncoder.basic(t => SolrToken.contentAll(t.value)) + + given segment[F[_]](using + et: SolrTokenEncoder[F, Segment.Text], + ef: SolrTokenEncoder[F, Segment.Field] + ): SolrTokenEncoder[F, Segment] = + SolrTokenEncoder.curried[F, Segment] { ctx => + { + case s: Segment.Text => et.encode(ctx, s) + case s: Segment.Field => ef.encode(ctx, s) + } + } + + given segmentAnd[F[_]: Monad, A](using + se: SolrTokenEncoder[F, A] + ): SolrTokenEncoder[F, List[A]] = + SolrTokenEncoder.create[F, List[A]] { (ctx, nel) => + nel.traverse(se.encode(ctx, _)).map(_.toSeq.foldAnd) + } + + given query[F[_]: Monad](using + se: SolrTokenEncoder[F, List[Segment]] + ): SolrTokenEncoder[F, Query] = + se.contramap(_.segments) diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrToken.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrToken.scala new file mode 100644 index 00000000..fb20f2d6 --- /dev/null +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrToken.scala @@ -0,0 +1,103 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.solr.query + +import cats.syntax.all.* +import io.renku.search.query.Field +import cats.data.NonEmptyList +import cats.Monoid +import java.time.Instant +import io.renku.search.solr.schema.EntityDocumentSchema.Fields as SolrField +import io.renku.solr.client.schema.FieldName + +opaque type SolrToken = String + +object SolrToken: + def apply(str: String): SolrToken = str + def escaped(str: String): SolrToken = Escape.queryChars(str) + + def contentAll(text: String): SolrToken = + s"${SolrField.contentAll.name}:${Escape.queryChars(text)}" + + def fieldIs(field: Field, value: SolrToken): SolrToken = + val name = solrField(field).name + s"${name}:${value.value}" + + def orFieldIs(field: Field, values: NonEmptyList[String]): SolrToken = + values.map(Escape.queryChars).map(fieldIs(field, _)).toList.foldOr + + def dateRange(field: Field, min: Instant, max: Instant): SolrToken = + s"${solrField(field).name}:[$min TO $max]" + + def dateIs(field: Field, date: Instant): SolrToken = fieldIs(field, date.toString) + def dateGt(field: Field, date: Instant): SolrToken = ??? + def dateLt(field: Field, date: Instant): SolrToken = ??? + + val allTypes: SolrToken = fieldIs(Field.Type, "*") + + private def solrField(field: Field): FieldName = + field match + case Field.ProjectId => SolrField.id + case Field.Name => SolrField.name + case Field.Slug => SolrField.slug + case Field.Visibility => SolrField.visibility + case Field.CreatedBy => SolrField.createdBy + case Field.Created => SolrField.creationDate + case Field.Type => SolrField.entityType + + private val orMonoid: Monoid[SolrToken] = + Monoid.instance("", (a, b) => if (a.isEmpty) b else if (b.isEmpty) a else s"$a OR $b") + private val andMonoid: Monoid[SolrToken] = Monoid.instance( + "", + (a, b) => if (a.isEmpty) b else if (b.isEmpty) a else s"$a AND $b" + ) + + extension (self: SolrToken) + def value: String = self + + extension (self: Seq[SolrToken]) + def foldOr: SolrToken = + given Monoid[SolrToken] = orMonoid + if (self.isEmpty) "" + else if (self.tail.isEmpty) self.head + else s"( ${self.combineAll} )" + + def foldAnd: SolrToken = + given Monoid[SolrToken] = andMonoid + if (self.isEmpty) "" + else if (self.tail.isEmpty) self.head + else s"( ${self.combineAll} )" + + // Escapes query characters for solr. This is taken from here: + // https://github.com/apache/solr/blob/bcb9f144974ed07aa3b66766302474542067b522/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java#L163 + // to not introduce too many dependencies only for this little function + private object Escape { + private[this] val specialChars = "\\+-!():^[]\"{}~*?|&;/" + + private inline def isSpecial(c: Char) = c.isWhitespace || specialChars.contains(c) + + def queryChars(s: String): String = { + val sb = new StringBuilder(); + s.foreach { c => + if (isSpecial(c)) sb.append('\\') + sb.append(c) + } + s.toString + } + } diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrTokenEncoder.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrTokenEncoder.scala new file mode 100644 index 00000000..e587c1b6 --- /dev/null +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrTokenEncoder.scala @@ -0,0 +1,103 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.solr.query + +import cats.Applicative +import cats.syntax.all.* +import cats.effect.kernel.Sync +import scala.deriving.* +import scala.collection.AbstractIterable + +trait SolrTokenEncoder[F[_], A]: + def encode(ctx: Context[F], value: A): F[SolrToken] + final def contramap[B](f: B => A): SolrTokenEncoder[F, B] = + SolrTokenEncoder.create((ctx, b) => encode(ctx, f(b))) + +object SolrTokenEncoder: + def apply[F[_], A](using e: SolrTokenEncoder[F, A]): SolrTokenEncoder[F, A] = e + + def create[F[_], A]( + f: (ctx: Context[F], value: A) => F[SolrToken] + ): SolrTokenEncoder[F, A] = + new SolrTokenEncoder[F, A]: + def encode(ctx: Context[F], value: A) = f(ctx, value) + + def curried[F[_], A](f: Context[F] => A => F[SolrToken]): SolrTokenEncoder[F, A] = + create[F, A]((ctx, v) => f(ctx)(v)) + + inline def derived[F[_]: Sync, A](using Mirror.Of[A]): SolrTokenEncoder[F, A] = + Macros.derived[F, A] + + def basic[F[_]: Applicative, A](f: A => SolrToken): SolrTokenEncoder[F, A] = + create[F, A]((_, v) => f(v).pure[F]) + + private object Macros { + import scala.compiletime.* + + inline def derived[F[_]: Sync, T](using m: Mirror.Of[T]): SolrTokenEncoder[F, T] = + lazy val elemInstances = summonInstances[F, T, m.MirroredElemTypes] + inline m match + case s: Mirror.SumOf[T] => sumTokenEncoder(s, elemInstances) + case p: Mirror.ProductOf[T] => prodTokenEncoder(p, elemInstances) + + inline def summonInstances[F[_]: Sync, T, Elems <: Tuple] + : List[SolrTokenEncoder[F, ?]] = + inline erasedValue[Elems] match + case _: (elem *: elems) => + deriveOrSummon[F, T, elem] :: summonInstances[F, T, elems] + case _: EmptyTuple => Nil + + inline def deriveOrSummon[F[_]: Sync, T, Elem]: SolrTokenEncoder[F, Elem] = + inline erasedValue[Elem] match +// case _: T => deriveRec[F, T, Elem] + case _ => summonInline[SolrTokenEncoder[F, Elem]] + + /// we don't need recursive derivation right now + // inline def deriveRec[F[_]:Sync, T, Elem]: SolrTokenEncoder[F, Elem] = + // inline erasedValue[T] match + // case _: Elem => error("infinite recursive derivation") + // case _ => Macros.derived[F, Elem](using Sync[F], summonInline[Mirror.Of[Elem]]) // recursive derivation + + def sumTokenEncoder[F[_]: Sync, T]( + s: Mirror.SumOf[T], + elems: => List[SolrTokenEncoder[F, ?]] + ): SolrTokenEncoder[F, T] = + SolrTokenEncoder.create[F, T] { (ctx, v) => + val ord = s.ordinal(v) + elems(ord).asInstanceOf[SolrTokenEncoder[F, Any]].encode(ctx, v) + } + + def prodTokenEncoder[F[_]: Sync, T]( + s: Mirror.ProductOf[T], + elems: => List[SolrTokenEncoder[F, ?]] + ): SolrTokenEncoder[F, T] = + SolrTokenEncoder.create[F, T] { (ctx, v) => + val vel = iterable(v) + .zip(elems) + .map { case (va, ea) => + ea.asInstanceOf[SolrTokenEncoder[F, Any]].encode(ctx, va) + } + .toList + .sequence + vel.map(_.foldAnd) + } + + def iterable[T](p: T): Iterable[Any] = new AbstractIterable[Any]: + def iterator: Iterator[Any] = p.asInstanceOf[Product].productIterator + } diff --git a/project/Dependencies.scala b/project/Dependencies.scala index c78fd420..0f274eee 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -16,7 +16,6 @@ object Dependencies { val ducktape = "0.1.11" val fs2 = "3.9.4" val http4s = "0.23.25" - val luceneQueryParser = "9.9.2" val redis4Cats = "1.5.2" val scalacheckEffectMunit = "1.0.4" val scodec = "2.2.2" @@ -26,10 +25,6 @@ object Dependencies { val tapir = "1.9.10" } - val luceneQueryParser = Seq( - "org.apache.lucene" % "lucene-queryparser" % V.luceneQueryParser - ) - val catsScalaCheck = Seq( "io.chrisdavenport" %% "cats-scalacheck" % V.catsScalaCheck ) From 86ba6d5f0eb5fd74b4f01f8e974d455b365ac951 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Fri, 23 Feb 2024 22:53:51 +0100 Subject: [PATCH 02/17] WIP: solr query --- .../io/renku/search/api/SearchApiImpl.scala | 2 +- .../io/renku/search/api/tapir/Params.scala | 2 +- .../search/query/parse/QueryParser.scala | 3 +- .../search/query/parse/QueryParserSpec.scala | 4 +- .../renku/search/solr/documents/Project.scala | 3 +- .../io/renku/search/solr/query/Context.scala | 12 ++ .../solr/query/QueryTokenEncoders.scala | 21 ++-- .../renku/search/solr/query/SolrToken.scala | 108 +++++++++++------- nix/scripts/redis-push | 2 +- 9 files changed, 101 insertions(+), 56 deletions(-) diff --git a/modules/search-api/src/main/scala/io/renku/search/api/SearchApiImpl.scala b/modules/search-api/src/main/scala/io/renku/search/api/SearchApiImpl.scala index 7daaa49a..feb280fb 100644 --- a/modules/search-api/src/main/scala/io/renku/search/api/SearchApiImpl.scala +++ b/modules/search-api/src/main/scala/io/renku/search/api/SearchApiImpl.scala @@ -47,7 +47,7 @@ private class SearchApiImpl[F[_]: Async](solrClient: SearchSolrClient[F]) phrase: String ): Throwable => F[Either[String, SearchResult]] = err => - val message = s"Finding by '$phrase' phrase failed" + val message = s"Finding by '$phrase' phrase failed: ${err.getMessage}" Scribe[F] .error(message, err) .as(message) diff --git a/modules/search-api/src/main/scala/io/renku/search/api/tapir/Params.scala b/modules/search-api/src/main/scala/io/renku/search/api/tapir/Params.scala index c851fa54..7f4c62bb 100644 --- a/modules/search-api/src/main/scala/io/renku/search/api/tapir/Params.scala +++ b/modules/search-api/src/main/scala/io/renku/search/api/tapir/Params.scala @@ -42,7 +42,7 @@ object Params extends TapirCodecs with TapirBorerJson { .validate(Validator.max(100)) .default(PageDef.default.limit) - (page / perPage).map(PageDef.fromPage.tupled)(Tuple.fromProductTyped) + (page / perPage).map(PageDef.fromPage.tupled)(p => (p.page, p.limit)) } val queryInput: EndpointInput[QueryInput] = query.and(pageDef).mapTo[QueryInput] diff --git a/modules/search-query/src/main/scala/io/renku/search/query/parse/QueryParser.scala b/modules/search-query/src/main/scala/io/renku/search/query/parse/QueryParser.scala index 56e5a0b1..26b15dbe 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/parse/QueryParser.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/parse/QueryParser.scala @@ -76,13 +76,14 @@ private[query] object QueryParser { nelOf(entityType, commaSep) val termIs: P[FieldTerm] = { - val field = fieldNameFrom(Field.values.toSet - Field.Created - Field.Visibility) + val field = fieldNameFrom(Field.values.toSet - Field.Created - Field.Visibility - Field.Type) ((field <* is) ~ values).map { case (f, v) => f match case Field.Name => FieldTerm.NameIs(v) case Field.ProjectId => FieldTerm.ProjectIdIs(v) case Field.Slug => FieldTerm.SlugIs(v) case Field.CreatedBy => FieldTerm.CreatedByIs(v) + // following fields are excluded from the field list above case Field.Visibility => sys.error("visibility not allowed") case Field.Created => sys.error("created not allowed") case Field.Type => sys.error("type not allowed") diff --git a/modules/search-query/src/test/scala/io/renku/search/query/parse/QueryParserSpec.scala b/modules/search-query/src/test/scala/io/renku/search/query/parse/QueryParserSpec.scala index 2c263f4a..64372263 100644 --- a/modules/search-query/src/test/scala/io/renku/search/query/parse/QueryParserSpec.scala +++ b/modules/search-query/src/test/scala/io/renku/search/query/parse/QueryParserSpec.scala @@ -24,6 +24,7 @@ import io.renku.search.query.Query.Segment import io.renku.search.query.Comparison.{GreaterThan, LowerThan} import munit.{FunSuite, ScalaCheckSuite} import org.scalacheck.Prop +import io.renku.search.model.EntityType class QueryParserSpec extends ScalaCheckSuite with ParserSuite { @@ -70,7 +71,8 @@ class QueryParserSpec extends ScalaCheckSuite with ParserSuite { val data = List( "projectId:id5" -> FieldTerm.ProjectIdIs(Nel.of("id5")), "name:\"my project\"" -> FieldTerm.NameIs(Nel.of("my project")), - "slug:ab1,ab2" -> FieldTerm.SlugIs(Nel.of("ab1", "ab2")) + "slug:ab1,ab2" -> FieldTerm.SlugIs(Nel.of("ab1", "ab2")), + "type:project" -> FieldTerm.TypeIs(Nel.of(EntityType.Project)) ) data.foreach { case (in, expect) => assertEquals(p.run(in), expect) diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/documents/Project.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/documents/Project.scala index 491558f2..650e2649 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/documents/Project.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/documents/Project.scala @@ -33,7 +33,8 @@ final case class Project( description: Option[projects.Description] = None, createdBy: users.Id, creationDate: projects.CreationDate, - members: Seq[users.Id] = Seq.empty + members: Seq[users.Id] = Seq.empty, + score: Option[Double] = None ) object Project: diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/Context.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/Context.scala index 7cd7fcfb..c242da55 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/Context.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/Context.scala @@ -20,7 +20,9 @@ package io.renku.search.solr.query import java.time.Instant import cats.effect.{Clock, Sync} +import cats.syntax.all.* import java.time.ZoneId +import cats.Applicative trait Context[F[_]]: def currentTime: F[Instant] @@ -31,3 +33,13 @@ object Context: new Context[F]: def currentTime: F[Instant] = Clock[F].realTimeInstant def zoneId: F[ZoneId] = Sync[F].delay(ZoneId.systemDefault()) + + def fixed[F[_]: Applicative](time: Instant, zone: ZoneId): Context[F] = + new Context[F]: + def currentTime = time.pure[F] + def zoneId = zone.pure[F] + + def fixedZone[F[_]: Applicative: Clock](zone: ZoneId): Context[F] = + new Context[F]: + def currentTime = Clock[F].realTimeInstant + def zoneId = zone.pure[F] diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/QueryTokenEncoders.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/QueryTokenEncoders.scala index 66ed11ab..d8cd615b 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/QueryTokenEncoders.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/QueryTokenEncoders.scala @@ -33,35 +33,36 @@ trait QueryTokenEncoders: given projectIdIs[F[_]: Applicative]: SolrTokenEncoder[F, FieldTerm.ProjectIdIs] = SolrTokenEncoder.basic { case FieldTerm.ProjectIdIs(ids) => - SolrToken.orFieldIs(Field.ProjectId, ids) + SolrToken.orFieldIs(Field.ProjectId, ids.map(SolrToken.fromString)) } given nameIs[F[_]: Applicative]: SolrTokenEncoder[F, FieldTerm.NameIs] = SolrTokenEncoder.basic { case FieldTerm.NameIs(names) => - SolrToken.orFieldIs(Field.Name, names) + SolrToken.orFieldIs(Field.Name, names.map(SolrToken.fromString)) } given typeIs[F[_]: Applicative]: SolrTokenEncoder[F, FieldTerm.TypeIs] = SolrTokenEncoder.basic { case FieldTerm.TypeIs(values) => - SolrToken.orFieldIs(Field.Type, values.map(_.name)) + SolrToken.orFieldIs(Field.Type, values.map(SolrToken.fromEntityType)) } given slugIs[F[_]: Applicative]: SolrTokenEncoder[F, FieldTerm.SlugIs] = SolrTokenEncoder.basic { case FieldTerm.SlugIs(names) => - SolrToken.orFieldIs(Field.Slug, names) + SolrToken.orFieldIs(Field.Slug, names.map(SolrToken.fromString)) } given createdByIs[F[_]: Applicative]: SolrTokenEncoder[F, FieldTerm.CreatedByIs] = SolrTokenEncoder.basic { case FieldTerm.CreatedByIs(names) => - SolrToken.orFieldIs(Field.CreatedBy, names) + SolrToken.orFieldIs(Field.CreatedBy, names.map(SolrToken.fromString)) } given visibilityIs[F[_]: Applicative]: SolrTokenEncoder[F, FieldTerm.VisibilityIs] = SolrTokenEncoder.basic { case FieldTerm.VisibilityIs(values) => - SolrToken.orFieldIs(Field.Visibility, values.map(_.name)) + SolrToken.orFieldIs(Field.Visibility, values.map(SolrToken.fromVisibility)) } given created[F[_]: Monad]: SolrTokenEncoder[F, FieldTerm.Created] = + val created = SolrToken.fromField(Field.Created) SolrTokenEncoder.create[F, FieldTerm.Created] { case (ctx, FieldTerm.Created(Comparison.Is, values)) => (ctx.currentTime, ctx.zoneId).mapN { (ref, zone) => @@ -69,8 +70,8 @@ trait QueryTokenEncoders: .map(_.resolve(ref, zone)) .map { case (min, maxOpt) => maxOpt - .map(max => SolrToken.dateRange(Field.Created, min, max)) - .getOrElse(SolrToken.dateIs(Field.Created, min)) + .map(max => created === SolrToken.fromDateRange(min, max)) + .getOrElse(created === SolrToken.fromInstant(min)) } .toList .foldOr @@ -81,7 +82,7 @@ trait QueryTokenEncoders: values .map(_.resolve(ref, zone)) .map { case (min, maxOpt) => - SolrToken.dateGt(Field.Created, maxOpt.getOrElse(min)) + created > SolrToken.fromInstant(maxOpt.getOrElse(min)) } .toList .foldOr @@ -92,7 +93,7 @@ trait QueryTokenEncoders: values .map(_.resolve(ref, zone)) .map { case (min, _) => - SolrToken.dateLt(Field.Created, min) + created < SolrToken.fromInstant(min) } .toList .foldOr diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrToken.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrToken.scala index fb20f2d6..767baf59 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrToken.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrToken.scala @@ -23,73 +23,101 @@ import io.renku.search.query.Field import cats.data.NonEmptyList import cats.Monoid import java.time.Instant +import io.renku.search.solr.documents.{Project as SolrProject, User as SolrUser} import io.renku.search.solr.schema.EntityDocumentSchema.Fields as SolrField import io.renku.solr.client.schema.FieldName +import io.renku.search.query.Comparison +import io.renku.search.model.EntityType +import io.renku.search.model.projects.Visibility opaque type SolrToken = String object SolrToken: - def apply(str: String): SolrToken = str - def escaped(str: String): SolrToken = Escape.queryChars(str) + val empty: SolrToken = "" + def fromString(str: String): SolrToken = Escape.queryChars(str) + def fromVisibility(v: Visibility): SolrToken = v.name + def fromEntityType(et: EntityType): SolrToken = + et match + case EntityType.Project => SolrProject.entityType + case EntityType.User => SolrUser.entityType + + def fromField(field: Field): SolrToken = + (field match + case Field.ProjectId => SolrField.id + case Field.Name => SolrField.name + case Field.Slug => SolrField.slug + case Field.Visibility => SolrField.visibility + case Field.CreatedBy => SolrField.createdBy + case Field.Created => SolrField.creationDate + case Field.Type => SolrField.entityType + ).name + + def fromInstant(ts: Instant): SolrToken = ts.toString + def fromDateRange(min: Instant, max: Instant): SolrToken = s"[$min TO $max]" + + def fromComparison(op: Comparison): SolrToken = + op match + case Comparison.Is => ":" + case Comparison.GreaterThan => ">" + case Comparison.LowerThan => "<" def contentAll(text: String): SolrToken = s"${SolrField.contentAll.name}:${Escape.queryChars(text)}" - def fieldIs(field: Field, value: SolrToken): SolrToken = - val name = solrField(field).name - s"${name}:${value.value}" + def orFieldIs(field: Field, values: NonEmptyList[SolrToken]): SolrToken = + values.map(fieldIs(field, _)).toList.foldOr - def orFieldIs(field: Field, values: NonEmptyList[String]): SolrToken = - values.map(Escape.queryChars).map(fieldIs(field, _)).toList.foldOr + def dateIs(field: Field, date: Instant): SolrToken = fieldIs(field, fromInstant(date)) + def dateGt(field: Field, date: Instant): SolrToken = + fieldOp(field, Comparison.GreaterThan, date.toString) + def dateLt(field: Field, date: Instant): SolrToken = + fieldOp(field, Comparison.LowerThan, date.toString) - def dateRange(field: Field, min: Instant, max: Instant): SolrToken = - s"${solrField(field).name}:[$min TO $max]" + //TODO: currently only projects work, user can't be decoded + val allTypes: SolrToken = fieldIs(Field.Type, "Project") - def dateIs(field: Field, date: Instant): SolrToken = fieldIs(field, date.toString) - def dateGt(field: Field, date: Instant): SolrToken = ??? - def dateLt(field: Field, date: Instant): SolrToken = ??? + private def fieldOp(field: Field, op: Comparison, value: SolrToken): SolrToken = + val cmp = fromComparison(op) + val f = fromField(field) + f ~ cmp ~ value - val allTypes: SolrToken = fieldIs(Field.Type, "*") + def fieldIs(field: Field, value: SolrToken): SolrToken = + fieldOp(field, Comparison.Is, value) - private def solrField(field: Field): FieldName = - field match - case Field.ProjectId => SolrField.id - case Field.Name => SolrField.name - case Field.Slug => SolrField.slug - case Field.Visibility => SolrField.visibility - case Field.CreatedBy => SolrField.createdBy - case Field.Created => SolrField.creationDate - case Field.Type => SolrField.entityType + private def monoidWith(sep: String): Monoid[SolrToken] = + Monoid.instance(empty, (a, b) => if (a.isEmpty) b else if (b.isEmpty) a else s"$a$sep$b") + private val orMonoid: Monoid[SolrToken] = monoidWith(" OR ") + private val andMonoid: Monoid[SolrToken] = monoidWith(" AND ") + private val spaceMonoid: Monoid[SolrToken] = monoidWith(" ") - private val orMonoid: Monoid[SolrToken] = - Monoid.instance("", (a, b) => if (a.isEmpty) b else if (b.isEmpty) a else s"$a OR $b") - private val andMonoid: Monoid[SolrToken] = Monoid.instance( - "", - (a, b) => if (a.isEmpty) b else if (b.isEmpty) a else s"$a AND $b" - ) extension (self: SolrToken) def value: String = self + def isEmpty: Boolean = self.isEmpty + def nonEmpty: Boolean = !self.isEmpty + def ~(next: SolrToken): SolrToken = self + next + def +=(next: SolrToken): SolrToken = spaceMonoid.combine(self, next) + def &&(next: SolrToken): SolrToken = andMonoid.combine(self, next) + def ||(next: SolrToken): SolrToken = orMonoid.combine(self, next) + def ===(next: SolrToken): SolrToken = self ~ Comparison.Is.token ~ next + def <(next: SolrToken): SolrToken = self ~ Comparison.LowerThan.token ~ next + def >(next: SolrToken): SolrToken = self ~ Comparison.GreaterThan.token ~ next + + extension (self: Comparison) + def token: SolrToken = fromComparison(self) extension (self: Seq[SolrToken]) - def foldOr: SolrToken = - given Monoid[SolrToken] = orMonoid - if (self.isEmpty) "" - else if (self.tail.isEmpty) self.head - else s"( ${self.combineAll} )" - - def foldAnd: SolrToken = - given Monoid[SolrToken] = andMonoid - if (self.isEmpty) "" - else if (self.tail.isEmpty) self.head - else s"( ${self.combineAll} )" + def foldM(using Monoid[SolrToken]): SolrToken = + val all = self.combineAll + if (self.sizeIs <= 1) all else s"($all)" + def foldOr: SolrToken = foldM(using orMonoid) + def foldAnd: SolrToken = foldM(using andMonoid) // Escapes query characters for solr. This is taken from here: // https://github.com/apache/solr/blob/bcb9f144974ed07aa3b66766302474542067b522/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java#L163 // to not introduce too many dependencies only for this little function private object Escape { private[this] val specialChars = "\\+-!():^[]\"{}~*?|&;/" - private inline def isSpecial(c: Char) = c.isWhitespace || specialChars.contains(c) def queryChars(s: String): String = { diff --git a/nix/scripts/redis-push b/nix/scripts/redis-push index 400a88dc..0cbaf7c2 100644 --- a/nix/scripts/redis-push +++ b/nix/scripts/redis-push @@ -11,4 +11,4 @@ redis_port=${RS_REDIS_PORT:-6379} header='{"source":"dev","type":"project.created","dataContentType":"application/avro+json","schemaVersion":"1","time":0,"requestId":"r1"}' payload=$(jq --null-input --arg id "$1" --arg name "$2" --arg slug "$1/$2" '{"id":$id,"name":$name,"slug":$slug, "repositories":[],"visibility":"PUBLIC","description":{"string":"my project description"},"createdBy":"dev","creationDate":0,"members":[]}') -redis-cli -h $redis_host -p $redis_port XADD events '*' header "$header" payload "$payload" +redis-cli -h $redis_host -p $redis_port XADD events '*' headers "$header" payload "$payload" From 4da16413726a013c73fe7e869414967d94d196ef Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Fri, 23 Feb 2024 23:07:43 +0100 Subject: [PATCH 03/17] Add solr score to results --- .../main/scala/io/renku/search/api/data/SearchEntity.scala | 4 +++- .../scala/io/renku/search/provision/SearchProvisioner.scala | 2 +- .../src/main/scala/io/renku/solr/client/QueryData.scala | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/search-api/src/main/scala/io/renku/search/api/data/SearchEntity.scala b/modules/search-api/src/main/scala/io/renku/search/api/data/SearchEntity.scala index 4a457c1b..3612fc0d 100644 --- a/modules/search-api/src/main/scala/io/renku/search/api/data/SearchEntity.scala +++ b/modules/search-api/src/main/scala/io/renku/search/api/data/SearchEntity.scala @@ -20,6 +20,7 @@ package io.renku.search.api.data import io.bullet.borer.derivation.MapBasedCodecs.{deriveAllCodecs, deriveCodec} import io.bullet.borer.{AdtEncodingStrategy, Codec, Decoder, Encoder} +import io.bullet.borer.NullOptions.given import io.renku.search.model.* import sttp.tapir.Schema.SName import sttp.tapir.SchemaType.{SDateTime, SProductField} @@ -37,7 +38,8 @@ final case class Project( description: Option[projects.Description] = None, createdBy: User, creationDate: projects.CreationDate, - members: Seq[User] + members: Seq[User], + score: Option[Double] = None ) extends SearchEntity object Project: diff --git a/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisioner.scala b/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisioner.scala index 83c3d6d9..4889fdae 100644 --- a/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisioner.scala +++ b/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisioner.scala @@ -132,7 +132,7 @@ private class SearchProvisionerImpl[F[_]: Async]( (from: v1.Visibility) => projects.Visibility.unsafeFromString(from.name()) private lazy val toSolrDocuments: Seq[ProjectCreated] => Seq[Project] = - _.map(_.to[Project]) + _.map(_.into[Project].transform(Field.default(_.score))) private def markProcessedOnFailure( message: QueueMessage, diff --git a/modules/solr-client/src/main/scala/io/renku/solr/client/QueryData.scala b/modules/solr-client/src/main/scala/io/renku/solr/client/QueryData.scala index aee7010a..96a0b5eb 100644 --- a/modules/solr-client/src/main/scala/io/renku/solr/client/QueryData.scala +++ b/modules/solr-client/src/main/scala/io/renku/solr/client/QueryData.scala @@ -50,6 +50,6 @@ object QueryData: QueryData(query.q, Nil, query.limit, query.offset, Nil, Map.empty) def withChildren(query: QueryString): QueryData = - QueryData(query.q, Nil, query.limit, query.offset, Nil, Map("fl" -> "*,[child]")) + QueryData(query.q, Nil, query.limit, query.offset, Nil, Map("fl" -> "* score,[child]")) given Encoder[QueryData] = deriveEncoder From 7b2c626521b20680092f5f61e783b6471b19133e Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Tue, 27 Feb 2024 07:44:49 +0100 Subject: [PATCH 04/17] Add sbt to dev setup --- flake.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flake.nix b/flake.nix index a249a506..5b5b18ea 100644 --- a/flake.nix +++ b/flake.nix @@ -81,6 +81,7 @@ with selfPkgs; [ redis jq + sbt redis-push recreate-container @@ -107,6 +108,7 @@ with selfPkgs; [ redis jq + sbt redis-push vm-build From 75bfe2bcb428135a8387b32c7f8304121be148e4 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Tue, 27 Feb 2024 08:58:27 +0100 Subject: [PATCH 05/17] Account for different solr query types --- .../solr/client/SearchSolrClientImpl.scala | 9 ++--- ...coders.scala => LuceneQueryEncoders.scala} | 5 ++- .../solr/query/LuceneQueryInterpreter.scala | 35 +++++++++++++++++++ .../search/solr/query/QueryInterpreter.scala | 18 +++++----- .../renku/search/solr/query/SolrQuery.scala | 32 +++++++++++++++++ .../search/solr/query/SolrTokenEncoder.scala | 19 +++++----- 6 files changed, 91 insertions(+), 27 deletions(-) rename modules/search-solr-client/src/main/scala/io/renku/search/solr/query/{QueryTokenEncoders.scala => LuceneQueryEncoders.scala} (97%) create mode 100644 modules/search-solr-client/src/main/scala/io/renku/search/solr/query/LuceneQueryInterpreter.scala create mode 100644 modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrQuery.scala diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala index 18cc6ab2..bfe67c85 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala @@ -21,7 +21,7 @@ package io.renku.search.solr.client import cats.effect.Async import cats.syntax.all.* import io.renku.search.solr.documents.Project -import io.renku.search.solr.query.QueryInterpreter +import io.renku.search.solr.query.LuceneQueryInterpreter import io.renku.search.solr.schema.EntityDocumentSchema import io.renku.solr.client.{QueryData, QueryString, SolrClient} import io.renku.search.query.Query @@ -31,7 +31,8 @@ private class SearchSolrClientImpl[F[_]: Async](solrClient: SolrClient[F]) extends SearchSolrClient[F]: private[this] val logger = scribe.cats.effect[F] - private[this] val interpreter = QueryInterpreter[F] + private[this] val interpreter = LuceneQueryInterpreter.forSync[F] + override def insertProjects(projects: Seq[Project]): F[Unit] = solrClient.insert(projects).void @@ -42,10 +43,10 @@ private class SearchSolrClientImpl[F[_]: Async](solrClient: SolrClient[F]) offset: Int ): F[QueryResponse[Project]] = for { - solrQuery <- interpreter.solrQuery(query) + solrQuery <- interpreter.run(query) _ <- logger.debug(s"Query: ${query.render} ->Solr: $solrQuery") res <- solrClient - .query[Project](QueryData.withChildren(QueryString(solrQuery, limit, offset))) + .query[Project](QueryData.withChildren(QueryString(solrQuery.query, limit, offset))) } yield res override def findProjects(phrase: String): F[List[Project]] = diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/QueryTokenEncoders.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/LuceneQueryEncoders.scala similarity index 97% rename from modules/search-solr-client/src/main/scala/io/renku/search/solr/query/QueryTokenEncoders.scala rename to modules/search-solr-client/src/main/scala/io/renku/search/solr/query/LuceneQueryEncoders.scala index d8cd615b..cc30cd61 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/QueryTokenEncoders.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/LuceneQueryEncoders.scala @@ -26,10 +26,9 @@ import io.renku.search.query.Query import cats.Monad import cats.Applicative -import cats.effect.kernel.Sync import io.renku.search.query.Comparison -trait QueryTokenEncoders: +trait LuceneQueryEncoders: given projectIdIs[F[_]: Applicative]: SolrTokenEncoder[F, FieldTerm.ProjectIdIs] = SolrTokenEncoder.basic { case FieldTerm.ProjectIdIs(ids) => @@ -100,7 +99,7 @@ trait QueryTokenEncoders: } } - given fieldTerm[F[_]: Sync]: SolrTokenEncoder[F, FieldTerm] = + given fieldTerm[F[_]: Monad]: SolrTokenEncoder[F, FieldTerm] = SolrTokenEncoder.derived[F, FieldTerm] given fieldSegment[F[_]: Applicative](using diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/LuceneQueryInterpreter.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/LuceneQueryInterpreter.scala new file mode 100644 index 00000000..92060a71 --- /dev/null +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/LuceneQueryInterpreter.scala @@ -0,0 +1,35 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.solr.query + +import cats.Monad +import cats.effect.Sync +import cats.syntax.all.* +import io.renku.search.query.Query + +final class LuceneQueryInterpreter[F[_]: Monad] extends QueryInterpreter[F] with LuceneQueryEncoders: + private val encoder = SolrTokenEncoder[F, Query] + + def run(ctx: Context[F], query: Query): F[SolrQuery] = + if (query.isEmpty) SolrQuery.lucene(SolrToken.allTypes).pure[F] + else encoder.encode(ctx, query).map(t => SolrQuery.lucene(List(SolrToken.allTypes, t).foldAnd)) + +object LuceneQueryInterpreter: + def forSync[F[_]: Sync]: QueryInterpreter.WithContext[F] = + QueryInterpreter.withContext(LuceneQueryInterpreter[F], Context.forSync[F]) diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/QueryInterpreter.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/QueryInterpreter.scala index 4d68a8e2..02ad7787 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/QueryInterpreter.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/QueryInterpreter.scala @@ -18,17 +18,15 @@ package io.renku.search.solr.query -import cats.effect.Sync -import cats.syntax.all.* import io.renku.search.query.Query -final class QueryInterpreter[F[_]: Sync](ctx: Context[F]) extends QueryTokenEncoders: - private val encoder = SolrTokenEncoder[F, Query] - - def solrQuery(query: Query): F[String] = - if (query.isEmpty) SolrToken.allTypes.value.pure[F] - else encoder.encode(ctx, query).map(t => List(SolrToken.allTypes, t).foldAnd.value) +trait QueryInterpreter[F[_]]: + def run(ctx: Context[F], q: Query): F[SolrQuery] object QueryInterpreter: - def apply[F[_]: Sync]: QueryInterpreter[F] = - new QueryInterpreter[F](Context.forSync[F]) + trait WithContext[F[_]]: + def run(q: Query): F[SolrQuery] + + def withContext[F[_]](qi: QueryInterpreter[F], ctx: Context[F]): WithContext[F] = + new WithContext[F]: + def run(q: Query) = qi.run(ctx, q) diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrQuery.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrQuery.scala new file mode 100644 index 00000000..87561b89 --- /dev/null +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrQuery.scala @@ -0,0 +1,32 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.solr.query + +final case class SolrQuery( + queryType: SolrQuery.Type, + query: String +) + +object SolrQuery: + enum Type: + case Lucene + case Edismax + + def lucene(qstr: SolrToken): SolrQuery = + SolrQuery(Type.Lucene, qstr.value) diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrTokenEncoder.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrTokenEncoder.scala index e587c1b6..1822f20a 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrTokenEncoder.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrTokenEncoder.scala @@ -18,9 +18,8 @@ package io.renku.search.solr.query -import cats.Applicative +import cats.{Applicative, Monad} import cats.syntax.all.* -import cats.effect.kernel.Sync import scala.deriving.* import scala.collection.AbstractIterable @@ -41,7 +40,7 @@ object SolrTokenEncoder: def curried[F[_], A](f: Context[F] => A => F[SolrToken]): SolrTokenEncoder[F, A] = create[F, A]((ctx, v) => f(ctx)(v)) - inline def derived[F[_]: Sync, A](using Mirror.Of[A]): SolrTokenEncoder[F, A] = + inline def derived[F[_]: Monad, A](using Mirror.Of[A]): SolrTokenEncoder[F, A] = Macros.derived[F, A] def basic[F[_]: Applicative, A](f: A => SolrToken): SolrTokenEncoder[F, A] = @@ -50,31 +49,31 @@ object SolrTokenEncoder: private object Macros { import scala.compiletime.* - inline def derived[F[_]: Sync, T](using m: Mirror.Of[T]): SolrTokenEncoder[F, T] = + inline def derived[F[_]: Monad, T](using m: Mirror.Of[T]): SolrTokenEncoder[F, T] = lazy val elemInstances = summonInstances[F, T, m.MirroredElemTypes] inline m match case s: Mirror.SumOf[T] => sumTokenEncoder(s, elemInstances) case p: Mirror.ProductOf[T] => prodTokenEncoder(p, elemInstances) - inline def summonInstances[F[_]: Sync, T, Elems <: Tuple] + inline def summonInstances[F[_]: Monad, T, Elems <: Tuple] : List[SolrTokenEncoder[F, ?]] = inline erasedValue[Elems] match case _: (elem *: elems) => deriveOrSummon[F, T, elem] :: summonInstances[F, T, elems] case _: EmptyTuple => Nil - inline def deriveOrSummon[F[_]: Sync, T, Elem]: SolrTokenEncoder[F, Elem] = + inline def deriveOrSummon[F[_]: Monad, T, Elem]: SolrTokenEncoder[F, Elem] = inline erasedValue[Elem] match // case _: T => deriveRec[F, T, Elem] case _ => summonInline[SolrTokenEncoder[F, Elem]] /// we don't need recursive derivation right now - // inline def deriveRec[F[_]:Sync, T, Elem]: SolrTokenEncoder[F, Elem] = + // inline def deriveRec[F[_]:Monad, T, Elem]: SolrTokenEncoder[F, Elem] = // inline erasedValue[T] match // case _: Elem => error("infinite recursive derivation") - // case _ => Macros.derived[F, Elem](using Sync[F], summonInline[Mirror.Of[Elem]]) // recursive derivation + // case _ => Macros.derived[F, Elem](using Monad[F], summonInline[Mirror.Of[Elem]]) // recursive derivation - def sumTokenEncoder[F[_]: Sync, T]( + def sumTokenEncoder[F[_]: Monad, T]( s: Mirror.SumOf[T], elems: => List[SolrTokenEncoder[F, ?]] ): SolrTokenEncoder[F, T] = @@ -83,7 +82,7 @@ object SolrTokenEncoder: elems(ord).asInstanceOf[SolrTokenEncoder[F, Any]].encode(ctx, v) } - def prodTokenEncoder[F[_]: Sync, T]( + def prodTokenEncoder[F[_]: Monad, T]( s: Mirror.ProductOf[T], elems: => List[SolrTokenEncoder[F, ?]] ): SolrTokenEncoder[F, T] = From e76e95bafabbc6f7e3e12ad2a4234e7598fdcd6e Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Tue, 27 Feb 2024 10:25:47 +0100 Subject: [PATCH 06/17] Extend query parser to accept sort terms --- .../scala/io/renku/search/query/Order.scala | 70 +++++++++++++++++++ .../scala/io/renku/search/query/Query.scala | 7 +- .../io/renku/search/query/SortableField.scala | 42 +++++++++++ .../search/query/json/QueryJsonCodec.scala | 11 +++ .../search/query/parse/QueryParser.scala | 44 +++++++++--- .../renku/search/query/parse/QueryUtil.scala | 8 +-- .../renku/search/query/QueryGenerators.scala | 19 +++++ .../search/query/parse/QueryParserSpec.scala | 12 ++++ 8 files changed, 199 insertions(+), 14 deletions(-) create mode 100644 modules/search-query/src/main/scala/io/renku/search/query/Order.scala create mode 100644 modules/search-query/src/main/scala/io/renku/search/query/SortableField.scala diff --git a/modules/search-query/src/main/scala/io/renku/search/query/Order.scala b/modules/search-query/src/main/scala/io/renku/search/query/Order.scala new file mode 100644 index 00000000..36332956 --- /dev/null +++ b/modules/search-query/src/main/scala/io/renku/search/query/Order.scala @@ -0,0 +1,70 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.query + +import cats.data.NonEmptyList +import cats.syntax.all.* +import io.bullet.borer.{Decoder, Encoder} +import io.renku.search.query.parse.QueryParser + +final case class Order(fields: NonEmptyList[Order.OrderedBy]): + def render: String = + s"sort:${fields.render}" + +object Order: + def apply(field: OrderedBy, fields: OrderedBy*): Order = + Order(NonEmptyList(field, fields.toList)) + + def apply(field: (SortableField, Direction), fields: (SortableField, Direction)*): Order = + Order(NonEmptyList(field, fields.toList).map(OrderedBy.apply.tupled)) + + enum Direction: + case Asc + case Desc + + def name: String = productPrefix.toLowerCase + + object Direction: + def fromString(s: String): Either[String, Direction] = + Direction.values + .find(_.name.equalsIgnoreCase(s)) + .toRight(s"Invalid sort direction: $s") + + def unsafeFromString(s: String): Direction = + fromString(s).fold(sys.error, identity) + + final case class OrderedBy( + field: SortableField, + direction: Order.Direction + ): + def render: String = s"${field.name}-${direction.name}" + + object OrderedBy: + given Encoder[OrderedBy] = Encoder.forString.contramap(_.render) + given Decoder[OrderedBy] = + Decoder.forString.mapEither(s => QueryParser.orderedBy.parseAll(s).leftMap(_.show)) + + def fromString(s: String): Either[String, Order] = + QueryParser.sortTerm.parseAll(s).leftMap(_.show) + + def unsafeFromString(s: String): Order = + fromString(s).fold(sys.error, identity) + + extension (self: NonEmptyList[OrderedBy]) + def render: String = self.map(_.render).toList.mkString(",") diff --git a/modules/search-query/src/main/scala/io/renku/search/query/Query.scala b/modules/search-query/src/main/scala/io/renku/search/query/Query.scala index 1db4de84..a45b61d2 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/Query.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/Query.scala @@ -29,13 +29,14 @@ import io.renku.search.query.parse.{QueryParser, QueryUtil} import cats.kernel.Monoid final case class Query( - segments: List[Query.Segment] + segments: List[Query.Segment] ): def render: String = segments .map { case Query.Segment.Field(v) => v.asString case Query.Segment.Text(v) => v + case Query.Segment.Sort(v) => v.render } .mkString(" ") @@ -57,6 +58,7 @@ object Query: enum Segment: case Field(value: FieldTerm) case Text(value: String) + case Sort(value: Order) object Segment: given Monoid[Segment.Text] = @@ -68,6 +70,9 @@ object Query: else if (self.value.isEmpty) other else Segment.Text(s"${self.value} ${other.value}") + def sort(order: Order.OrderedBy, more: Order.OrderedBy*): Segment = + Segment.Sort(Order(NonEmptyList(order, more.toList))) + def text(phrase: String): Segment = Segment.Text(phrase) diff --git a/modules/search-query/src/main/scala/io/renku/search/query/SortableField.scala b/modules/search-query/src/main/scala/io/renku/search/query/SortableField.scala new file mode 100644 index 00000000..6fef6940 --- /dev/null +++ b/modules/search-query/src/main/scala/io/renku/search/query/SortableField.scala @@ -0,0 +1,42 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.query + +import io.bullet.borer.{Decoder, Encoder} + +enum SortableField: + case Name + case Created + case Score + + val name: String = Strings.lowerFirst(productPrefix) + +object SortableField: + given Encoder[SortableField] = Encoder.forString.contramap(_.name) + given Decoder[SortableField] = Decoder.forString.mapEither(fromString) + + private[this] val allNames: String = SortableField.values.map(_.name).mkString(", ") + + def fromString(str: String): Either[String, SortableField] = + SortableField.values + .find(_.name.equalsIgnoreCase(str)) + .toRight(s"Invalid field: $str. Allowed are: $allNames") + + def unsafeFromString(str: String): SortableField = + fromString(str).fold(sys.error, identity) diff --git a/modules/search-query/src/main/scala/io/renku/search/query/json/QueryJsonCodec.scala b/modules/search-query/src/main/scala/io/renku/search/query/json/QueryJsonCodec.scala index ddce9ffa..c9d456fd 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/json/QueryJsonCodec.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/json/QueryJsonCodec.scala @@ -46,15 +46,18 @@ import scala.collection.mutable.ListBuffer */ private[query] object QueryJsonCodec: private[this] val freeTextField = "_text" + private[this] val sortTextField = "_sort" enum Name: case FieldName(v: Field) + case SortName case TextName private given Decoder[Name] = new Decoder[Name]: def read(r: Reader): Name = if (r.tryReadString(freeTextField)) Name.TextName + else if (r.tryReadString(sortTextField)) Name.SortName else Decoder[Field].map(Name.FieldName.apply).read(r) private def writeNelValue[T: Encoder](w: Writer, ts: NonEmptyList[T]): w.type = @@ -94,6 +97,9 @@ private[query] object QueryJsonCodec: case Segment.Field(v) => w.write(v.field) writeFieldTermValue(w, v) + case Segment.Sort(v) => + w.write(sortTextField) + writeNelValue(w, v.fields) } w.writeMapClose() } @@ -136,6 +142,11 @@ private[query] object QueryJsonCodec: Decoder.forTuple[(Comparison, NonEmptyList[DateTimeRef])].read(r) Segment.Field(Created(cmp, date)) + case Name.SortName => + val values = readNel[Order.OrderedBy](r) + Segment.Sort(Order(values)) + + val decoder: Decoder[List[Segment]] = new Decoder[List[Segment]] { def read(r: Reader) = { diff --git a/modules/search-query/src/main/scala/io/renku/search/query/parse/QueryParser.scala b/modules/search-query/src/main/scala/io/renku/search/query/parse/QueryParser.scala index 26b15dbe..8b320a1b 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/parse/QueryParser.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/parse/QueryParser.scala @@ -42,6 +42,28 @@ private[query] object QueryParser { def fieldNameFrom(candidates: Set[Field]) = P.stringIn(mkFieldNames(candidates)).map(Field.unsafeFromString) + val sortableField: P[SortableField] = + P.stringIn( + SortableField.values + .map(_.name) + .toSeq ++ SortableField.values.map(_.name.toLowerCase).toSeq + ).map(SortableField.unsafeFromString) + + val sortDirection: P[Order.Direction] = + P.stringIn( + Order.Direction.values + .map(_.name) + .toSeq ++ Order.Direction.values.map(_.name.toLowerCase).toSeq + ).map(Order.Direction.unsafeFromString) + + val orderedBy: P[Order.OrderedBy] = + (sortableField ~ (P.string("-") *> sortDirection)).map { case (f, s) => + Order.OrderedBy(f, s) + } + + val orderedByNel: P[NonEmptyList[Order.OrderedBy]] = + nelOf(orderedBy, commaSep) + val comparison: P[Comparison] = P.stringIn(Comparison.values.map(_.asString)).map(Comparison.unsafeFromString) @@ -49,6 +71,9 @@ private[query] object QueryParser { val gt: P[Unit] = P.string(Comparison.GreaterThan.asString) val lt: P[Unit] = P.string(Comparison.LowerThan.asString) + val sortTerm: P[Order] = + (P.string("sort").with1 *> (is *> orderedByNel)).map(Order.apply) + val visibility: P[Visibility] = P.stringIn( Visibility.values @@ -76,17 +101,17 @@ private[query] object QueryParser { nelOf(entityType, commaSep) val termIs: P[FieldTerm] = { - val field = fieldNameFrom(Field.values.toSet - Field.Created - Field.Visibility - Field.Type) + val field = fieldNameFrom( + Field.values.toSet - Field.Created - Field.Visibility - Field.Type + ) ((field <* is) ~ values).map { case (f, v) => f match - case Field.Name => FieldTerm.NameIs(v) - case Field.ProjectId => FieldTerm.ProjectIdIs(v) - case Field.Slug => FieldTerm.SlugIs(v) - case Field.CreatedBy => FieldTerm.CreatedByIs(v) - // following fields are excluded from the field list above - case Field.Visibility => sys.error("visibility not allowed") - case Field.Created => sys.error("created not allowed") - case Field.Type => sys.error("type not allowed") + case Field.Name => FieldTerm.NameIs(v) + case Field.ProjectId => FieldTerm.ProjectIdIs(v) + case Field.Slug => FieldTerm.SlugIs(v) + case Field.CreatedBy => FieldTerm.CreatedByIs(v) + // other fields are excluded from the field list above + case f => sys.error(s"$f not allowed") } } @@ -114,6 +139,7 @@ private[query] object QueryParser { val segment: P[Query.Segment] = fieldTerm.map(Query.Segment.Field.apply) | + sortTerm.map(Query.Segment.Sort.apply) | freeText.map(Query.Segment.Text.apply) val query: P[Query] = diff --git a/modules/search-query/src/main/scala/io/renku/search/query/parse/QueryUtil.scala b/modules/search-query/src/main/scala/io/renku/search/query/parse/QueryUtil.scala index 5e249720..dc553509 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/parse/QueryUtil.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/parse/QueryUtil.scala @@ -40,11 +40,11 @@ private[query] object QueryUtil { case (t1: Segment.Text, tc) => loop(rest, tc |+| Some(t1), result) - case (f: Segment.Field, Some(tc)) => - loop(rest, None, f :: tc :: result) + case t @ ((_: Segment.Field) | (_: Segment.Sort), Some(tc)) => + loop(rest, None, t._1 :: tc :: result) - case (f: Segment.Field, None) => - loop(rest, None, f :: result) + case t @ ((_: Segment.Field) | (_: Segment.Sort), None) => + loop(rest, None, t._1 :: result) case Nil => (curr.toList ::: result).reverse diff --git a/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala b/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala index 6c016caa..591c0e26 100644 --- a/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala +++ b/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala @@ -77,6 +77,18 @@ object QueryGenerators: val field: Gen[Field] = Gen.oneOf(Field.values.toSeq) + val sortableField: Gen[SortableField] = + Gen.oneOf(SortableField.values.toSeq) + + val sortDirection: Gen[Order.Direction] = + Gen.oneOf(Order.Direction.values.toSeq) + + val orderedBy: Gen[Order.OrderedBy] = + for { + field <- sortableField + dir <- sortDirection + } yield Order.OrderedBy(field, dir) + // TODO move to commons val visibility: Gen[Visibility] = Gen.oneOf(Visibility.values.toSeq) @@ -153,9 +165,16 @@ object QueryGenerators: Gen.listOfN(len, phrase).map(_.mkString(" ")) } + val sortTerm: Gen[Order] = + Gen.choose(1,5).flatMap { len => + nelOfN(len, orderedBy).map(Order.apply) + } + + val segment: Gen[Query.Segment] = Gen.oneOf( fieldTerm.map(Query.Segment.Field.apply), + sortTerm.map(Query.Segment.Sort.apply), freeText.map(Query.Segment.Text.apply) ) diff --git a/modules/search-query/src/test/scala/io/renku/search/query/parse/QueryParserSpec.scala b/modules/search-query/src/test/scala/io/renku/search/query/parse/QueryParserSpec.scala index 64372263..01911ec9 100644 --- a/modules/search-query/src/test/scala/io/renku/search/query/parse/QueryParserSpec.scala +++ b/modules/search-query/src/test/scala/io/renku/search/query/parse/QueryParserSpec.scala @@ -28,6 +28,18 @@ import io.renku.search.model.EntityType class QueryParserSpec extends ScalaCheckSuite with ParserSuite { + test("sort term") { + val p = QueryParser.sortTerm + assertEquals(p.run("sort:name-asc"), Order(SortableField.Name -> Order.Direction.Asc)) + assertEquals( + p.run("sort:name-asc,score-desc"), + Order( + SortableField.Name -> Order.Direction.Asc, + SortableField.Score -> Order.Direction.Desc + ) + ) + } + test("string list") { val p = QueryParser.values assertEquals(p.run("a,b,c"), nel("a", "b", "c")) From 44b66c1864169a79742029054709e8fe1f9a78af Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Tue, 27 Feb 2024 15:07:39 +0100 Subject: [PATCH 07/17] Pass sort parameter to solr client --- flake.nix | 2 + modules/search-query-docs/docs/manual.md | 29 +++++++ .../solr/client/SearchSolrClientImpl.scala | 7 +- .../solr/query/LuceneQueryEncoders.scala | 77 +++++++++++-------- .../solr/query/LuceneQueryInterpreter.scala | 14 +++- .../renku/search/solr/query/SolrQuery.scala | 27 +++++-- .../search/solr/query/SolrSortCreate.scala | 43 +++++++++++ .../search/solr/query/SolrTokenEncoder.scala | 10 +-- .../solr/schema/EntityDocumentSchema.scala | 2 + .../io/renku/solr/client/QueryData.scala | 13 +++- .../scala/io/renku/solr/client/SolrSort.scala | 56 ++++++++++++++ 11 files changed, 228 insertions(+), 52 deletions(-) create mode 100644 modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrSortCreate.scala create mode 100644 modules/solr-client/src/main/scala/io/renku/solr/client/SolrSort.scala diff --git a/flake.nix b/flake.nix index 5b5b18ea..b119a6f3 100644 --- a/flake.nix +++ b/flake.nix @@ -82,6 +82,7 @@ redis jq sbt + scala-cli redis-push recreate-container @@ -109,6 +110,7 @@ redis jq sbt + scala-cli redis-push vm-build diff --git a/modules/search-query-docs/docs/manual.md b/modules/search-query-docs/docs/manual.md index e71c3cf6..eb70e5c8 100644 --- a/modules/search-query-docs/docs/manual.md +++ b/modules/search-query-docs/docs/manual.md @@ -176,3 +176,32 @@ created:2023-03,2023-06 ``` The above means to match entities created in March 2023 or June 2023. + +## Sorting + +The query allows to define terms for sorting. Sorting is limited to +specific fields, which are: + +```scala mdoc:passthrough +println( + SortableField.values.map(e => s"`${e.name}`").mkString("- ", "\n- ", "") +) +``` + +Sorting by a field is defined by writing the field name, followed by a +dash and the sort direction. Multiple such definitions can be +specified, using a comma separated list. Alternatively, multiple +`sort:…` terms will be combined into a single one in the order they +appear. + +Example: +```scala mdoc:passthrough +val str = Order(SortableField.Score -> Order.Direction.Desc, SortableField.Created -> Order.Direction.Asc).render +println(s"`$str`") +``` +is equivalent to +```scala mdoc:passthrough +val str1 = Order(SortableField.Score -> Order.Direction.Desc).render +val str2 = Order(SortableField.Created -> Order.Direction.Asc).render +println(s"`$str1 $str2`") +``` diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala index bfe67c85..0eccb6b4 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala @@ -33,7 +33,6 @@ private class SearchSolrClientImpl[F[_]: Async](solrClient: SolrClient[F]) private[this] val logger = scribe.cats.effect[F] private[this] val interpreter = LuceneQueryInterpreter.forSync[F] - override def insertProjects(projects: Seq[Project]): F[Unit] = solrClient.insert(projects).void @@ -46,7 +45,11 @@ private class SearchSolrClientImpl[F[_]: Async](solrClient: SolrClient[F]) solrQuery <- interpreter.run(query) _ <- logger.debug(s"Query: ${query.render} ->Solr: $solrQuery") res <- solrClient - .query[Project](QueryData.withChildren(QueryString(solrQuery.query, limit, offset))) + .query[Project]( + QueryData + .withChildren(QueryString(solrQuery.query.value, limit, offset)) + .copy(sort = solrQuery.sort) + ) } yield res override def findProjects(phrase: String): F[List[Project]] = diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/LuceneQueryEncoders.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/LuceneQueryEncoders.scala index cc30cd61..b3d78106 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/LuceneQueryEncoders.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/LuceneQueryEncoders.scala @@ -32,32 +32,34 @@ trait LuceneQueryEncoders: given projectIdIs[F[_]: Applicative]: SolrTokenEncoder[F, FieldTerm.ProjectIdIs] = SolrTokenEncoder.basic { case FieldTerm.ProjectIdIs(ids) => - SolrToken.orFieldIs(Field.ProjectId, ids.map(SolrToken.fromString)) + SolrQuery(SolrToken.orFieldIs(Field.ProjectId, ids.map(SolrToken.fromString))) } given nameIs[F[_]: Applicative]: SolrTokenEncoder[F, FieldTerm.NameIs] = SolrTokenEncoder.basic { case FieldTerm.NameIs(names) => - SolrToken.orFieldIs(Field.Name, names.map(SolrToken.fromString)) + SolrQuery(SolrToken.orFieldIs(Field.Name, names.map(SolrToken.fromString))) } given typeIs[F[_]: Applicative]: SolrTokenEncoder[F, FieldTerm.TypeIs] = SolrTokenEncoder.basic { case FieldTerm.TypeIs(values) => - SolrToken.orFieldIs(Field.Type, values.map(SolrToken.fromEntityType)) + SolrQuery(SolrToken.orFieldIs(Field.Type, values.map(SolrToken.fromEntityType))) } given slugIs[F[_]: Applicative]: SolrTokenEncoder[F, FieldTerm.SlugIs] = SolrTokenEncoder.basic { case FieldTerm.SlugIs(names) => - SolrToken.orFieldIs(Field.Slug, names.map(SolrToken.fromString)) + SolrQuery(SolrToken.orFieldIs(Field.Slug, names.map(SolrToken.fromString))) } given createdByIs[F[_]: Applicative]: SolrTokenEncoder[F, FieldTerm.CreatedByIs] = SolrTokenEncoder.basic { case FieldTerm.CreatedByIs(names) => - SolrToken.orFieldIs(Field.CreatedBy, names.map(SolrToken.fromString)) + SolrQuery(SolrToken.orFieldIs(Field.CreatedBy, names.map(SolrToken.fromString))) } given visibilityIs[F[_]: Applicative]: SolrTokenEncoder[F, FieldTerm.VisibilityIs] = SolrTokenEncoder.basic { case FieldTerm.VisibilityIs(values) => - SolrToken.orFieldIs(Field.Visibility, values.map(SolrToken.fromVisibility)) + SolrQuery( + SolrToken.orFieldIs(Field.Visibility, values.map(SolrToken.fromVisibility)) + ) } given created[F[_]: Monad]: SolrTokenEncoder[F, FieldTerm.Created] = @@ -65,37 +67,43 @@ trait LuceneQueryEncoders: SolrTokenEncoder.create[F, FieldTerm.Created] { case (ctx, FieldTerm.Created(Comparison.Is, values)) => (ctx.currentTime, ctx.zoneId).mapN { (ref, zone) => - values - .map(_.resolve(ref, zone)) - .map { case (min, maxOpt) => - maxOpt - .map(max => created === SolrToken.fromDateRange(min, max)) - .getOrElse(created === SolrToken.fromInstant(min)) - } - .toList - .foldOr + SolrQuery( + values + .map(_.resolve(ref, zone)) + .map { case (min, maxOpt) => + maxOpt + .map(max => created === SolrToken.fromDateRange(min, max)) + .getOrElse(created === SolrToken.fromInstant(min)) + } + .toList + .foldOr + ) } case (ctx, FieldTerm.Created(Comparison.GreaterThan, values)) => (ctx.currentTime, ctx.zoneId).mapN { (ref, zone) => - values - .map(_.resolve(ref, zone)) - .map { case (min, maxOpt) => - created > SolrToken.fromInstant(maxOpt.getOrElse(min)) - } - .toList - .foldOr + SolrQuery( + values + .map(_.resolve(ref, zone)) + .map { case (min, maxOpt) => + created > SolrToken.fromInstant(maxOpt.getOrElse(min)) + } + .toList + .foldOr + ) } case (ctx, FieldTerm.Created(Comparison.LowerThan, values)) => (ctx.currentTime, ctx.zoneId).mapN { (ref, zone) => - values - .map(_.resolve(ref, zone)) - .map { case (min, _) => - created < SolrToken.fromInstant(min) - } - .toList - .foldOr + SolrQuery( + values + .map(_.resolve(ref, zone)) + .map { case (min, _) => + created < SolrToken.fromInstant(min) + } + .toList + .foldOr + ) } } @@ -112,16 +120,21 @@ trait LuceneQueryEncoders: } given textSegment[F[_]: Applicative]: SolrTokenEncoder[F, Segment.Text] = - SolrTokenEncoder.basic(t => SolrToken.contentAll(t.value)) + SolrTokenEncoder.basic(t => SolrQuery(SolrToken.contentAll(t.value))) + + given sortSegment[F[_]: Applicative]: SolrTokenEncoder[F, Segment.Sort] = + SolrTokenEncoder.basic(t => SolrQuery.sort(t.value)) given segment[F[_]](using et: SolrTokenEncoder[F, Segment.Text], - ef: SolrTokenEncoder[F, Segment.Field] + ef: SolrTokenEncoder[F, Segment.Field], + es: SolrTokenEncoder[F, Segment.Sort] ): SolrTokenEncoder[F, Segment] = SolrTokenEncoder.curried[F, Segment] { ctx => { case s: Segment.Text => et.encode(ctx, s) case s: Segment.Field => ef.encode(ctx, s) + case s: Segment.Sort => es.encode(ctx, s) } } @@ -129,7 +142,7 @@ trait LuceneQueryEncoders: se: SolrTokenEncoder[F, A] ): SolrTokenEncoder[F, List[A]] = SolrTokenEncoder.create[F, List[A]] { (ctx, nel) => - nel.traverse(se.encode(ctx, _)).map(_.toSeq.foldAnd) + nel.traverse(se.encode(ctx, _)).map(_.toSeq.combineAll) } given query[F[_]: Monad](using diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/LuceneQueryInterpreter.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/LuceneQueryInterpreter.scala index 92060a71..4f28ee7a 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/LuceneQueryInterpreter.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/LuceneQueryInterpreter.scala @@ -23,12 +23,20 @@ import cats.effect.Sync import cats.syntax.all.* import io.renku.search.query.Query -final class LuceneQueryInterpreter[F[_]: Monad] extends QueryInterpreter[F] with LuceneQueryEncoders: +/** Provides conversion into solrs standard query. See + * https://solr.apache.org/guide/solr/latest/query-guide/standard-query-parser.html + */ +final class LuceneQueryInterpreter[F[_]: Monad] + extends QueryInterpreter[F] + with LuceneQueryEncoders: private val encoder = SolrTokenEncoder[F, Query] def run(ctx: Context[F], query: Query): F[SolrQuery] = - if (query.isEmpty) SolrQuery.lucene(SolrToken.allTypes).pure[F] - else encoder.encode(ctx, query).map(t => SolrQuery.lucene(List(SolrToken.allTypes, t).foldAnd)) + if (query.isEmpty) SolrQuery(SolrToken.allTypes).pure[F] + else + encoder + .encode(ctx, query) +// .map(t => SolrQuery(List(SolrToken.allTypes, t.query).foldAnd)) object LuceneQueryInterpreter: def forSync[F[_]: Sync]: QueryInterpreter.WithContext[F] = diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrQuery.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrQuery.scala index 87561b89..92b4817c 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrQuery.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrQuery.scala @@ -18,15 +18,26 @@ package io.renku.search.solr.query +import cats.Monoid +import cats.syntax.all.* +import io.renku.search.query.Order +import io.renku.solr.client.SolrSort + final case class SolrQuery( - queryType: SolrQuery.Type, - query: String -) + query: SolrToken, + sort: SolrSort +): + def ++(next: SolrQuery): SolrQuery = + SolrQuery(query && next.query, sort ++ next.sort) + object SolrQuery: - enum Type: - case Lucene - case Edismax + val empty: SolrQuery = SolrQuery(SolrToken.empty, SolrSort.empty) + + def apply(e: SolrToken): SolrQuery = + SolrQuery(e, SolrSort.empty) + + def sort(order: Order): SolrQuery = + SolrQuery(SolrToken.empty, SolrSortCreate(order.fields)) - def lucene(qstr: SolrToken): SolrQuery = - SolrQuery(Type.Lucene, qstr.value) + given Monoid[SolrQuery] = Monoid.instance(empty, (a, b) => a ++ b) diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrSortCreate.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrSortCreate.scala new file mode 100644 index 00000000..ffa49510 --- /dev/null +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrSortCreate.scala @@ -0,0 +1,43 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.solr.query + +import io.renku.search.query.{Order, SortableField} +import io.renku.search.solr.schema.EntityDocumentSchema.Fields as SolrField +import io.renku.solr.client.schema.FieldName +import io.renku.solr.client.SolrSort +import cats.data.NonEmptyList + +private object SolrSortCreate: + private def fromField(field: SortableField): FieldName = + field match + case SortableField.Name => SolrField.name + case SortableField.Score => SolrField.score + case SortableField.Created => SolrField.creationDate + + private def fromDirection(d: Order.Direction): SolrSort.Direction = + d match + case Order.Direction.Asc => SolrSort.Direction.Asc + case Order.Direction.Desc => SolrSort.Direction.Desc + + def apply(ts: Order.OrderedBy*): SolrSort = + SolrSort(ts.map(e => (fromField(e.field), fromDirection(e.direction))): _*) + + def apply(ts: NonEmptyList[Order.OrderedBy]): SolrSort = + apply(ts.toList: _*) diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrTokenEncoder.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrTokenEncoder.scala index 1822f20a..d404e3f2 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrTokenEncoder.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrTokenEncoder.scala @@ -24,7 +24,7 @@ import scala.deriving.* import scala.collection.AbstractIterable trait SolrTokenEncoder[F[_], A]: - def encode(ctx: Context[F], value: A): F[SolrToken] + def encode(ctx: Context[F], value: A): F[SolrQuery] final def contramap[B](f: B => A): SolrTokenEncoder[F, B] = SolrTokenEncoder.create((ctx, b) => encode(ctx, f(b))) @@ -32,18 +32,18 @@ object SolrTokenEncoder: def apply[F[_], A](using e: SolrTokenEncoder[F, A]): SolrTokenEncoder[F, A] = e def create[F[_], A]( - f: (ctx: Context[F], value: A) => F[SolrToken] + f: (ctx: Context[F], value: A) => F[SolrQuery] ): SolrTokenEncoder[F, A] = new SolrTokenEncoder[F, A]: def encode(ctx: Context[F], value: A) = f(ctx, value) - def curried[F[_], A](f: Context[F] => A => F[SolrToken]): SolrTokenEncoder[F, A] = + def curried[F[_], A](f: Context[F] => A => F[SolrQuery]): SolrTokenEncoder[F, A] = create[F, A]((ctx, v) => f(ctx)(v)) inline def derived[F[_]: Monad, A](using Mirror.Of[A]): SolrTokenEncoder[F, A] = Macros.derived[F, A] - def basic[F[_]: Applicative, A](f: A => SolrToken): SolrTokenEncoder[F, A] = + def basic[F[_]: Applicative, A](f: A => SolrQuery): SolrTokenEncoder[F, A] = create[F, A]((_, v) => f(v).pure[F]) private object Macros { @@ -94,7 +94,7 @@ object SolrTokenEncoder: } .toList .sequence - vel.map(_.foldAnd) + vel.map(_.combineAll) } def iterable[T](p: T): Iterable[Any] = new AbstractIterable[Any]: diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/schema/EntityDocumentSchema.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/schema/EntityDocumentSchema.scala index a6e8b60a..f2fe9a44 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/schema/EntityDocumentSchema.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/schema/EntityDocumentSchema.scala @@ -38,6 +38,8 @@ object EntityDocumentSchema: val nestParent: FieldName = FieldName("_nest_parent_") // catch-all field val contentAll: FieldName = FieldName("content_all") + // virtual score field + val score: FieldName = FieldName("score") object FieldTypes: val id: FieldType = FieldType.id(TypeName("SearchId")).makeDocValue diff --git a/modules/solr-client/src/main/scala/io/renku/solr/client/QueryData.scala b/modules/solr-client/src/main/scala/io/renku/solr/client/QueryData.scala index 96a0b5eb..ef61c856 100644 --- a/modules/solr-client/src/main/scala/io/renku/solr/client/QueryData.scala +++ b/modules/solr-client/src/main/scala/io/renku/solr/client/QueryData.scala @@ -28,6 +28,7 @@ final case class QueryData( limit: Int, offset: Int, fields: Seq[FieldName], + sort: SolrSort, params: Map[String, String] ): def nextPage: QueryData = @@ -47,9 +48,17 @@ final case class QueryData( object QueryData: def apply(query: QueryString): QueryData = - QueryData(query.q, Nil, query.limit, query.offset, Nil, Map.empty) + QueryData(query.q, Nil, query.limit, query.offset, Nil, SolrSort.empty, Map.empty) def withChildren(query: QueryString): QueryData = - QueryData(query.q, Nil, query.limit, query.offset, Nil, Map("fl" -> "* score,[child]")) + QueryData( + query.q, + Nil, + query.limit, + query.offset, + Nil, + SolrSort.empty, + Map("fl" -> "* score,[child]") + ) given Encoder[QueryData] = deriveEncoder diff --git a/modules/solr-client/src/main/scala/io/renku/solr/client/SolrSort.scala b/modules/solr-client/src/main/scala/io/renku/solr/client/SolrSort.scala new file mode 100644 index 00000000..bd3a8e70 --- /dev/null +++ b/modules/solr-client/src/main/scala/io/renku/solr/client/SolrSort.scala @@ -0,0 +1,56 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.solr.client + +import cats.kernel.Monoid +import io.renku.solr.client.schema.FieldName +import io.bullet.borer.Encoder + +opaque type SolrSort = Seq[(FieldName, SolrSort.Direction)] + +object SolrSort: + enum Direction: + case Asc + case Desc + val name: String = productPrefix.toLowerCase + + object Direction: + def fromString(s: String): Either[String, Direction] = + Direction.values.find(_.toString.equalsIgnoreCase(s)).toRight(s"Invalid sort direction: $s") + def unsafeFromString(s: String): Direction = + fromString(s).fold(sys.error, identity) + + given Encoder[Direction] = Encoder.forString.contramap(_.name) + + def apply(s: (FieldName, Direction)*): SolrSort = s + + val empty: SolrSort = Seq.empty + + extension (self: SolrSort) + def isEmpty: Boolean = self.isEmpty + def nonEmpty: Boolean = !self.isEmpty + def ++(next: SolrSort): SolrSort = + Monoid[SolrSort].combine(self, next) + + given Monoid[SolrSort] = + Monoid.instance(empty, (a, b) => if (a.isEmpty) b else if (b.isEmpty) a else a ++ b) + + given Encoder[SolrSort] = Encoder.forString.contramap(list => + list.map { case (f, d) => s"${f.name} ${d.name}" }.mkString(",") + ) From 90c7d528db51400fb833358e770b2c0d4a8b0c10 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Tue, 27 Feb 2024 15:08:51 +0100 Subject: [PATCH 08/17] Scalafmt --- .../main/scala/io/renku/search/model/EntityType.scala | 1 - .../scala/io/renku/search/query/DateTimeRef.scala | 9 ++++----- .../main/scala/io/renku/search/query/EntityType.scala | 1 + .../main/scala/io/renku/search/query/FieldTerm.scala | 5 +++-- .../src/main/scala/io/renku/search/query/Order.scala | 5 ++++- .../src/main/scala/io/renku/search/query/Query.scala | 4 ++-- .../io/renku/search/query/json/QueryJsonCodec.scala | 1 - .../scala/io/renku/search/query/QueryGenerators.scala | 3 +-- .../scala/io/renku/search/solr/query/SolrQuery.scala | 5 ++--- .../scala/io/renku/search/solr/query/SolrToken.scala | 11 ++++++----- .../main/scala/io/renku/solr/client/SolrSort.scala | 4 +++- 11 files changed, 26 insertions(+), 23 deletions(-) diff --git a/modules/commons/src/main/scala/io/renku/search/model/EntityType.scala b/modules/commons/src/main/scala/io/renku/search/model/EntityType.scala index 42fe5e91..3bc87e3c 100644 --- a/modules/commons/src/main/scala/io/renku/search/model/EntityType.scala +++ b/modules/commons/src/main/scala/io/renku/search/model/EntityType.scala @@ -36,6 +36,5 @@ object EntityType: def unsafeFromString(str: String): EntityType = fromString(str).fold(sys.error, identity) - given Encoder[EntityType] = Encoder.forString.contramap(_.name) given Decoder[EntityType] = Decoder.forString.mapEither(fromString) diff --git a/modules/search-query/src/main/scala/io/renku/search/query/DateTimeRef.scala b/modules/search-query/src/main/scala/io/renku/search/query/DateTimeRef.scala index a3927dc4..b4a1b67c 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/DateTimeRef.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/DateTimeRef.scala @@ -39,7 +39,8 @@ enum DateTimeRef: */ def resolve(ref: Instant, zoneId: ZoneId): (Instant, Option[Instant]) = this match case Relative(RelativeDate.Today) => (ref, None) - case Relative(RelativeDate.Yesterday) => (ref.atZone(zoneId).minusDays(1).toInstant, None) + case Relative(RelativeDate.Yesterday) => + (ref.atZone(zoneId).minusDays(1).toInstant, None) case Literal(pdate) => val min = pdate.instantMin(zoneId) val max = pdate.instantMax(zoneId) @@ -52,12 +53,10 @@ enum DateTimeRef: case rd: RelativeDate => Relative(rd).resolve(ref, zoneId)._1.atZone(zoneId) - if (cdate.range) (ts.minus(cdate.amount).toInstant, Some(ts.plus(cdate.amount).toInstant)) + if (cdate.range) + (ts.minus(cdate.amount).toInstant, Some(ts.plus(cdate.amount).toInstant)) else (ts.plus(cdate.amount).toInstant, None) - - - object DateTimeRef: given Encoder[DateTimeRef] = Encoder.forString.contramap(_.asString) given Decoder[DateTimeRef] = Decoder.forString.mapEither { str => diff --git a/modules/search-query/src/main/scala/io/renku/search/query/EntityType.scala b/modules/search-query/src/main/scala/io/renku/search/query/EntityType.scala index e69de29b..8b137891 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/EntityType.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/EntityType.scala @@ -0,0 +1 @@ + diff --git a/modules/search-query/src/main/scala/io/renku/search/query/FieldTerm.scala b/modules/search-query/src/main/scala/io/renku/search/query/FieldTerm.scala index 771d0d52..51ac2a4c 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/FieldTerm.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/FieldTerm.scala @@ -23,7 +23,8 @@ import io.renku.search.model.EntityType import io.renku.search.model.projects.Visibility enum FieldTerm(val field: Field, val cmp: Comparison): - case TypeIs(values: NonEmptyList[EntityType]) extends FieldTerm(Field.Type, Comparison.Is) + case TypeIs(values: NonEmptyList[EntityType]) + extends FieldTerm(Field.Type, Comparison.Is) case ProjectIdIs(values: NonEmptyList[String]) extends FieldTerm(Field.ProjectId, Comparison.Is) case NameIs(values: NonEmptyList[String]) extends FieldTerm(Field.Name, Comparison.Is) @@ -37,7 +38,7 @@ enum FieldTerm(val field: Field, val cmp: Comparison): private[query] def asString = val value = this match - case TypeIs(values) => + case TypeIs(values) => val ts = values.toList.distinct.map(_.name) ts.mkString(",") case ProjectIdIs(values) => FieldTerm.nelToString(values) diff --git a/modules/search-query/src/main/scala/io/renku/search/query/Order.scala b/modules/search-query/src/main/scala/io/renku/search/query/Order.scala index 36332956..de80624e 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/Order.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/Order.scala @@ -31,7 +31,10 @@ object Order: def apply(field: OrderedBy, fields: OrderedBy*): Order = Order(NonEmptyList(field, fields.toList)) - def apply(field: (SortableField, Direction), fields: (SortableField, Direction)*): Order = + def apply( + field: (SortableField, Direction), + fields: (SortableField, Direction)* + ): Order = Order(NonEmptyList(field, fields.toList).map(OrderedBy.apply.tupled)) enum Direction: diff --git a/modules/search-query/src/main/scala/io/renku/search/query/Query.scala b/modules/search-query/src/main/scala/io/renku/search/query/Query.scala index a45b61d2..3de80a66 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/Query.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/Query.scala @@ -29,14 +29,14 @@ import io.renku.search.query.parse.{QueryParser, QueryUtil} import cats.kernel.Monoid final case class Query( - segments: List[Query.Segment] + segments: List[Query.Segment] ): def render: String = segments .map { case Query.Segment.Field(v) => v.asString case Query.Segment.Text(v) => v - case Query.Segment.Sort(v) => v.render + case Query.Segment.Sort(v) => v.render } .mkString(" ") diff --git a/modules/search-query/src/main/scala/io/renku/search/query/json/QueryJsonCodec.scala b/modules/search-query/src/main/scala/io/renku/search/query/json/QueryJsonCodec.scala index c9d456fd..e32d9a8a 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/json/QueryJsonCodec.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/json/QueryJsonCodec.scala @@ -146,7 +146,6 @@ private[query] object QueryJsonCodec: val values = readNel[Order.OrderedBy](r) Segment.Sort(Order(values)) - val decoder: Decoder[List[Segment]] = new Decoder[List[Segment]] { def read(r: Reader) = { diff --git a/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala b/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala index 591c0e26..4bb09b2f 100644 --- a/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala +++ b/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala @@ -166,11 +166,10 @@ object QueryGenerators: } val sortTerm: Gen[Order] = - Gen.choose(1,5).flatMap { len => + Gen.choose(1, 5).flatMap { len => nelOfN(len, orderedBy).map(Order.apply) } - val segment: Gen[Query.Segment] = Gen.oneOf( fieldTerm.map(Query.Segment.Field.apply), diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrQuery.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrQuery.scala index 92b4817c..8ceac60c 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrQuery.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrQuery.scala @@ -24,13 +24,12 @@ import io.renku.search.query.Order import io.renku.solr.client.SolrSort final case class SolrQuery( - query: SolrToken, - sort: SolrSort + query: SolrToken, + sort: SolrSort ): def ++(next: SolrQuery): SolrQuery = SolrQuery(query && next.query, sort ++ next.sort) - object SolrQuery: val empty: SolrQuery = SolrQuery(SolrToken.empty, SolrSort.empty) diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrToken.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrToken.scala index 767baf59..cc78fff4 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrToken.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrToken.scala @@ -73,7 +73,7 @@ object SolrToken: def dateLt(field: Field, date: Instant): SolrToken = fieldOp(field, Comparison.LowerThan, date.toString) - //TODO: currently only projects work, user can't be decoded + // TODO: currently only projects work, user can't be decoded val allTypes: SolrToken = fieldIs(Field.Type, "Project") private def fieldOp(field: Field, op: Comparison, value: SolrToken): SolrToken = @@ -85,12 +85,14 @@ object SolrToken: fieldOp(field, Comparison.Is, value) private def monoidWith(sep: String): Monoid[SolrToken] = - Monoid.instance(empty, (a, b) => if (a.isEmpty) b else if (b.isEmpty) a else s"$a$sep$b") + Monoid.instance( + empty, + (a, b) => if (a.isEmpty) b else if (b.isEmpty) a else s"$a$sep$b" + ) private val orMonoid: Monoid[SolrToken] = monoidWith(" OR ") private val andMonoid: Monoid[SolrToken] = monoidWith(" AND ") private val spaceMonoid: Monoid[SolrToken] = monoidWith(" ") - extension (self: SolrToken) def value: String = self def isEmpty: Boolean = self.isEmpty @@ -103,8 +105,7 @@ object SolrToken: def <(next: SolrToken): SolrToken = self ~ Comparison.LowerThan.token ~ next def >(next: SolrToken): SolrToken = self ~ Comparison.GreaterThan.token ~ next - extension (self: Comparison) - def token: SolrToken = fromComparison(self) + extension (self: Comparison) def token: SolrToken = fromComparison(self) extension (self: Seq[SolrToken]) def foldM(using Monoid[SolrToken]): SolrToken = diff --git a/modules/solr-client/src/main/scala/io/renku/solr/client/SolrSort.scala b/modules/solr-client/src/main/scala/io/renku/solr/client/SolrSort.scala index bd3a8e70..45e506ae 100644 --- a/modules/solr-client/src/main/scala/io/renku/solr/client/SolrSort.scala +++ b/modules/solr-client/src/main/scala/io/renku/solr/client/SolrSort.scala @@ -32,7 +32,9 @@ object SolrSort: object Direction: def fromString(s: String): Either[String, Direction] = - Direction.values.find(_.toString.equalsIgnoreCase(s)).toRight(s"Invalid sort direction: $s") + Direction.values + .find(_.toString.equalsIgnoreCase(s)) + .toRight(s"Invalid sort direction: $s") def unsafeFromString(s: String): Direction = fromString(s).fold(sys.error, identity) From fa1bc5ad83eb0fc49e0c3b45b55a7d194d7b7d62 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Tue, 27 Feb 2024 15:33:31 +0100 Subject: [PATCH 09/17] Fix tests --- .../io/renku/search/api/SearchApiSpec.scala | 5 ++++- .../provision/SearchProvisionerSpec.scala | 4 ++-- .../io/renku/search/query/EntityType.scala | 17 +++++++++++++++++ .../solr/client/SearchSolrClientSpec.scala | 6 ++++-- 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/modules/search-api/src/test/scala/io/renku/search/api/SearchApiSpec.scala b/modules/search-api/src/test/scala/io/renku/search/api/SearchApiSpec.scala index 061f2f5f..f78abd2c 100644 --- a/modules/search-api/src/test/scala/io/renku/search/api/SearchApiSpec.scala +++ b/modules/search-api/src/test/scala/io/renku/search/api/SearchApiSpec.scala @@ -43,9 +43,12 @@ class SearchApiSpec extends CatsEffectSuite with SearchSolrSpec: results <- searchApi .query(mkQuery("matching")) .map(_.fold(err => fail(s"Calling Search API failed with $err"), identity)) - } yield assert(results.items contains toApiProject(project1)) + } yield assert(results.items.map(scoreToNone) contains toApiProject(project1)) } + private def scoreToNone(e: SearchEntity): SearchEntity = e match + case p: Project => p.copy(score = None) + private def mkQuery(phrase: String): QueryInput = QueryInput.pageOne(Query.parse(phrase).fold(sys.error, identity)) diff --git a/modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala b/modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala index 825a6349..977bdaf9 100644 --- a/modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala +++ b/modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala @@ -66,7 +66,7 @@ class SearchProvisionerSpec extends CatsEffectSuite with QueueSpec with SearchSo .awakeEvery[IO](500 millis) .evalMap(_ => solrClient.findProjects("*")) .flatMap(Stream.emits(_)) - .evalMap(d => solrDocs.update(_ + d)) + .evalMap(d => solrDocs.update(_ + d.copy(score = None))) .compile .drain .start @@ -103,7 +103,7 @@ class SearchProvisionerSpec extends CatsEffectSuite with QueueSpec with SearchSo .evalMap(_ => solrClient.findProjects("*")) .flatMap(Stream.emits(_)) .evalTap(IO.println) - .evalMap(d => solrDocs.update(_ + d)) + .evalMap(d => solrDocs.update(_ + d.copy(score = None))) .compile .drain .start diff --git a/modules/search-query/src/main/scala/io/renku/search/query/EntityType.scala b/modules/search-query/src/main/scala/io/renku/search/query/EntityType.scala index 8b137891..c68d874d 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/EntityType.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/EntityType.scala @@ -1 +1,18 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientSpec.scala b/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientSpec.scala index c92ebdf7..30407bab 100644 --- a/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientSpec.scala +++ b/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientSpec.scala @@ -21,6 +21,7 @@ package io.renku.search.solr.client import cats.effect.IO import io.renku.search.solr.client.SearchSolrClientGenerators.* import munit.CatsEffectSuite +import io.renku.search.query.Query class SearchSolrClientSpec extends CatsEffectSuite with SearchSolrSpec: @@ -30,7 +31,8 @@ class SearchSolrClientSpec extends CatsEffectSuite with SearchSolrSpec: projectDocumentGen("solr-project", "solr project description").generateOne for { _ <- client.insertProjects(Seq(project)) - r <- client.findProjects("solr") - _ = assert(r contains project) + r <- client.queryProjects(Query.parse("solr").toOption.get, 10,0) + _ <- IO.println(r.responseBody.docs) + _ = assert(r.responseBody.docs.map(_.copy(score = None)) contains project) } yield () } From 2fcd7604590859f3d9094d383f5431eac23c8561 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Tue, 27 Feb 2024 15:41:07 +0100 Subject: [PATCH 10/17] Move common code to commons module --- .../commons/generators/CommonGenerators.scala | 33 +++++++++++++++++++ .../io/renku/search/query/EntityType.scala | 1 - .../scala/io/renku/search/query/Order.scala | 1 + .../search/query/parse/QueryParser.scala | 2 +- .../renku/search/query/QueryGenerators.scala | 23 +++++-------- .../solr/client/SearchSolrClientSpec.scala | 2 +- 6 files changed, 44 insertions(+), 18 deletions(-) create mode 100644 modules/commons/src/test/scala/io/renku/commons/generators/CommonGenerators.scala diff --git a/modules/commons/src/test/scala/io/renku/commons/generators/CommonGenerators.scala b/modules/commons/src/test/scala/io/renku/commons/generators/CommonGenerators.scala new file mode 100644 index 00000000..8eafb24a --- /dev/null +++ b/modules/commons/src/test/scala/io/renku/commons/generators/CommonGenerators.scala @@ -0,0 +1,33 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.commons.generators + +import cats.data.NonEmptyList +import org.scalacheck.Gen +import io.renku.search.model.projects.Visibility + +object CommonGenerators: + val visibility: Gen[Visibility] = + Gen.oneOf(Visibility.values.toSeq) + + def nelOfN[A](n: Int, gen: Gen[A]): Gen[NonEmptyList[A]] = + for { + e0 <- gen + en <- Gen.listOfN(n - 1, gen) + } yield NonEmptyList(e0, en) diff --git a/modules/search-query/src/main/scala/io/renku/search/query/EntityType.scala b/modules/search-query/src/main/scala/io/renku/search/query/EntityType.scala index c68d874d..924f8219 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/EntityType.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/EntityType.scala @@ -15,4 +15,3 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - diff --git a/modules/search-query/src/main/scala/io/renku/search/query/Order.scala b/modules/search-query/src/main/scala/io/renku/search/query/Order.scala index de80624e..18aa8418 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/Order.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/Order.scala @@ -62,6 +62,7 @@ object Order: given Encoder[OrderedBy] = Encoder.forString.contramap(_.render) given Decoder[OrderedBy] = Decoder.forString.mapEither(s => QueryParser.orderedBy.parseAll(s).leftMap(_.show)) + given cats.Order[OrderedBy] = cats.Order.by(_.render) def fromString(s: String): Either[String, Order] = QueryParser.sortTerm.parseAll(s).leftMap(_.show) diff --git a/modules/search-query/src/main/scala/io/renku/search/query/parse/QueryParser.scala b/modules/search-query/src/main/scala/io/renku/search/query/parse/QueryParser.scala index 8b320a1b..0fbe5438 100644 --- a/modules/search-query/src/main/scala/io/renku/search/query/parse/QueryParser.scala +++ b/modules/search-query/src/main/scala/io/renku/search/query/parse/QueryParser.scala @@ -62,7 +62,7 @@ private[query] object QueryParser { } val orderedByNel: P[NonEmptyList[Order.OrderedBy]] = - nelOf(orderedBy, commaSep) + nelOf(orderedBy, commaSep).map(_.distinct) val comparison: P[Comparison] = P.stringIn(Comparison.values.map(_.asString)).map(Comparison.unsafeFromString) diff --git a/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala b/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala index 4bb09b2f..93077aa7 100644 --- a/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala +++ b/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala @@ -20,6 +20,7 @@ package io.renku.search.query import cats.data.NonEmptyList import cats.syntax.all.* +import io.renku.commons.generators.CommonGenerators import io.renku.search.model.projects.Visibility import io.renku.search.query.parse.QueryUtil import org.scalacheck.Gen @@ -89,17 +90,6 @@ object QueryGenerators: dir <- sortDirection } yield Order.OrderedBy(field, dir) - // TODO move to commons - val visibility: Gen[Visibility] = - Gen.oneOf(Visibility.values.toSeq) - - // TODO move to commons - def nelOfN[A](n: Int, gen: Gen[A]): Gen[NonEmptyList[A]] = - for { - e0 <- gen - en <- Gen.listOfN(n - 1, gen) - } yield NonEmptyList(e0, en) - private val alphaNumChars = ('a' to 'z') ++ ('A' to 'Z') ++ ('0' to '9') private val simpleWord: Gen[String] = { val len = Gen.choose(2, 12) @@ -121,7 +111,7 @@ object QueryGenerators: } private val stringValues: Gen[NonEmptyList[String]] = - Gen.choose(1, 4).flatMap(n => nelOfN(n, phrase)) + Gen.choose(1, 4).flatMap(n => CommonGenerators.nelOfN(n, phrase)) val projectIdTerm: Gen[FieldTerm] = stringValues.map(FieldTerm.ProjectIdIs(_)) @@ -137,7 +127,10 @@ object QueryGenerators: val visibilityTerm: Gen[FieldTerm] = Gen - .frequency(10 -> visibility.map(NonEmptyList.one), 1 -> nelOfN(2, visibility)) + .frequency( + 10 -> CommonGenerators.visibility.map(NonEmptyList.one), + 1 -> CommonGenerators.nelOfN(2, CommonGenerators.visibility) + ) .map(vs => FieldTerm.VisibilityIs(vs.distinct)) private val comparison: Gen[Comparison] = @@ -147,7 +140,7 @@ object QueryGenerators: for { cmp <- comparison len <- Gen.frequency(5 -> Gen.const(1), 1 -> Gen.choose(1, 3)) - pd <- nelOfN(len, dateTimeRef) + pd <- CommonGenerators.nelOfN(len, dateTimeRef) } yield FieldTerm.Created(cmp, pd) val fieldTerm: Gen[FieldTerm] = @@ -167,7 +160,7 @@ object QueryGenerators: val sortTerm: Gen[Order] = Gen.choose(1, 5).flatMap { len => - nelOfN(len, orderedBy).map(Order.apply) + CommonGenerators.nelOfN(len, orderedBy).map(_.distinct).map(Order.apply) } val segment: Gen[Query.Segment] = diff --git a/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientSpec.scala b/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientSpec.scala index 30407bab..62f691fb 100644 --- a/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientSpec.scala +++ b/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientSpec.scala @@ -31,7 +31,7 @@ class SearchSolrClientSpec extends CatsEffectSuite with SearchSolrSpec: projectDocumentGen("solr-project", "solr project description").generateOne for { _ <- client.insertProjects(Seq(project)) - r <- client.queryProjects(Query.parse("solr").toOption.get, 10,0) + r <- client.queryProjects(Query.parse("solr").toOption.get, 10, 0) _ <- IO.println(r.responseBody.docs) _ = assert(r.responseBody.docs.map(_.copy(score = None)) contains project) } yield () From d23f3bb59811edaab755587cfbba0ad0a0b6aa1d Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Tue, 27 Feb 2024 15:47:05 +0100 Subject: [PATCH 11/17] Fix compile error in test --- .../io/renku/search/provision/SearchProvisionerSpec.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala b/modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala index 977bdaf9..5ddb89a6 100644 --- a/modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala +++ b/modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala @@ -125,7 +125,8 @@ class SearchProvisionerSpec extends CatsEffectSuite with QueueSpec with SearchSo Field.computed( _.visibility, pc => projects.Visibility.unsafeFromString(pc.visibility.name()) - ) + ), + Field.default(_.score) ) override def munitFixtures: Seq[Fixture[_]] = From 4a4e499ff69e58d667ae360ecfe59d05f128c037 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Tue, 27 Feb 2024 16:42:34 +0100 Subject: [PATCH 12/17] Add few more tests --- .../renku/search/query/DateTimeRefSpec.scala | 79 ++++++++++++++ .../solr/query/LuceneQueryEncoderSpec.scala | 103 ++++++++++++++++++ 2 files changed, 182 insertions(+) create mode 100644 modules/search-query/src/test/scala/io/renku/search/query/DateTimeRefSpec.scala create mode 100644 modules/search-solr-client/src/test/scala/io/renku/search/solr/query/LuceneQueryEncoderSpec.scala diff --git a/modules/search-query/src/test/scala/io/renku/search/query/DateTimeRefSpec.scala b/modules/search-query/src/test/scala/io/renku/search/query/DateTimeRefSpec.scala new file mode 100644 index 00000000..0a334a42 --- /dev/null +++ b/modules/search-query/src/test/scala/io/renku/search/query/DateTimeRefSpec.scala @@ -0,0 +1,79 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.query + +import cats.syntax.all.* +import munit.FunSuite +import java.time.Instant +import java.time.ZoneId +import java.time.Period + +class DateTimeRefSpec extends FunSuite: + + val refDate: Instant = Instant.parse("2024-02-27T15:34:55Z") + val utc: ZoneId = ZoneId.of("UTC") + + test("resolve relative dates") { + assertEquals( + RelativeDate.Today.resolve, + (refDate, None) + ) + assertEquals( + DateTimeRef.Relative(RelativeDate.Yesterday).resolve(refDate, utc), + (refDate.atZone(utc).minusDays(1).toInstant(), None) + ) + } + + test("resolve partial date") { + val may = PartialDateTime.unsafeFromString("2023-05") + assertEquals( + may.resolve, + (may.instantMin(utc), may.instantMax(utc).some) + ) + val exact = PartialDateTime.fromInstant(Instant.EPOCH) + assertEquals(exact.resolve, (exact.instantMin(utc), None)) + } + + test("resolve date calc") { + val may = PartialDateTime.unsafeFromString("2023-05") + val calc1 = DateTimeCalc(may, Period.ofDays(5), false) + assertEquals( + calc1.resolve, + (may.instantMin(utc).atZone(utc).plusDays(5).toInstant(), None) + ) + + val calc2 = DateTimeCalc(may, Period.ofDays(-5), false) + assertEquals( + calc2.resolve, + (may.instantMin(utc).atZone(utc).minusDays(5).toInstant(), None) + ) + + val range = DateTimeCalc(may, Period.ofDays(5), true) + assertEquals( + range.resolve, + ( + may.instantMin(utc).atZone(utc).minusDays(5).toInstant(), + may.instantMin(utc).atZone(utc).plusDays(5).toInstant().some + ) + ) + } + + extension (r: RelativeDate) def resolve = DateTimeRef(r).resolve(refDate, utc) + extension (d: PartialDateTime) def resolve = DateTimeRef(d).resolve(refDate, utc) + extension (r: DateTimeCalc) def resolve = DateTimeRef(r).resolve(refDate, utc) diff --git a/modules/search-solr-client/src/test/scala/io/renku/search/solr/query/LuceneQueryEncoderSpec.scala b/modules/search-solr-client/src/test/scala/io/renku/search/solr/query/LuceneQueryEncoderSpec.scala new file mode 100644 index 00000000..c9b04401 --- /dev/null +++ b/modules/search-solr-client/src/test/scala/io/renku/search/solr/query/LuceneQueryEncoderSpec.scala @@ -0,0 +1,103 @@ +/* + * Copyright 2024 Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.renku.search.solr.query + +import munit.FunSuite +import cats.Id +import cats.data.NonEmptyList as Nel +import io.renku.search.query.{Comparison, FieldTerm} +import java.time.{Instant, ZoneId} +import io.renku.search.query.* + +class LuceneQueryEncoderSpec extends FunSuite with LuceneQueryEncoders: + + val refDate: Instant = Instant.parse("2024-02-27T15:34:55Z") + val utc: ZoneId = ZoneId.of("UTC") + + val ctx: Context[Id] = Context.fixed(refDate, utc) + val createdEncoder = SolrTokenEncoder[Id, FieldTerm.Created] + + test("use date-max for greater-than"): + val pd = PartialDateTime.unsafeFromString("2023-05") + val date: FieldTerm.Created = + FieldTerm.Created(Comparison.GreaterThan, Nel.of(DateTimeRef(pd))) + assertEquals( + createdEncoder.encode(ctx, date), + SolrQuery( + SolrToken.fromField(Field.Created) ~ + SolrToken.fromComparison(Comparison.GreaterThan) ~ + SolrToken.fromInstant(pd.instantMax(utc)) + ) + ) + + test("use date-min for lower-than"): + val pd = PartialDateTime.unsafeFromString("2023-05") + val date: FieldTerm.Created = + FieldTerm.Created(Comparison.LowerThan, Nel.of(DateTimeRef(pd))) + assertEquals( + createdEncoder.encode(ctx, date), + SolrQuery( + SolrToken.fromField(Field.Created) ~ + SolrToken.fromComparison(Comparison.LowerThan) ~ + SolrToken.fromInstant(pd.instantMin(utc)) + ) + ) + + test("created comparison is"): + val cToday: FieldTerm.Created = + FieldTerm.Created(Comparison.Is, Nel.of(DateTimeRef(RelativeDate.Today))) + assertEquals( + createdEncoder.encode(ctx, cToday), + SolrQuery(SolrToken.dateIs(Field.Created, refDate)) + ) + + test("single range"): + val pd = PartialDateTime.unsafeFromString("2023-05") + val date: FieldTerm.Created = + FieldTerm.Created(Comparison.Is, Nel.of(DateTimeRef(pd))) + assertEquals( + createdEncoder.encode(ctx, date), + SolrQuery( + SolrToken.fieldIs( + Field.Created, + SolrToken.fromDateRange(pd.instantMin(utc), pd.instantMax(utc)) + ) + ) + ) + + test("multiple range"): + val pd1 = PartialDateTime.unsafeFromString("2023-05") + val pd2 = PartialDateTime.unsafeFromString("2023-08") + val date: FieldTerm.Created = + FieldTerm.Created(Comparison.Is, Nel.of(DateTimeRef(pd1), DateTimeRef(pd2))) + assertEquals( + createdEncoder.encode(ctx, date), + SolrQuery( + List( + SolrToken.fieldIs( + Field.Created, + SolrToken.fromDateRange(pd1.instantMin(utc), pd1.instantMax(utc)) + ), + SolrToken.fieldIs( + Field.Created, + SolrToken.fromDateRange(pd2.instantMin(utc), pd2.instantMax(utc)) + ) + ).foldOr + ) + ) From cea84a66e25c1ba8911a251d78685b6932318e13 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Tue, 27 Feb 2024 16:45:38 +0100 Subject: [PATCH 13/17] Remove skeleton for recursive derivation We use semi-automatic derivation anyways --- .../renku/search/solr/query/SolrTokenEncoder.scala | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrTokenEncoder.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrTokenEncoder.scala index d404e3f2..86f8a2b1 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrTokenEncoder.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/SolrTokenEncoder.scala @@ -59,20 +59,9 @@ object SolrTokenEncoder: : List[SolrTokenEncoder[F, ?]] = inline erasedValue[Elems] match case _: (elem *: elems) => - deriveOrSummon[F, T, elem] :: summonInstances[F, T, elems] + summonInline[SolrTokenEncoder[F, elem]] :: summonInstances[F, T, elems] case _: EmptyTuple => Nil - inline def deriveOrSummon[F[_]: Monad, T, Elem]: SolrTokenEncoder[F, Elem] = - inline erasedValue[Elem] match -// case _: T => deriveRec[F, T, Elem] - case _ => summonInline[SolrTokenEncoder[F, Elem]] - - /// we don't need recursive derivation right now - // inline def deriveRec[F[_]:Monad, T, Elem]: SolrTokenEncoder[F, Elem] = - // inline erasedValue[T] match - // case _: Elem => error("infinite recursive derivation") - // case _ => Macros.derived[F, Elem](using Monad[F], summonInline[Mirror.Of[Elem]]) // recursive derivation - def sumTokenEncoder[F[_]: Monad, T]( s: Mirror.SumOf[T], elems: => List[SolrTokenEncoder[F, ?]] From 0a690a27c759f44217ee9353d1fa965fe8e9f42c Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Tue, 27 Feb 2024 16:51:56 +0100 Subject: [PATCH 14/17] Remove unused code --- .../io/renku/search/solr/query/LuceneQueryInterpreter.scala | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/LuceneQueryInterpreter.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/LuceneQueryInterpreter.scala index 4f28ee7a..a1759660 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/LuceneQueryInterpreter.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/query/LuceneQueryInterpreter.scala @@ -33,10 +33,7 @@ final class LuceneQueryInterpreter[F[_]: Monad] def run(ctx: Context[F], query: Query): F[SolrQuery] = if (query.isEmpty) SolrQuery(SolrToken.allTypes).pure[F] - else - encoder - .encode(ctx, query) -// .map(t => SolrQuery(List(SolrToken.allTypes, t.query).foldAnd)) + else encoder.encode(ctx, query) object LuceneQueryInterpreter: def forSync[F[_]: Sync]: QueryInterpreter.WithContext[F] = From 9a228876e3b14e1b7e22b775c64448bf1c2502f6 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Tue, 27 Feb 2024 16:53:58 +0100 Subject: [PATCH 15/17] Remove println --- .../scala/io/renku/search/solr/client/SearchSolrClientSpec.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientSpec.scala b/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientSpec.scala index 62f691fb..6294b6ef 100644 --- a/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientSpec.scala +++ b/modules/search-solr-client/src/test/scala/io/renku/search/solr/client/SearchSolrClientSpec.scala @@ -32,7 +32,6 @@ class SearchSolrClientSpec extends CatsEffectSuite with SearchSolrSpec: for { _ <- client.insertProjects(Seq(project)) r <- client.queryProjects(Query.parse("solr").toOption.get, 10, 0) - _ <- IO.println(r.responseBody.docs) _ = assert(r.responseBody.docs.map(_.copy(score = None)) contains project) } yield () } From 0da4d3f3bb4cf04a3030d9ef3f463a4089fea8bb Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Tue, 27 Feb 2024 17:05:35 +0100 Subject: [PATCH 16/17] Merge Model- CommonGenerators --- .../generators => search/model}/CommonGenerators.scala | 6 +----- .../test/scala/io/renku/search/query/QueryGenerators.scala | 6 +++--- 2 files changed, 4 insertions(+), 8 deletions(-) rename modules/commons/src/test/scala/io/renku/{commons/generators => search/model}/CommonGenerators.scala (86%) diff --git a/modules/commons/src/test/scala/io/renku/commons/generators/CommonGenerators.scala b/modules/commons/src/test/scala/io/renku/search/model/CommonGenerators.scala similarity index 86% rename from modules/commons/src/test/scala/io/renku/commons/generators/CommonGenerators.scala rename to modules/commons/src/test/scala/io/renku/search/model/CommonGenerators.scala index 8eafb24a..5ab2314c 100644 --- a/modules/commons/src/test/scala/io/renku/commons/generators/CommonGenerators.scala +++ b/modules/commons/src/test/scala/io/renku/search/model/CommonGenerators.scala @@ -16,16 +16,12 @@ * limitations under the License. */ -package io.renku.commons.generators +package io.renku.search.model import cats.data.NonEmptyList import org.scalacheck.Gen -import io.renku.search.model.projects.Visibility object CommonGenerators: - val visibility: Gen[Visibility] = - Gen.oneOf(Visibility.values.toSeq) - def nelOfN[A](n: Int, gen: Gen[A]): Gen[NonEmptyList[A]] = for { e0 <- gen diff --git a/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala b/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala index 93077aa7..9a2401d1 100644 --- a/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala +++ b/modules/search-query/src/test/scala/io/renku/search/query/QueryGenerators.scala @@ -20,7 +20,7 @@ package io.renku.search.query import cats.data.NonEmptyList import cats.syntax.all.* -import io.renku.commons.generators.CommonGenerators +import io.renku.search.model.{CommonGenerators, ModelGenerators} import io.renku.search.model.projects.Visibility import io.renku.search.query.parse.QueryUtil import org.scalacheck.Gen @@ -128,8 +128,8 @@ object QueryGenerators: val visibilityTerm: Gen[FieldTerm] = Gen .frequency( - 10 -> CommonGenerators.visibility.map(NonEmptyList.one), - 1 -> CommonGenerators.nelOfN(2, CommonGenerators.visibility) + 10 -> ModelGenerators.visibilityGen.map(NonEmptyList.one), + 1 -> CommonGenerators.nelOfN(2, ModelGenerators.visibilityGen) ) .map(vs => FieldTerm.VisibilityIs(vs.distinct)) From 2d17a5064751ce3b1776380aae2d138b9f51768d Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Tue, 27 Feb 2024 17:09:35 +0100 Subject: [PATCH 17/17] Remove children transformer, as it is not needed currently --- .../io/renku/search/solr/client/SearchSolrClientImpl.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala b/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala index 0eccb6b4..40bd8cd3 100644 --- a/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala +++ b/modules/search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala @@ -46,8 +46,7 @@ private class SearchSolrClientImpl[F[_]: Async](solrClient: SolrClient[F]) _ <- logger.debug(s"Query: ${query.render} ->Solr: $solrQuery") res <- solrClient .query[Project]( - QueryData - .withChildren(QueryString(solrQuery.query.value, limit, offset)) + QueryData(QueryString(solrQuery.query.value, limit, offset)) .copy(sort = solrQuery.sort) ) } yield res