feat: Convert user query to solr (#33)

- Convert the user query from what is currently supported into a solr (standard) query. 
- Adds solrs `score` field to the response
- Allow to pass custom sorting
- Allow to filter by entity type

build.sbt

@@ -197,8 +197,7 @@ lazy val searchSolrClient = project
     name := "search-solr-client",
     libraryDependencies ++=
       Dependencies.catsCore ++
-        Dependencies.catsEffect ++
-        Dependencies.luceneQueryParser
+        Dependencies.catsEffect
   )
   .dependsOn(
     avroCodec % "compile->compile;test->test",

flake.nix

@@ -81,6 +81,8 @@
           with selfPkgs; [
             redis
             jq
+            sbt
+            scala-cli
 
             redis-push
             recreate-container
@@ -107,6 +109,8 @@
           with selfPkgs; [
             redis
             jq
+            sbt
+            scala-cli
 
             redis-push
             vm-build

modules/commons/src/main/scala/io/renku/search/model/EntityType.scala

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2024 Swiss Data Science Center (SDSC)
+ * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+ * Eidgenössische Technische Hochschule Zürich (ETHZ).
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.renku.search.model
+
+import io.bullet.borer.Encoder
+import io.bullet.borer.Decoder
+
+enum EntityType:
+  case Project
+  case User
+
+  def name: String = productPrefix.toLowerCase
+
+object EntityType:
+  def fromString(str: String): Either[String, EntityType] =
+    EntityType.values
+      .find(_.name.equalsIgnoreCase(str))
+      .toRight(s"Invalid entity type: $str")
+
+  def unsafeFromString(str: String): EntityType =
+    fromString(str).fold(sys.error, identity)
+
+  given Encoder[EntityType] = Encoder.forString.contramap(_.name)
+  given Decoder[EntityType] = Decoder.forString.mapEither(fromString)

modules/commons/src/test/scala/io/renku/search/model/CommonGenerators.scala

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2024 Swiss Data Science Center (SDSC)
+ * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+ * Eidgenössische Technische Hochschule Zürich (ETHZ).
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.renku.search.model
+
+import cats.data.NonEmptyList
+import org.scalacheck.Gen
+
+object CommonGenerators:
+  def nelOfN[A](n: Int, gen: Gen[A]): Gen[NonEmptyList[A]] =
+    for {
+      e0 <- gen
+      en <- Gen.listOfN(n - 1, gen)
+    } yield NonEmptyList(e0, en)

modules/search-api/src/main/scala/io/renku/search/api/HttpApplication.scala

@@ -37,8 +37,8 @@ final class HttpApplication[F[_]: Async](searchApi: SearchApi[F]) extends Http4s
 
   private val prefix = "/search"
 
-  private val search = new SearchRoutes[F](searchApi)
-  private val openapi = new OpenApiRoute[F](prefix, "Renku Search API", search.endpoints)
+  private val search = SearchRoutes[F](searchApi)
+  private val openapi = OpenApiRoute[F](prefix, "Renku Search API", search.endpoints)
 
   lazy val router: HttpApp[F] =
     Router[F](

modules/search-api/src/main/scala/io/renku/search/api/SearchApiImpl.scala

@@ -47,7 +47,7 @@ private class SearchApiImpl[F[_]: Async](solrClient: SearchSolrClient[F])
       phrase: String
   ): Throwable => F[Either[String, SearchResult]] =
     err =>
-      val message = s"Finding by '$phrase' phrase failed"
+      val message = s"Finding by '$phrase' phrase failed: ${err.getMessage}"
       Scribe[F]
         .error(message, err)
         .as(message)

modules/search-api/src/main/scala/io/renku/search/api/data/SearchEntity.scala

@@ -20,6 +20,7 @@ package io.renku.search.api.data
 
 import io.bullet.borer.derivation.MapBasedCodecs.{deriveAllCodecs, deriveCodec}
 import io.bullet.borer.{AdtEncodingStrategy, Codec, Decoder, Encoder}
+import io.bullet.borer.NullOptions.given
 import io.renku.search.model.*
 import sttp.tapir.Schema.SName
 import sttp.tapir.SchemaType.{SDateTime, SProductField}
@@ -37,7 +38,8 @@ final case class Project(
     description: Option[projects.Description] = None,
     createdBy: User,
     creationDate: projects.CreationDate,
-    members: Seq[User]
+    members: Seq[User],
+    score: Option[Double] = None
 ) extends SearchEntity
 
 object Project:

modules/search-api/src/main/scala/io/renku/search/api/tapir/Params.scala

@@ -42,7 +42,7 @@ object Params extends TapirCodecs with TapirBorerJson {
         .validate(Validator.max(100))
         .default(PageDef.default.limit)
 
-    (page / perPage).map(PageDef.fromPage.tupled)(Tuple.fromProductTyped)
+    (page / perPage).map(PageDef.fromPage.tupled)(p => (p.page, p.limit))
   }
 
   val queryInput: EndpointInput[QueryInput] = query.and(pageDef).mapTo[QueryInput]

modules/search-api/src/test/scala/io/renku/search/api/SearchApiSpec.scala

@@ -43,9 +43,12 @@ class SearchApiSpec extends CatsEffectSuite with SearchSolrSpec:
         results <- searchApi
           .query(mkQuery("matching"))
           .map(_.fold(err => fail(s"Calling Search API failed with $err"), identity))
-      } yield assert(results.items contains toApiProject(project1))
+      } yield assert(results.items.map(scoreToNone) contains toApiProject(project1))
     }
 
+  private def scoreToNone(e: SearchEntity): SearchEntity = e match
+    case p: Project => p.copy(score = None)
+
   private def mkQuery(phrase: String): QueryInput =
     QueryInput.pageOne(Query.parse(phrase).fold(sys.error, identity))
 

modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisioner.scala

@@ -132,7 +132,7 @@ private class SearchProvisionerImpl[F[_]: Async](
     (from: v1.Visibility) => projects.Visibility.unsafeFromString(from.name())
 
   private lazy val toSolrDocuments: Seq[ProjectCreated] => Seq[Project] =
-    _.map(_.to[Project])
+    _.map(_.into[Project].transform(Field.default(_.score)))
 
   private def markProcessedOnFailure(
       message: QueueMessage,

modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala

@@ -66,7 +66,7 @@ class SearchProvisionerSpec extends CatsEffectSuite with QueueSpec with SearchSo
             .awakeEvery[IO](500 millis)
             .evalMap(_ => solrClient.findProjects("*"))
             .flatMap(Stream.emits(_))
-            .evalMap(d => solrDocs.update(_ + d))
+            .evalMap(d => solrDocs.update(_ + d.copy(score = None)))
             .compile
             .drain
             .start
@@ -103,7 +103,7 @@ class SearchProvisionerSpec extends CatsEffectSuite with QueueSpec with SearchSo
             .evalMap(_ => solrClient.findProjects("*"))
             .flatMap(Stream.emits(_))
             .evalTap(IO.println)
-            .evalMap(d => solrDocs.update(_ + d))
+            .evalMap(d => solrDocs.update(_ + d.copy(score = None)))
             .compile
             .drain
             .start
@@ -125,7 +125,8 @@ class SearchProvisionerSpec extends CatsEffectSuite with QueueSpec with SearchSo
         Field.computed(
           _.visibility,
           pc => projects.Visibility.unsafeFromString(pc.visibility.name())
-        )
+        ),
+        Field.default(_.score)
       )
 
   override def munitFixtures: Seq[Fixture[_]] =

modules/search-query-docs/docs/manual.md

@@ -73,6 +73,7 @@ Multiple alternative values can be given as a comma separated list.
 The following fields are available:
 
 ```scala mdoc:passthrough
+import io.renku.search.model.EntityType
 import io.renku.search.query.*
 println(Field.values.map(e => s"`${e.name}`").mkString("- ", "\n- ", ""))
 ```
@@ -81,6 +82,22 @@ Each field allows to specify one or more values, separated by comma.
 The value must be separated by a `:`. For date fields, additional `<`
 and `>` is supported.
 
+### EntityTypes
+
+The field `type` allows to search for specific entity types. If it is
+missing, all entity types are included in the result. Entity types are:
+
+```scala mdoc:passthrough
+println(
+  EntityType.values.map(e => s"`${e.name}`").mkString("- ", "\n- ", "")
+)
+```
+
+Example:
+```scala mdoc:passthrough
+println(s" `${Field.Type.name}:${EntityType.Project.name}`")
+```
+
 ### Dates
 
 Date fields, like
@@ -159,3 +176,32 @@ created:2023-03,2023-06
 ```
 
 The above means to match entities created in March 2023 or June 2023.
+
+## Sorting
+
+The query allows to define terms for sorting. Sorting is limited to
+specific fields, which are:
+
+```scala mdoc:passthrough
+println(
+  SortableField.values.map(e => s"`${e.name}`").mkString("- ", "\n- ", "")
+)
+```
+
+Sorting by a field is defined by writing the field name, followed by a
+dash and the sort direction. Multiple such definitions can be
+specified, using a comma separated list. Alternatively, multiple
+`sort:…` terms will be combined into a single one in the order they
+appear.
+
+Example:
+```scala mdoc:passthrough
+val str = Order(SortableField.Score -> Order.Direction.Desc, SortableField.Created -> Order.Direction.Asc).render
+println(s"`$str`")
+```
+is equivalent to
+```scala mdoc:passthrough
+val str1 = Order(SortableField.Score -> Order.Direction.Desc).render
+val str2 = Order(SortableField.Created -> Order.Direction.Asc).render
+println(s"`$str1 $str2`")
+```

modules/search-query/src/main/scala/io/renku/search/query/DateTimeRef.scala

@@ -21,6 +21,8 @@ package io.renku.search.query
 import cats.syntax.all.*
 import io.bullet.borer.{Decoder, Encoder}
 import io.renku.search.query.parse.DateTimeParser
+import java.time.Instant
+import java.time.ZoneId
 
 enum DateTimeRef:
   case Literal(ref: PartialDateTime)
@@ -32,6 +34,29 @@ enum DateTimeRef:
     case Relative(ref) => ref.name
     case Calc(ref)     => ref.asString
 
+  /** Resolves the date-time reference to a concrete instant using the given reference
+    * date. It either returns a single instant or a time range.
+    */
+  def resolve(ref: Instant, zoneId: ZoneId): (Instant, Option[Instant]) = this match
+    case Relative(RelativeDate.Today) => (ref, None)
+    case Relative(RelativeDate.Yesterday) =>
+      (ref.atZone(zoneId).minusDays(1).toInstant, None)
+    case Literal(pdate) =>
+      val min = pdate.instantMin(zoneId)
+      val max = pdate.instantMax(zoneId)
+      (min, Some(max).filter(_ != min))
+    case Calc(cdate) =>
+      val ts = cdate.ref match
+        case pd: PartialDateTime =>
+          pd.instantMin(zoneId).atZone(zoneId)
+
+        case rd: RelativeDate =>
+          Relative(rd).resolve(ref, zoneId)._1.atZone(zoneId)
+
+      if (cdate.range)
+        (ts.minus(cdate.amount).toInstant, Some(ts.plus(cdate.amount).toInstant))
+      else (ts.plus(cdate.amount).toInstant, None)
+
 object DateTimeRef:
   given Encoder[DateTimeRef] = Encoder.forString.contramap(_.asString)
   given Decoder[DateTimeRef] = Decoder.forString.mapEither { str =>

modules/search-query/src/main/scala/io/renku/search/query/EntityType.scala

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2024 Swiss Data Science Center (SDSC)
+ * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+ * Eidgenössische Technische Hochschule Zürich (ETHZ).
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */

modules/search-query/src/main/scala/io/renku/search/query/Field.scala

@@ -27,6 +27,7 @@ enum Field:
   case Visibility
   case Created
   case CreatedBy
+  case Type
 
   val name: String = Strings.lowerFirst(productPrefix)
 

modules/search-query/src/main/scala/io/renku/search/query/FieldTerm.scala

@@ -19,9 +19,12 @@
 package io.renku.search.query
 
 import cats.data.NonEmptyList
+import io.renku.search.model.EntityType
 import io.renku.search.model.projects.Visibility
 
 enum FieldTerm(val field: Field, val cmp: Comparison):
+  case TypeIs(values: NonEmptyList[EntityType])
+      extends FieldTerm(Field.Type, Comparison.Is)
   case ProjectIdIs(values: NonEmptyList[String])
       extends FieldTerm(Field.ProjectId, Comparison.Is)
   case NameIs(values: NonEmptyList[String]) extends FieldTerm(Field.Name, Comparison.Is)
@@ -35,6 +38,9 @@ enum FieldTerm(val field: Field, val cmp: Comparison):
 
   private[query] def asString =
     val value = this match
+      case TypeIs(values) =>
+        val ts = values.toList.distinct.map(_.name)
+        ts.mkString(",")
       case ProjectIdIs(values) => FieldTerm.nelToString(values)
       case NameIs(values)      => FieldTerm.nelToString(values)
       case SlugIs(values)      => FieldTerm.nelToString(values)