feat: Enable paging for search endpoints

build.sbt

@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import com.github.sbt.git.SbtGit.GitKeys._
 
 organization := "io.renku"
 name := "renku-search"
@@ -90,9 +91,11 @@ lazy val commons = project
       val targets = sources.map(s => targetDir / s.name)
       IO.copy(sources.zip(targets))
       targets
-    }.taskValue
+    }.taskValue,
+    buildInfoKeys := Seq(name, version, gitHeadCommit, gitDescribedVersion),
+    buildInfoPackage := "io.renku.search"
   )
-  .enablePlugins(AutomateHeaderPlugin)
+  .enablePlugins(AutomateHeaderPlugin, BuildInfoPlugin)
   .disablePlugins(DbTestPlugin, RevolverPlugin)
 
 lazy val http4sBorer = project

flake.nix

@@ -12,6 +12,7 @@
     {
       overlays.default = final: prev: {
         solr = self.packages.${prev.system}.solr;
+        openapi-doc = self.packages.${prev.system}.openapi-doc;
       };
       nixosConfigurations = let
         selfOverlay = {
@@ -55,19 +56,27 @@
       formatter = pkgs.alejandra;
       packages =
         ((import ./nix/dev-scripts.nix) {inherit (pkgs) concatTextFile writeShellScriptBin;})
-        // {
+        // rec {
           solr = pkgs.callPackage (import ./nix/solr.nix) {};
+          swagger-ui = pkgs.callPackage (import ./nix/swagger-ui.nix) {};
+          openapi-doc = pkgs.callPackage (import ./nix/openapi-doc.nix) {inherit swagger-ui;};
         };
 
       devShells = rec {
         default = container;
         container = pkgs.mkShell {
+          RS_SOLR_HOST = "rsdev";
           RS_SOLR_URL = "http://rsdev:8983/solr";
+          RS_SOLR_CORE = "rsdev-test";
           RS_REDIS_HOST = "rsdev";
           RS_REDIS_PORT = "6379";
           RS_CONTAINER = "rsdev";
           RS_LOG_LEVEL = "3";
 
+          #don't start docker container for dbTests
+          NO_SOLR = "true";
+          NO_REDIS = "true";
+
           buildInputs = with pkgs;
           with selfPkgs; [
             redis
@@ -79,15 +88,21 @@
             solr-create-core
             solr-delete-core
             solr-recreate-core
+            solr-recreate-dbtests-cores
           ];
         };
         vm = pkgs.mkShell {
           RS_SOLR_URL = "http://localhost:18983/solr";
+          RS_SOLR_CORE = "rsdev-test";
           RS_REDIS_HOST = "localhost";
           RS_REDIS_PORT = "16379";
           VM_SSH_PORT = "10022";
           RS_LOG_LEVEL = "3";
 
+          #don't start docker container for dbTests
+          NO_SOLR = "true";
+          NO_REDIS = "true";
+
           buildInputs = with pkgs;
           with selfPkgs; [
             redis
@@ -100,6 +115,7 @@
             vm-solr-create-core
             vm-solr-delete-core
             vm-solr-recreate-core
+            solr-recreate-dbtests-cores
           ];
         };
       };

modules/search-api/src/main/scala/io/renku/search/api/HttpApplication.scala

@@ -21,89 +21,27 @@ package io.renku.search.api
 import cats.effect.{Async, Resource}
 import cats.syntax.all.*
 import fs2.io.net.Network
-import io.circe.syntax.given
-import io.renku.search.http.borer.TapirBorerJson
 import io.renku.solr.client.SolrConfig
 import org.http4s.dsl.Http4sDsl
 import org.http4s.server.Router
 import org.http4s.{HttpApp, HttpRoutes, Response}
-import sttp.apispec.openapi.Server
-import sttp.apispec.openapi.circe.given
-import sttp.tapir.*
-import sttp.tapir.docs.openapi.OpenAPIDocsInterpreter
-import sttp.tapir.server.ServerEndpoint
-import sttp.tapir.server.http4s.Http4sServerInterpreter
-import io.renku.search.query.Query
-import io.renku.search.query.docs.SearchQueryManual
+import io.renku.search.api.routes.*
 
 object HttpApplication:
   def apply[F[_]: Async: Network](
       solrConfig: SolrConfig
   ): Resource[F, HttpApp[F]] =
     SearchApi[F](solrConfig).map(new HttpApplication[F](_).router)
 
-class HttpApplication[F[_]: Async](searchApi: SearchApi[F])
-    extends Http4sDsl[F]
-    with TapirBorerJson
-    with TapirCodecs:
+final class HttpApplication[F[_]: Async](searchApi: SearchApi[F]) extends Http4sDsl[F]:
 
-  private val businessRoot = "search"
+  private val prefix = "/search"
+
+  private val search = new SearchRoutes[F](searchApi)
+  private val openapi = new OpenApiRoute[F](prefix, "Renku Search API", search.endpoints)
 
   lazy val router: HttpApp[F] =
     Router[F](
-      s"/$businessRoot" -> businessRoutes,
-      "/" -> operationsRoutes
+      prefix -> (openapi.routes <+> search.routes),
+      "/" -> OperationRoutes[F]
     ).orNotFound
-
-  private lazy val businessRoutes: HttpRoutes[F] =
-    Http4sServerInterpreter[F]().toRoutes(openAPIEndpoint :: businessEndpoints)
-
-  private lazy val businessEndpoints: List[ServerEndpoint[Any, F]] =
-    List(
-      searchEndpointGet.serverLogic(searchApi.query),
-      searchEndpointPost.serverLogic(searchApi.query)
-    )
-
-  private lazy val searchEndpointGet
-      : PublicEndpoint[Query, String, List[SearchEntity], Any] =
-    val q =
-      query[Query]("q").description("User defined query e.g. renku")
-    endpoint.get
-      .in(q)
-      .errorOut(borerJsonBody[String])
-      .out(borerJsonBody[List[SearchEntity]])
-      .description(SearchQueryManual.markdown)
-
-  private val searchEndpointPost: PublicEndpoint[Query, String, List[SearchEntity], Any] =
-    endpoint.post
-      .errorOut(borerJsonBody[String])
-      .in(
-        borerJsonBody[Query]
-          .example(
-            Query(Query.Segment.nameIs("proj-name1"), Query.Segment.text("flight sim"))
-          )
-      )
-      .out(borerJsonBody[List[SearchEntity]])
-      .description(SearchQueryManual.markdown)
-
-  private lazy val openAPIEndpoint =
-    val docs = OpenAPIDocsInterpreter()
-      .serverEndpointsToOpenAPI(businessEndpoints, "Search API", "0.0.1")
-      .servers(List(Server(url = "/search", description = "Renku Search API".some)))
-
-    endpoint
-      .in("spec.json")
-      .get
-      .out(stringJsonBody)
-      .description("OpenAPI docs")
-      .serverLogic(_ => docs.asJson.spaces2.asRight.pure[F])
-
-  private lazy val operationsRoutes: HttpRoutes[F] =
-    Http4sServerInterpreter[F]().toRoutes(List(pingEndpoint))
-
-  private lazy val pingEndpoint =
-    endpoint.get
-      .in("ping")
-      .out(stringBody)
-      .description("Ping")
-      .serverLogic[F](_ => "pong".asRight[Unit].pure[F])

modules/search-api/src/main/scala/io/renku/search/api/SearchApi.scala

@@ -22,11 +22,10 @@ import cats.effect.{Async, Resource}
 import fs2.io.net.Network
 import io.renku.search.solr.client.SearchSolrClient
 import io.renku.solr.client.SolrConfig
-import io.renku.search.query.Query
+import io.renku.search.api.data.*
 
 trait SearchApi[F[_]]:
-  def find(phrase: String): F[Either[String, List[SearchEntity]]]
-  def query(query: Query): F[Either[String, List[SearchEntity]]]
+  def query(query: QueryInput): F[Either[String, SearchResult]]
 
 object SearchApi:
   def apply[F[_]: Async: Network](

modules/search-api/src/main/scala/io/renku/search/api/SearchApiImpl.scala

@@ -24,52 +24,52 @@ import io.renku.search.solr.client.SearchSolrClient
 import io.renku.search.solr.documents.{Project as SolrProject, User as SolrUser}
 import org.http4s.dsl.Http4sDsl
 import scribe.Scribe
-import io.renku.search.query.Query
+import io.renku.search.api.data.*
+import io.renku.solr.client.QueryResponse
 
 private class SearchApiImpl[F[_]: Async](solrClient: SearchSolrClient[F])
     extends Http4sDsl[F]
     with SearchApi[F]:
 
   private given Scribe[F] = scribe.cats[F]
 
-  override def find(phrase: String): F[Either[String, List[SearchEntity]]] =
+  override def query(query: QueryInput): F[Either[String, SearchResult]] =
     solrClient
-      .findProjects(phrase)
-      .map(toApiModel)
+      .queryProjects(query.query, query.page.limit + 1, query.page.offset)
+      .map(toApiResult(query.page))
       .map(_.asRight[String])
-      .handleErrorWith(errorResponse(phrase))
-      .widen
-
-  override def query(query: Query): F[Either[String, List[SearchEntity]]] =
-    solrClient
-      .queryProjects(query)
-      .map(toApiModel)
-      .map(_.asRight[String])
-      .handleErrorWith(errorResponse(query.render))
+      .handleErrorWith(errorResponse(query.query.render))
       .widen
 
   private def errorResponse(
       phrase: String
-  ): Throwable => F[Either[String, List[Project]]] =
+  ): Throwable => F[Either[String, SearchResult]] =
     err =>
       val message = s"Finding by '$phrase' phrase failed"
       Scribe[F]
         .error(message, err)
         .as(message)
-        .map(_.asLeft[List[Project]])
+        .map(_.asLeft[SearchResult])
+
+  private def toApiProject(p: SolrProject): SearchEntity =
+    def toUser(user: SolrUser): User = User(user.id)
+    Project(
+      p.id,
+      p.name,
+      p.slug,
+      p.repositories,
+      p.visibility,
+      p.description,
+      toUser(p.createdBy),
+      p.creationDate,
+      p.members.map(toUser)
+    )
 
-  private def toApiModel(entities: List[SolrProject]): List[Project] =
-    entities.map { p =>
-      def toUser(user: SolrUser): User = User(user.id)
-      Project(
-        p.id,
-        p.name,
-        p.slug,
-        p.repositories,
-        p.visibility,
-        p.description,
-        toUser(p.createdBy),
-        p.creationDate,
-        p.members.map(toUser)
-      )
-    }
+  private def toApiResult(currentPage: PageDef)(
+      solrResult: QueryResponse[SolrProject]
+  ): SearchResult =
+    val hasMore = solrResult.responseBody.docs.size > currentPage.limit
+    val pageInfo = PageWithTotals(currentPage, solrResult.responseBody.numFound, hasMore)
+    val items = solrResult.responseBody.docs.map(toApiProject)
+    if (hasMore) SearchResult(items.init, pageInfo)
+    else SearchResult(items, pageInfo)

modules/search-api/src/main/scala/io/renku/search/api/data/PageDef.scala

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2024 Swiss Data Science Center (SDSC)
+ * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+ * Eidgenössische Technische Hochschule Zürich (ETHZ).
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.renku.search.api.data
+
+import io.bullet.borer.Encoder
+import io.bullet.borer.derivation.MapBasedCodecs
+import io.bullet.borer.Decoder
+
+final case class PageDef(
+    limit: Int,
+    offset: Int
+):
+  require(limit > 0, "limit must be >0")
+  require(offset >= 0, "offset must be positive")
+
+  val page: Int =
+    1 + (offset / limit)
+
+object PageDef:
+  val default: PageDef = PageDef(25, 0)
+
+  def fromPage(pageNum: Int, perPage: Int): PageDef =
+    PageDef(perPage, (pageNum - 1).abs * perPage)
+
+  given Encoder[PageDef] = MapBasedCodecs.deriveEncoder
+  given Decoder[PageDef] = MapBasedCodecs.deriveDecoder

modules/search-api/src/main/scala/io/renku/search/api/data/PageWithTotals.scala

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2024 Swiss Data Science Center (SDSC)
+ * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+ * Eidgenössische Technische Hochschule Zürich (ETHZ).
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.renku.search.api.data
+
+final case class PageWithTotals(
+    page: PageDef,
+    prevPage: Option[Int],
+    nextPage: Option[Int],
+    totalResult: Long,
+    totalPages: Int
+)
+
+object PageWithTotals:
+  def apply(page: PageDef, totalResults: Long, hasMore: Boolean): PageWithTotals =
+    PageWithTotals(
+      page,
+      Option(page.page - 1).filter(_ > 0),
+      Option(page.page + 1).filter(_ => hasMore),
+      totalResults,
+      math.ceil(totalResults.toDouble / page.limit).toInt
+    )