Return details for a project namespace

The solr query is amended with another subquery to resolve existing `namespace` fields in the search results. This is only interesting for projects, as these entities use that field denoting "being in that namespace". `Group` and `User` entities on the other hand use the `namespace` field to *define* this namespace. So the sub-query is constraint to only `User` or `Group` entities. Solr applies this to all result documents, but it is only decoded for projects. `User` and `Group` entities resolve to themselves. So we potentially transfer lots of data from solr only to discard it, sadly. I couldn't find a way around it with the current model. It would be nice to have a clear distinction between defining the namespace and being a member of it (using different fields), but unfortunately we decided against that in the past. The search http api *replaces* the value of the `namespace` field with the full object of either `User` or `Group` in its results, making this a breaking change wrt the search api result structure. Since SOLR cannot guarantee to have a `User` or `Group` being resolved given a namespace, the result is defined as optional. When this happens an error is logged and `None` is returned as a result.
SwissDataScienceCenter · Jun 19, 2024 · 5531675 · 5531675
1 parent ea36370
commit 5531675
Show file tree

Hide file tree

Showing 14 changed files with 194 additions and 69 deletions.
diff --git a/modules/search-api/src/main/scala/io/renku/search/api/data/EntityConverter.scala b/modules/search-api/src/main/scala/io/renku/search/api/data/EntityConverter.scala
@@ -31,7 +31,15 @@ trait EntityConverter:
       id = p.id,
       name = p.name,
       slug = p.slug,
-      namespace = p.namespace,
+      namespace = p.namespaceDetails.flatMap(_.docs.headOption).flatMap {
+        case u: UserDocument  => Some(user(u))
+        case g: GroupDocument => Some(group(g))
+        case v =>
+          scribe.error(
+            s"Error converting project namespace due to incorrect data. A user or group was expected as the project namespace of ${p.id}/${p.slug}, but found: $v"
+          )
+          None
+      },
       repositories = p.repositories,
       visibility = p.visibility,
       description = p.description,

diff --git a/modules/search-api/src/main/scala/io/renku/search/api/data/FacetData.scala b/modules/search-api/src/main/scala/io/renku/search/api/data/FacetData.scala
@@ -21,9 +21,7 @@ package io.renku.search.api.data
 import io.bullet.borer.Decoder
 import io.bullet.borer.Encoder
 import io.bullet.borer.derivation.MapBasedCodecs
-import io.renku.search.api.tapir.SchemaSyntax.*
 import io.renku.search.model.EntityType
-import sttp.tapir.Schema
 
 final case class FacetData(
     entityType: Map[EntityType, Int]
@@ -34,16 +32,3 @@ object FacetData:
 
   given Decoder[FacetData] = MapBasedCodecs.deriveDecoder
   given Encoder[FacetData] = MapBasedCodecs.deriveEncoder
-  given Schema[FacetData] = {
-    given Schema[Map[EntityType, Int]] = Schema.schemaForMap(_.name)
-    Schema
-      .derived[FacetData]
-      .jsonExample(
-        FacetData(
-          Map(
-            EntityType.Project -> 15,
-            EntityType.User -> 3
-          )
-        )
-      )
-  }
diff --git a/modules/search-api/src/main/scala/io/renku/search/api/data/PageDef.scala b/modules/search-api/src/main/scala/io/renku/search/api/data/PageDef.scala
@@ -21,7 +21,6 @@ package io.renku.search.api.data
 import io.bullet.borer.Decoder
 import io.bullet.borer.Encoder
 import io.bullet.borer.derivation.MapBasedCodecs
-import sttp.tapir.Schema
 
 final case class PageDef(
     limit: Int,
@@ -41,4 +40,3 @@ object PageDef:
 
   given Encoder[PageDef] = MapBasedCodecs.deriveEncoder
   given Decoder[PageDef] = MapBasedCodecs.deriveDecoder
-  given Schema[PageDef] = Schema.derived
diff --git a/modules/search-api/src/main/scala/io/renku/search/api/data/PageWithTotals.scala b/modules/search-api/src/main/scala/io/renku/search/api/data/PageWithTotals.scala
@@ -21,7 +21,6 @@ package io.renku.search.api.data
 import io.bullet.borer.NullOptions.given
 import io.bullet.borer.derivation.MapBasedCodecs
 import io.bullet.borer.{Decoder, Encoder}
-import sttp.tapir.Schema
 
 final case class PageWithTotals(
     page: PageDef,
@@ -34,7 +33,6 @@ final case class PageWithTotals(
 object PageWithTotals:
   given Encoder[PageWithTotals] = MapBasedCodecs.deriveEncoder
   given Decoder[PageWithTotals] = MapBasedCodecs.deriveDecoder
-  given Schema[PageWithTotals] = Schema.derived
 
   def apply(page: PageDef, totalResults: Long, hasMore: Boolean): PageWithTotals =
     PageWithTotals(

diff --git a/modules/search-api/src/main/scala/io/renku/search/api/data/SearchEntity.scala b/modules/search-api/src/main/scala/io/renku/search/api/data/SearchEntity.scala
@@ -26,6 +26,15 @@ import io.renku.search.model.*
 
 sealed trait SearchEntity:
   def id: Id
+  def widen: SearchEntity = this
+
+sealed trait UserOrGroup:
+  def id: Id
+
+object UserOrGroup:
+  given AdtEncodingStrategy = AdtEncodingStrategy.flat(SearchEntity.discriminatorField)
+  given Decoder[UserOrGroup] = MapBasedCodecs.deriveDecoder[UserOrGroup]
+  given Encoder[UserOrGroup] = EncoderSupport.derive[UserOrGroup]
 
 object SearchEntity:
   private[api] val discriminatorField = "type"
@@ -37,7 +46,7 @@ object SearchEntity:
       id: Id,
       name: Name,
       slug: Slug,
-      namespace: Option[Namespace],
+      namespace: Option[UserOrGroup],
       repositories: Seq[Repository],
       visibility: Visibility,
       description: Option[Description] = None,
@@ -60,6 +69,7 @@ object SearchEntity:
       lastName: Option[LastName] = None,
       score: Option[Double] = None
   ) extends SearchEntity
+      with UserOrGroup
 
   object User:
     given Encoder[User] = EncoderSupport.deriveWithDiscriminator(discriminatorField)
@@ -73,6 +83,7 @@ object SearchEntity:
       description: Option[Description] = None,
       score: Option[Double] = None
   ) extends SearchEntity
+      with UserOrGroup
   object Group:
     given Encoder[Group] = EncoderSupport.deriveWithDiscriminator(discriminatorField)
     given Decoder[Group] = MapBasedCodecs.deriveDecoder

diff --git a/modules/search-api/src/main/scala/io/renku/search/api/tapir/ApiSchema.scala b/modules/search-api/src/main/scala/io/renku/search/api/tapir/ApiSchema.scala
@@ -24,20 +24,36 @@ import io.renku.search.api.data.*
 import io.renku.search.api.data.SearchEntity.*
 import io.renku.search.api.tapir.SchemaSyntax.*
 import io.renku.search.model.*
+import io.renku.search.query.Query
 import sttp.tapir.Schema.SName
 import sttp.tapir.SchemaType.*
 import sttp.tapir.generic.Configuration
 import sttp.tapir.{FieldName, Schema, SchemaType}
 
 /** tapir schema definitions for the search api data structures */
 trait ApiSchema extends ApiSchema.Primitives:
+  given Schema[Query] = Schema.anyObject[Query]
+  given Schema[QueryInput] = Schema.derived
+
   given Schema[User] = Schema
     .derived[User]
-    .jsonExample(ApiSchema.exampleUser)
+    .jsonExample(ApiSchema.exampleUser.widen)
 
   given Schema[Group] = Schema
     .derived[Group]
-    .jsonExample(ApiSchema.exampleGroup)
+    .jsonExample(ApiSchema.exampleGroup.widen)
+
+  given (using
+      userSchema: Schema[User],
+      groupSchema: Schema[Group]
+  ): Schema[UserOrGroup] =
+    Schema
+      .derived[UserOrGroup]
+      .withDiscriminator(
+        SearchEntity.discriminatorField,
+        Map("User" -> userSchema, "Group" -> groupSchema)
+      )
+      .jsonExample(ApiSchema.exampleGroup: UserOrGroup)
 
   given (using userSchema: Schema[User]): Schema[Project] = Schema
     .derived[Project]
@@ -57,31 +73,37 @@ trait ApiSchema extends ApiSchema.Primitives:
         userSchema.copy(schemaType = userType.copy(fields = df :: userType.fields))
       schemaOptUser.copy(schemaType = SOption(nextUserSchema)(identity))
     }
-    .jsonExample(ApiSchema.exampleProject)
+    .jsonExample(ApiSchema.exampleProject.widen)
 
   given (using
       projectSchema: Schema[Project],
       userSchema: Schema[User],
-      groupSchema: Schema[Group]
-  ): Schema[SearchEntity] = {
-    val derived = Schema.derived[SearchEntity]
-    derived.schemaType match {
-      case s: SCoproduct[?] =>
-        derived.copy(schemaType =
-          s.addDiscriminatorField(
-            FieldName(SearchEntity.discriminatorField),
-            Schema.string,
-            List(
-              projectSchema.name.map(SRef(_)).map("Project" -> _),
-              userSchema.name.map(SRef(_)).map("User" -> _),
-              groupSchema.name.map(SRef(_)).map("Group" -> _)
-            ).flatten.toMap
+      groupSchema: Schema[Group],
+      ug: Schema[UserOrGroup]
+  ): Schema[SearchEntity] =
+    Schema
+      .derived[SearchEntity]
+      .withDiscriminator(
+        SearchEntity.discriminatorField,
+        Map("Project" -> projectSchema, "User" -> userSchema, "Group" -> groupSchema)
+      )
+
+  given Schema[FacetData] = {
+    given Schema[Map[EntityType, Int]] = Schema.schemaForMap(_.name)
+    Schema
+      .derived[FacetData]
+      .jsonExample(
+        FacetData(
+          Map(
+            EntityType.Project -> 15,
+            EntityType.User -> 3
           )
         )
-      case s => derived
-    }
+      )
   }
 
+  given Schema[PageDef] = Schema.derived
+  given Schema[PageWithTotals] = Schema.derived
   given Schema[SearchResult] = Schema.derived
 end ApiSchema
 
@@ -100,33 +122,53 @@ object ApiSchema:
     given Schema[FirstName] = Schema.string[FirstName]
     given Schema[LastName] = Schema.string[LastName]
     given Schema[Email] = Schema.string[Email]
+    given Schema[EntityType] = Schema.derivedEnumeration[EntityType].defaultStringBased
+
+    extension [A](self: Schema[A])
+      def withDiscriminator(property: String, subs: Map[String, Schema[?]]): Schema[A] =
+        self.schemaType match {
+          case s: SCoproduct[?] =>
+            self.copy(schemaType =
+              s.addDiscriminatorField(
+                FieldName(property),
+                Schema.string,
+                subs.toList
+                  .map { case (value, schema) =>
+                    schema.name.map(SRef(_)).map(value -> _)
+                  }
+                  .flatten
+                  .toMap
+              )
+            )
+          case _ => self
+        }
   end Primitives
 
-  val exampleUser: SearchEntity = User(
+  val exampleUser: SearchEntity.User = User(
     Id("1CAF4C73F50D4514A041C9EDDB025A36"),
     Some(Namespace("renku/renku")),
     Some(FirstName("Albert")),
     Some(LastName("Einstein")),
     Some(2.1)
   )
 
-  val exampleGroup: SearchEntity = Group(
+  val exampleGroup: SearchEntity.Group = Group(
     Id("2CAF4C73F50D4514A041C9EDDB025A36"),
     Name("SDSC"),
     Namespace("SDSC"),
     Some(Description("SDSC group")),
     Some(1.1)
   )
 
-  val exampleProject: SearchEntity = Project(
+  val exampleProject: SearchEntity.Project = Project(
     id = Id("01HRA7AZ2Q234CDQWGA052F8MK"),
     name = Name("renku"),
     slug = Slug("renku"),
-    namespace = Some(Namespace("renku/renku")),
+    namespace = Some(exampleGroup),
     repositories = Seq(Repository("https://github.com/renku")),
     visibility = Visibility.Public,
     description = Some(Description("Renku project")),
-    createdBy = Some(exampleUser.asInstanceOf[User]),
+    createdBy = Some(exampleUser),
     creationDate = CreationDate(Instant.now),
     keywords = List(Keyword("data"), Keyword("science")),
     score = Some(1.0)

diff --git a/modules/search-api/src/main/scala/io/renku/search/api/tapir/TapirCodecs.scala b/modules/search-api/src/main/scala/io/renku/search/api/tapir/TapirCodecs.scala
@@ -18,8 +18,6 @@
 
 package io.renku.search.api.tapir
 
-import cats.syntax.all.*
-
 import io.renku.search.api.data.*
 import io.renku.search.model.{EntityType, Id}
 import io.renku.search.query.Query
@@ -29,14 +27,9 @@ trait TapirCodecs:
   given Codec[String, Query, CodecFormat.TextPlain] =
     Codec.string.mapEither(Query.parse(_))(_.render)
 
-  given Schema[Query] = Schema.anyObject[Query]
-  given Schema[QueryInput] = Schema.derived
-
   given Codec[String, EntityType, CodecFormat.TextPlain] =
     Codec.string.mapEither(EntityType.fromString(_))(_.name)
 
-  given Schema[EntityType] = Schema.derivedEnumeration.defaultStringBased
-
   given Codec[String, AuthToken.JwtToken, CodecFormat.TextPlain] =
     Codec.string.map(AuthToken.JwtToken(_))(_.render)
 

diff --git a/modules/search-api/src/test/scala/io/renku/search/api/SearchApiSpec.scala b/modules/search-api/src/test/scala/io/renku/search/api/SearchApiSpec.scala
@@ -45,12 +45,14 @@ class SearchApiSpec extends CatsEffectSuite with SearchSolrSuite:
       "matching",
       "matching description",
       Gen.const(None),
+      Gen.const(None),
       Gen.const(Visibility.Public)
     ).generateOne
     val project2 = projectDocumentGen(
       "disparate",
       "disparate description",
       Gen.const(None),
+      Gen.const(None),
       Gen.const(Visibility.Public)
     ).generateOne
     for {
@@ -75,12 +77,13 @@ class SearchApiSpec extends CatsEffectSuite with SearchSolrSuite:
       "exclusive",
       "exclusive description",
       Gen.const(None),
+      Gen.const(None),
       Gen.const(Visibility.Public)
     ).generateOne.copy(createdBy = userId)
     for {
       client <- IO(searchSolrClient())
       searchApi = new SearchApiImpl[IO](client)
-      _ <- client.upsert(project :: user :: Nil)
+      _ <- client.upsert[EntityDocument](project :: user :: Nil)
       results <- searchApi
         .query(AuthContext.anonymous)(mkQuery("exclusive"))
         .map(_.fold(err => fail(s"Calling Search API failed with $err"), identity))

diff --git a/.../search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala b/.../search-solr-client/src/main/scala/io/renku/search/solr/client/SearchSolrClientImpl.scala
@@ -41,6 +41,7 @@ private class SearchSolrClientImpl[F[_]: Async](solrClient: SolrClient[F])
   private val interpreter = LuceneQueryInterpreter.forSync[F]
 
   private val creatorDetails: FieldName = FieldName("creatorDetails")
+  private val namespaceDetails: FieldName = FieldName("namespaceDetails")
 
   private val typeTerms = Facet.Terms(
     EntityDocumentSchema.Fields.entityType,
@@ -79,6 +80,14 @@ private class SearchSolrClientImpl[F[_]: Async](solrClient: SolrClient[F])
                 1
               ).withFields(FieldName.all)
             )
+            .addSubQuery(
+              namespaceDetails,
+              SubQuery(
+                "{!terms f=namespace v=$row.namespace}",
+                "(_type:User OR _type:Group) AND _kind:fullentity",
+                1
+              ).withFields(FieldName.all)
+            )
         )
     } yield res