Skip to content

Commit

Permalink
feat: support for UserAdded event (#44)
Browse files Browse the repository at this point in the history
  • Loading branch information
jachro authored Mar 7, 2024
1 parent e0b45ea commit 024bade
Show file tree
Hide file tree
Showing 23 changed files with 468 additions and 129 deletions.
21 changes: 21 additions & 0 deletions modules/commons/src/main/scala/io/renku/search/model/users.scala
Original file line number Diff line number Diff line change
Expand Up @@ -29,3 +29,24 @@ object users:
extension (self: Id) def value: String = self
given Transformer[String, Id] = apply
given Codec[Id] = Codec.bimap[String, Id](_.value, Id.apply)

opaque type FirstName = String
object FirstName:
def apply(v: String): FirstName = v
extension (self: FirstName) def value: String = self
given Transformer[String, FirstName] = apply
given Codec[FirstName] = Codec.bimap[String, FirstName](_.value, FirstName.apply)

opaque type LastName = String
object LastName:
def apply(v: String): LastName = v
extension (self: LastName) def value: String = self
given Transformer[String, LastName] = apply
given Codec[LastName] = Codec.bimap[String, LastName](_.value, LastName.apply)

opaque type Email = String
object Email:
def apply(v: String): Email = v
extension (self: Email) def value: String = self
given Transformer[String, Email] = apply
given Codec[Email] = Codec.bimap[String, Email](_.value, LastName.apply)
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,19 @@

package io.renku.search.model

import io.renku.search.model.projects.*
import cats.syntax.all.*
import org.scalacheck.Gen
import org.scalacheck.cats.implicits.*

import java.time.Instant
import java.time.temporal.ChronoUnit

object ModelGenerators {
object ModelGenerators:

val visibilityGen: Gen[Visibility] = Gen.oneOf(Visibility.values.toList)
val creationDateGen: Gen[CreationDate] = instantGen().map(CreationDate.apply)
val projectVisibilityGen: Gen[projects.Visibility] =
Gen.oneOf(projects.Visibility.values.toList)
val projectCreationDateGen: Gen[projects.CreationDate] =
instantGen().map(projects.CreationDate.apply)

private def instantGen(
min: Instant = Instant.EPOCH,
Expand All @@ -37,4 +40,19 @@ object ModelGenerators {
.chooseNum(min.toEpochMilli, max.toEpochMilli)
.map(Instant.ofEpochMilli(_).truncatedTo(ChronoUnit.MILLIS))

}
val userIdGen: Gen[users.Id] = Gen.uuid.map(uuid => users.Id(uuid.toString))
val userFirstNameGen: Gen[users.FirstName] = Gen
.oneOf("Eike", "Kuba", "Ralf", "Lorenzo", "Jean-Pierre", "Alfonso")
.map(users.FirstName.apply)
val userLastNameGen: Gen[users.LastName] = Gen
.oneOf("Kowalski", "Doe", "Tourist", "Milkman", "Da Silva", "Bar")
.map(users.LastName.apply)
def userEmailGen(first: users.FirstName, last: users.LastName): Gen[users.Email] = Gen
.oneOf("mail.com", "hotmail.com", "epfl.ch", "ethz.ch")
.map(v => users.Email(s"$first.$last@$v"))
val userEmailGen: Gen[users.Email] =
(
Gen.oneOf("mail.com", "hotmail.com", "epfl.ch", "ethz.ch"),
userFirstNameGen,
userLastNameGen
).mapN((f, l, p) => users.Email(s"$f.$l@$p"))
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

package io.renku.events

import io.renku.events.v1.{ProjectCreated, Visibility}
import io.renku.events.v1.{ProjectCreated, UserAdded, Visibility}
import org.scalacheck.Gen
import org.scalacheck.Gen.alphaNumChar

Expand Down Expand Up @@ -46,6 +46,19 @@ object EventsGenerators:
Instant.now().truncatedTo(ChronoUnit.MILLIS)
)

def userAddedGen(prefix: String): Gen[UserAdded] =
for
id <- Gen.uuid.map(_.toString)
firstName <- Gen.option(stringGen(max = 5).map(v => s"$prefix-$v"))
lastName <- stringGen(max = 5).map(v => s"$prefix-$v")
email <- Gen.option(stringGen(max = 5).map(host => s"$lastName@$host.com"))
yield UserAdded(
id,
firstName,
Some(lastName),
email
)

def stringGen(max: Int): Gen[String] =
Gen
.chooseNum(3, max)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import io.github.arainko.ducktape.*
import io.renku.search.api.data.*
import io.renku.search.model.users
import io.renku.search.solr.client.SearchSolrClient
import io.renku.search.solr.documents.Project as SolrProject
import io.renku.search.solr.documents.Entity as SolrEntity
import io.renku.solr.client.QueryResponse
import org.http4s.dsl.Http4sDsl
import scribe.Scribe
Expand All @@ -37,7 +37,7 @@ private class SearchApiImpl[F[_]: Async](solrClient: SearchSolrClient[F])

override def query(query: QueryInput): F[Either[String, SearchResult]] =
solrClient
.queryProjects(query.query, query.page.limit + 1, query.page.offset)
.queryEntity(query.query, query.page.limit + 1, query.page.offset)
.map(toApiResult(query.page))
.map(_.asRight[String])
.handleErrorWith(errorResponse(query.query.render))
Expand All @@ -54,14 +54,14 @@ private class SearchApiImpl[F[_]: Async](solrClient: SearchSolrClient[F])
.map(_.asLeft[SearchResult])

private def toApiResult(currentPage: PageDef)(
solrResult: QueryResponse[SolrProject]
solrResult: QueryResponse[SolrEntity]
): SearchResult =
val hasMore = solrResult.responseBody.docs.size > currentPage.limit
val pageInfo = PageWithTotals(currentPage, solrResult.responseBody.numFound, hasMore)
val items = solrResult.responseBody.docs.map(toApiProject)
val items = solrResult.responseBody.docs.map(toApiEntity)
if (hasMore) SearchResult(items.init, pageInfo)
else SearchResult(items, pageInfo)

private lazy val toApiProject: SolrProject => Project =
given Transformer[users.Id, User] = (id: users.Id) => User(id)
_.to[Project]
private lazy val toApiEntity: SolrEntity => SearchEntity =
given Transformer[users.Id, UserId] = (id: users.Id) => UserId(id)
_.to[SearchEntity]
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,12 @@

package io.renku.search.api.data

import io.bullet.borer.NullOptions.given
import io.bullet.borer.derivation.MapBasedCodecs.{deriveAllCodecs, deriveCodec}
import io.bullet.borer.{AdtEncodingStrategy, Codec, Decoder, Encoder}
import io.bullet.borer.NullOptions.given
import io.renku.search.model.*
import sttp.tapir.Schema.SName
import sttp.tapir.SchemaType.{SDateTime, SProductField}
import sttp.tapir.SchemaType.{SCoproduct, SDateTime, SProductField, SRef}
import sttp.tapir.generic.Configuration
import sttp.tapir.{FieldName, Schema, SchemaType}

Expand All @@ -36,7 +36,7 @@ final case class Project(
repositories: Seq[projects.Repository],
visibility: projects.Visibility,
description: Option[projects.Description] = None,
createdBy: User,
createdBy: UserId,
creationDate: projects.CreationDate,
score: Option[Double] = None
) extends SearchEntity
Expand All @@ -52,14 +52,26 @@ object Project:
private given Schema[projects.CreationDate] = Schema(SDateTime())
given Schema[Project] = Schema.derived[Project]

final case class UserId(id: users.Id)
object UserId:
given Codec[UserId] = deriveCodec[UserId]

private given Schema[users.Id] = Schema.string[users.Id]
given Schema[UserId] = Schema.derived[UserId]

final case class User(
id: users.Id
)
id: users.Id,
firstName: Option[users.FirstName] = None,
lastName: Option[users.LastName] = None,
email: Option[users.Email] = None,
score: Option[Double] = None
) extends SearchEntity

object User:
given Codec[User] = deriveCodec[User]

private given Schema[users.Id] = Schema.string[users.Id]
private given Schema[users.FirstName] = Schema.string[users.FirstName]
private given Schema[users.LastName] = Schema.string[users.LastName]
private given Schema[users.Email] = Schema.string[users.Email]
given Schema[User] = Schema.derived[User]

object SearchEntity:
Expand All @@ -71,13 +83,14 @@ object SearchEntity:
given Schema[SearchEntity] = {
val derived = Schema.derived[SearchEntity]
derived.schemaType match {
case s: SchemaType.SCoproduct[_] =>
case s: SCoproduct[_] =>
derived.copy(schemaType =
s.addDiscriminatorField(
FieldName(discriminatorField),
Schema.string,
List(
implicitly[Schema[Project]].name.map(SchemaType.SRef(_)).map("Project" -> _)
summon[Schema[Project]].name.map(SRef(_)).map("Project" -> _),
summon[Schema[User]].name.map(SRef(_)).map("User" -> _)
).flatten.toMap
)
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,19 @@
package io.renku.search.api

import cats.effect.IO
import cats.syntax.all.*
import io.github.arainko.ducktape.*
import io.renku.search.api.data.*
import io.renku.search.model.users
import io.renku.search.query.Query
import io.renku.search.solr.client.SearchSolrClientGenerators.*
import io.renku.search.solr.client.SearchSolrSpec
import io.renku.search.solr.documents.Project as SolrProject
import io.renku.search.solr.documents.Project.given
import io.renku.search.solr.documents.{
Entity as SolrEntity,
Project as SolrProject,
User as SolrUser
}
import munit.CatsEffectSuite
import scribe.Scribe

Expand All @@ -44,15 +49,36 @@ class SearchApiSpec extends CatsEffectSuite with SearchSolrSpec:
results <- searchApi
.query(mkQuery("matching"))
.map(_.fold(err => fail(s"Calling Search API failed with $err"), identity))
} yield assert(results.items.map(scoreToNone) contains toApiProject(project1))
} yield assert {
results.items.map(scoreToNone) contains toApiEntity(project1)
}
}

test("return Project and User entities"):
withSearchSolrClient().use { client =>
val project = projectDocumentGen("exclusive", "exclusive description").generateOne
val user = SolrUser(project.createdBy, users.FirstName("exclusive").some)
val searchApi = new SearchApiImpl[IO](client)
for {
_ <- client.insert(project :: Nil)
_ <- client.insert(user :: Nil)
results <- searchApi
.query(mkQuery("exclusive"))
.map(_.fold(err => fail(s"Calling Search API failed with $err"), identity))
} yield assert {
toApiEntities(project, user).diff(results.items.map(scoreToNone)).isEmpty
}
}

private def scoreToNone(e: SearchEntity): SearchEntity = e match
case p: Project => p.copy(score = None)
case e: Project => e.copy(score = None)
case e: User => e.copy(score = None)

private def mkQuery(phrase: String): QueryInput =
QueryInput.pageOne(Query.parse(phrase).fold(sys.error, identity))
QueryInput.pageOne(Query.parse(s"Fields $phrase").fold(sys.error, identity))

private def toApiEntities(e: SolrEntity*) = e.map(toApiEntity)

private def toApiProject(p: SolrProject) =
given Transformer[users.Id, User] = (id: users.Id) => User(id)
p.to[Project]
private def toApiEntity(e: SolrEntity) =
given Transformer[users.Id, UserId] = (id: users.Id) => UserId(id)
e.to[SearchEntity]
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,11 @@

package io.renku.search.provision

import cats.effect.{ExitCode, IO, IOApp, Temporal}
import cats.effect.*
import cats.syntax.all.*
import io.renku.logging.LoggingSetup
import io.renku.search.provision.project.ProjectCreatedProvisioning
import io.renku.search.provision.user.UserAddedProvisioning
import io.renku.search.solr.schema.Migrations
import io.renku.solr.client.migration.SchemaMigrator
import scribe.Scribe
Expand All @@ -36,18 +38,31 @@ object Microservice extends IOApp:
config <- loadConfig
_ <- IO(LoggingSetup.doConfigure(config.verbosity))
_ <- runSolrMigrations(config)
_ <- startProvisioning(config)
_ <- startProvisioners(config)
} yield ExitCode.Success

private def startProvisioning(cfg: SearchProvisionConfig): IO[Unit] =
ProjectCreatedProvisioning
.make[IO](cfg.queuesConfig.projectCreated, cfg.redisConfig, cfg.solrConfig)
.evalMap(_.provisioningProcess.start)
.use(_ => IO.never)
.handleErrorWith { err =>
Scribe[IO].error("Starting provisioning failure, retrying", err) >>
Temporal[IO].delayBy(startProvisioning(cfg), cfg.retryOnErrorDelay)
}
private def startProvisioners(cfg: SearchProvisionConfig): IO[Unit] =
List(
"ProjectCreated" -> ProjectCreatedProvisioning
.make[IO](cfg.queuesConfig.projectCreated, cfg.redisConfig, cfg.solrConfig),
"UserAdded" -> UserAddedProvisioning
.make[IO](cfg.queuesConfig.userAdded, cfg.redisConfig, cfg.solrConfig)
).traverse_(startProcess(cfg))

private def startProcess(
cfg: SearchProvisionConfig
): ((String, Resource[IO, SolrProvisioningProcess[IO]])) => IO[Unit] = {
case t @ (name, resource) =>
resource
.evalMap(_.provisioningProcess.start)
.use(_ => IO.never)
.handleErrorWith { err =>
Scribe[IO].error(
s"Starting provisioning process for '$name' failed, retrying",
err
) >> Temporal[IO].delayBy(startProcess(cfg)(t), cfg.retryOnErrorDelay)
}
}

private def runSolrMigrations(cfg: SearchProvisionConfig): IO[Unit] =
SchemaMigrator[IO](cfg.solrConfig)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,19 @@

package io.renku.search.provision

import cats.syntax.all.*
import ciris.{ConfigValue, Effect}
import io.renku.redis.client.QueueName
import io.renku.search.config.ConfigValues

final case class QueuesConfig(
projectCreated: QueueName
projectCreated: QueueName,
userAdded: QueueName
)

object QueuesConfig:
val config: ConfigValue[Effect, QueuesConfig] =
ConfigValues
.eventQueue("projectCreated")
.map(QueuesConfig.apply)
(
ConfigValues.eventQueue("projectCreated"),
ConfigValues.eventQueue("userAdded")
).mapN(QueuesConfig.apply)
Loading

0 comments on commit 024bade

Please sign in to comment.