diff --git a/build.sbt b/build.sbt index 3210cdfc..1469b243 100644 --- a/build.sbt +++ b/build.sbt @@ -27,7 +27,7 @@ releaseVersionBump := sbtrelease.Version.Bump.Minor releaseIgnoreUntrackedFiles := true releaseTagName := (ThisBuild / version).value -addCommandAlias("ci", "; lint; dbTests; publishLocal") +addCommandAlias("ci", "; lint; compile; Test/compile; dbTests; publishLocal") addCommandAlias( "lint", "; scalafmtSbtCheck; scalafmtCheckAll;" // Compile/scalafix --check; Test/scalafix --check @@ -214,7 +214,7 @@ lazy val messages = project commons % "compile->compile;test->test", avroCodec % "compile->compile;test->test" ) - .enablePlugins(AvroCodeGen, AutomateHeaderPlugin) + .enablePlugins(AutomateHeaderPlugin, AvroSchemaDownload) .disablePlugins(DbTestPlugin) lazy val configValues = project diff --git a/modules/messages/src/main/avro/messages.avdl b/modules/messages/src/main/avro/messages.avdl deleted file mode 100644 index 04885b52..00000000 --- a/modules/messages/src/main/avro/messages.avdl +++ /dev/null @@ -1,31 +0,0 @@ -@namespace("io.renku.messages") -protocol Messages { - enum Shapes { - SQUARE, TRIANGLE, CIRCLE - } - - /* An example record for a "project-created-event" */ - record ProjectCreated { - string id; - string name; - string description; - string? owner; - timestamp_ms createdAt; - } - - /* A project got updated */ - record ProjectUpdated { - string id; - string name; - string @aliases(["oldDescription"]) description; - Shapes icon; - union { string, int } index; - timestamp_ms updatedAt; - } - - record ProjectDeleted { - string id; - string name; - timestamp_ms deletedAt; - } -} \ No newline at end of file diff --git a/modules/messages/src/test/scala/io/renku/messages/SerializeDeserializeTest.scala b/modules/messages/src/test/scala/io/renku/messages/SerializeDeserializeTest.scala index aabca478..431cd96e 100644 --- a/modules/messages/src/test/scala/io/renku/messages/SerializeDeserializeTest.scala +++ b/modules/messages/src/test/scala/io/renku/messages/SerializeDeserializeTest.scala @@ -33,9 +33,13 @@ class SerializeDeserializeTest extends FunSuite { val data = ProjectCreated( UUID.randomUUID().toString, "my-project", - "a description for it", - None, - Instant.now().truncatedTo(ChronoUnit.MILLIS) + "slug", + Seq.empty, + Visibility.PUBLIC, + Some("a description for it"), + "created-by-me", + Instant.now().truncatedTo(ChronoUnit.MILLIS), + Seq.empty ) val avro = AvroIO(ProjectCreated.SCHEMA$) @@ -45,21 +49,4 @@ class SerializeDeserializeTest extends FunSuite { assertEquals(decoded, List(data)) } - test("serialize and deserialize ProjectUpdated") { - val data1 = ProjectUpdated( - UUID.randomUUID().toString, - "my-project", - "a description for it", - Shapes.CIRCLE, - Right(42), - Instant.now().truncatedTo(ChronoUnit.MILLIS) - ) - val data2 = data1.copy(index = Left("fourtytwo")) - val avro = AvroIO(ProjectUpdated.SCHEMA$) - - val bytes = avro.write(Seq(data1, data2)) - val decoded = avro.read[ProjectUpdated](bytes) - - assertEquals(decoded, List(data1, data2)) - } } diff --git a/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisioner.scala b/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisioner.scala index 1a6d2c9b..a3a012fc 100644 --- a/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisioner.scala +++ b/modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisioner.scala @@ -102,7 +102,9 @@ private class SearchProvisionerImpl[F[_]: Async]( } private lazy val toSolrDocuments: Seq[ProjectCreated] => Seq[Project] = - _.map(pc => Project(id = pc.id, name = pc.name, description = pc.description)) + _.map(pc => + Project(id = pc.id, name = pc.name, description = pc.description.getOrElse("")) + ) private def markProcessedOnFailure( message: Message diff --git a/modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala b/modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala index 91b64e36..2f7dd501 100644 --- a/modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala +++ b/modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala @@ -24,7 +24,7 @@ import fs2.Stream import fs2.concurrent.SignallingRef import io.renku.avro.codec.AvroIO import io.renku.avro.codec.encoders.all.given -import io.renku.messages.ProjectCreated +import io.renku.messages.{ProjectCreated, Visibility} import io.renku.queue.client.Encoding import io.renku.redis.client.RedisClientGenerators import io.renku.redis.client.RedisClientGenerators.* @@ -131,10 +131,20 @@ class SearchProvisionerSpec extends CatsEffectSuite with RedisSpec with SearchSo name = s"$prefix-${generateString(max = 5).sample.get}" desc = s"$prefix ${generateString(max = 10).sample.get}" ownerGen = generateString(max = 5).map(prefix + _) - yield ProjectCreated(uuid.toString, name, desc, Gen.option(ownerGen).sample.get, now) + yield ProjectCreated( + uuid.toString, + name, + "slug", + Seq.empty, + Visibility.PUBLIC, + Some(desc), + ownerGen.sample.get, + now, + Seq.empty + ) private def toSolrDocument(created: ProjectCreated): Project = - Project(created.id, created.name, created.description) + Project(created.id, created.name, created.description.getOrElse("")) override def munitFixtures: Seq[Fixture[_]] = List(withRedisClient, withSearchSolrClient) diff --git a/project/AvroCodeGen.scala b/project/AvroCodeGen.scala index 35d8b874..3960b3cb 100644 --- a/project/AvroCodeGen.scala +++ b/project/AvroCodeGen.scala @@ -6,7 +6,7 @@ import sbtavrohugger.SbtAvrohugger.autoImport.* object AvroCodeGen extends AutoPlugin { override def requires = SbtAvrohugger - override def projectSettings = Seq( + def avroHuggerSettings = Seq( libraryDependencies ++= Dependencies.avro, Compile / avroScalaCustomTypes := { avrohugger.format.SpecificRecord.defaultTypes.copy( @@ -17,7 +17,10 @@ object AvroCodeGen extends AutoPlugin { avrohugger.format.SpecificRecord.defaultTypes.copy( record = avrohugger.types.ScalaCaseClassWithSchema ) - }, + } + ) + + override def projectSettings = avroHuggerSettings ++ Seq( Compile / sourceGenerators += (Compile / avroScalaGenerate).taskValue ) } diff --git a/project/AvroSchemaDownload.scala b/project/AvroSchemaDownload.scala new file mode 100644 index 00000000..285cdb65 --- /dev/null +++ b/project/AvroSchemaDownload.scala @@ -0,0 +1,129 @@ +import sbt._ +import sbt.Keys._ +import com.github.sbt.git._ +import sbtavrohugger.SbtAvrohugger +import sbtavrohugger.SbtAvrohugger.autoImport.* +import java.io.File +import org.eclipse.jgit.api.ResetCommand.ResetType +import avrohugger.filesorter.AvscFileSorter + +object AvroSchemaDownload extends AutoPlugin { + + override def requires = GitPlugin && SbtAvrohugger + + object autoImport { + val schemaRepository = settingKey[String]("The repository to download") + val schemaRef = + settingKey[Option[String]]("The branch, tag or commit sha to checkout") + val schemaTargetDirectory = settingKey[File]("The directory to download into") + val schemaDownloadRepository = taskKey[Seq[File]]("Download the repository") + val schemaClearDownload = taskKey[Unit]("Removes all downloaded files") + val schemaEnableDownload = + settingKey[Boolean]("Whether to enable downloading schema repository") + } + + import autoImport._ + + override def projectSettings = AvroCodeGen.avroHuggerSettings ++ Seq( + schemaEnableDownload := true, + schemaRepository := "https://github.com/SwissDataScienceCenter/renku-schema", + schemaRef := Some("main"), + schemaTargetDirectory := (Compile / target).value / "renku-avro-schemas", + schemaClearDownload := { + val target = schemaTargetDirectory.value + IO.delete(target) + }, + schemaDownloadRepository := { + val logger = streams.value.log + val repo = schemaRepository.value + val refspec = schemaRef.value + val output = schemaTargetDirectory.value + val enabled = schemaEnableDownload.value + if (enabled) { + synchronizeSchemaFiles(logger, repo, refspec, output) + } else { + logger.info("Downloading avro schema files is disabled.") + } + Seq(output) + }, + Compile / avroSourceDirectories := + // need to do this custom correct ordering of inputs so the files are + // evaluated in correct "dependency order" (unfortunately, the plugin doesn't sort it out) + // must be constant values, because settingKeys are evaluated at project load time + Seq( + schemaTargetDirectory.value / "common", + schemaTargetDirectory.value / "project" + ), + Compile / sourceGenerators += Def + .sequential( + schemaDownloadRepository, + Compile / avroScalaGenerate, + Def.task { + val out = (Compile / avroScalaSource).value + val pkg = "io.renku.messages" + val logger = streams.value.log + evilHackAddPackage(logger, out, pkg) + } + ) + .taskValue + ) + + def synchronizeSchemaFiles( + logger: Logger, + repo: String, + refspec: Option[String], + target: File + ): Unit = + if (target.exists) updateRepository(logger, target, refspec) + else cloneRepository(logger, repo, refspec, target) + + def updateRepository(logger: Logger, base: File, refspec: Option[String]) = { + logger.info(s"Updating schema repository at $base") + val git = JGit(base) + git.porcelain.fetch().call() + switchBranch(logger, git, refspec) + } + + def cloneRepository( + logger: Logger, + repo: String, + refspec: Option[String], + target: File + ): Unit = { + logger.info(s"Downloading repository $repo to $target") + val jgit = JGit.clone(repo, target) + switchBranch(logger, jgit, refspec) + } + + def switchBranch(logger: Logger, git: JGit, refspec: Option[String]) = + refspec match { + case Some(ref) + if ref != git.branch && !git.currentTags.contains(ref) && !git.headCommitSha + .contains(ref) => + logger.info(s"Changing to $ref") + val cmd = git.porcelain.reset() + cmd.setMode(ResetType.HARD) + cmd.setRef(ref) + val res = cmd.call() + logger.info(s"Repository now on $res") + + case _ => () + } + + def evilHackAddPackage(logger: Logger, dir: File, pkg: String): Seq[File] = { + val pkgLine = s"package $pkg" + + def prependPackage(file: File) = { + val content = IO.read(file) + if (!content.startsWith("package ")) { + logger.info(s"Add package to: $file") + IO.write(file, s"$pkgLine;\n\n") // scala & java ... + IO.append(file, content) + } + file + } + + (dir ** "*.scala").get().map(prependPackage) ++ + (dir ** "*.java").get().map(prependPackage) + } +}