feat: Downloading avro schema files at build

build.sbt

@@ -27,7 +27,7 @@ releaseVersionBump := sbtrelease.Version.Bump.Minor
 releaseIgnoreUntrackedFiles := true
 releaseTagName := (ThisBuild / version).value
 
-addCommandAlias("ci", "; lint; dbTests; publishLocal")
+addCommandAlias("ci", "; lint; compile; Test/compile; dbTests; publishLocal")
 addCommandAlias(
   "lint",
   "; scalafmtSbtCheck; scalafmtCheckAll;" // Compile/scalafix --check; Test/scalafix --check
@@ -214,7 +214,7 @@ lazy val messages = project
     commons % "compile->compile;test->test",
     avroCodec % "compile->compile;test->test"
   )
-  .enablePlugins(AvroCodeGen, AutomateHeaderPlugin)
+  .enablePlugins(AutomateHeaderPlugin, AvroSchemaDownload)
   .disablePlugins(DbTestPlugin)
 
 lazy val configValues = project

modules/messages/src/test/scala/io/renku/messages/SerializeDeserializeTest.scala

@@ -33,9 +33,13 @@ class SerializeDeserializeTest extends FunSuite {
     val data = ProjectCreated(
       UUID.randomUUID().toString,
       "my-project",
-      "a description for it",
-      None,
-      Instant.now().truncatedTo(ChronoUnit.MILLIS)
+      "slug",
+      Seq.empty,
+      Visibility.PUBLIC,
+      Some("a description for it"),
+      "created-by-me",
+      Instant.now().truncatedTo(ChronoUnit.MILLIS),
+      Seq.empty
     )
     val avro = AvroIO(ProjectCreated.SCHEMA$)
 
@@ -45,21 +49,4 @@ class SerializeDeserializeTest extends FunSuite {
     assertEquals(decoded, List(data))
   }
 
-  test("serialize and deserialize ProjectUpdated") {
-    val data1 = ProjectUpdated(
-      UUID.randomUUID().toString,
-      "my-project",
-      "a description for it",
-      Shapes.CIRCLE,
-      Right(42),
-      Instant.now().truncatedTo(ChronoUnit.MILLIS)
-    )
-    val data2 = data1.copy(index = Left("fourtytwo"))
-    val avro = AvroIO(ProjectUpdated.SCHEMA$)
-
-    val bytes = avro.write(Seq(data1, data2))
-    val decoded = avro.read[ProjectUpdated](bytes)
-
-    assertEquals(decoded, List(data1, data2))
-  }
 }

modules/search-provision/src/main/scala/io/renku/search/provision/SearchProvisioner.scala

@@ -102,7 +102,9 @@ private class SearchProvisionerImpl[F[_]: Async](
     }
 
   private lazy val toSolrDocuments: Seq[ProjectCreated] => Seq[Project] =
-    _.map(pc => Project(id = pc.id, name = pc.name, description = pc.description))
+    _.map(pc =>
+      Project(id = pc.id, name = pc.name, description = pc.description.getOrElse(""))
+    )
 
   private def markProcessedOnFailure(
       message: Message

modules/search-provision/src/test/scala/io/renku/search/provision/SearchProvisionerSpec.scala

@@ -24,7 +24,7 @@ import fs2.Stream
 import fs2.concurrent.SignallingRef
 import io.renku.avro.codec.AvroIO
 import io.renku.avro.codec.encoders.all.given
-import io.renku.messages.ProjectCreated
+import io.renku.messages.{ProjectCreated, Visibility}
 import io.renku.queue.client.Encoding
 import io.renku.redis.client.RedisClientGenerators
 import io.renku.redis.client.RedisClientGenerators.*
@@ -131,10 +131,20 @@ class SearchProvisionerSpec extends CatsEffectSuite with RedisSpec with SearchSo
       name = s"$prefix-${generateString(max = 5).sample.get}"
       desc = s"$prefix ${generateString(max = 10).sample.get}"
       ownerGen = generateString(max = 5).map(prefix + _)
-    yield ProjectCreated(uuid.toString, name, desc, Gen.option(ownerGen).sample.get, now)
+    yield ProjectCreated(
+      uuid.toString,
+      name,
+      "slug",
+      Seq.empty,
+      Visibility.PUBLIC,
+      Some(desc),
+      ownerGen.sample.get,
+      now,
+      Seq.empty
+    )
 
   private def toSolrDocument(created: ProjectCreated): Project =
-    Project(created.id, created.name, created.description)
+    Project(created.id, created.name, created.description.getOrElse(""))
 
   override def munitFixtures: Seq[Fixture[_]] =
     List(withRedisClient, withSearchSolrClient)

project/AvroCodeGen.scala

@@ -6,7 +6,7 @@ import sbtavrohugger.SbtAvrohugger.autoImport.*
 object AvroCodeGen extends AutoPlugin {
   override def requires = SbtAvrohugger
 
-  override def projectSettings = Seq(
+  def avroHuggerSettings = Seq(
     libraryDependencies ++= Dependencies.avro,
     Compile / avroScalaCustomTypes := {
       avrohugger.format.SpecificRecord.defaultTypes.copy(
@@ -17,7 +17,10 @@ object AvroCodeGen extends AutoPlugin {
       avrohugger.format.SpecificRecord.defaultTypes.copy(
         record = avrohugger.types.ScalaCaseClassWithSchema
       )
-    },
+    }
+  )
+
+  override def projectSettings = avroHuggerSettings ++ Seq(
     Compile / sourceGenerators += (Compile / avroScalaGenerate).taskValue
   )
 }

project/AvroSchemaDownload.scala

@@ -0,0 +1,129 @@
+import sbt._
+import sbt.Keys._
+import com.github.sbt.git._
+import sbtavrohugger.SbtAvrohugger
+import sbtavrohugger.SbtAvrohugger.autoImport.*
+import java.io.File
+import org.eclipse.jgit.api.ResetCommand.ResetType
+import avrohugger.filesorter.AvscFileSorter
+
+object AvroSchemaDownload extends AutoPlugin {
+
+  override def requires = GitPlugin && SbtAvrohugger
+
+  object autoImport {
+    val schemaRepository = settingKey[String]("The repository to download")
+    val schemaRef =
+      settingKey[Option[String]]("The branch, tag or commit sha to checkout")
+    val schemaTargetDirectory = settingKey[File]("The directory to download into")
+    val schemaDownloadRepository = taskKey[Seq[File]]("Download the repository")
+    val schemaClearDownload = taskKey[Unit]("Removes all downloaded files")
+    val schemaEnableDownload =
+      settingKey[Boolean]("Whether to enable downloading schema repository")
+  }
+
+  import autoImport._
+
+  override def projectSettings = AvroCodeGen.avroHuggerSettings ++ Seq(
+    schemaEnableDownload := true,
+    schemaRepository := "https://github.com/SwissDataScienceCenter/renku-schema",
+    schemaRef := Some("main"),
+    schemaTargetDirectory := (Compile / target).value / "renku-avro-schemas",
+    schemaClearDownload := {
+      val target = schemaTargetDirectory.value
+      IO.delete(target)
+    },
+    schemaDownloadRepository := {
+      val logger = streams.value.log
+      val repo = schemaRepository.value
+      val refspec = schemaRef.value
+      val output = schemaTargetDirectory.value
+      val enabled = schemaEnableDownload.value
+      if (enabled) {
+        synchronizeSchemaFiles(logger, repo, refspec, output)
+      } else {
+        logger.info("Downloading avro schema files is disabled.")
+      }
+      Seq(output)
+    },
+    Compile / avroSourceDirectories :=
+      // need to do this custom correct ordering of inputs so the files are
+      // evaluated in correct "dependency order" (unfortunately, the plugin doesn't sort it out)
+      // must be constant values, because settingKeys are evaluated at project load time
+      Seq(
+        schemaTargetDirectory.value / "common",
+        schemaTargetDirectory.value / "project"
+      ),
+    Compile / sourceGenerators += Def
+      .sequential(
+        schemaDownloadRepository,
+        Compile / avroScalaGenerate,
+        Def.task {
+          val out = (Compile / avroScalaSource).value
+          val pkg = "io.renku.messages"
+          val logger = streams.value.log
+          evilHackAddPackage(logger, out, pkg)
+        }
+      )
+      .taskValue
+  )
+
+  def synchronizeSchemaFiles(
+      logger: Logger,
+      repo: String,
+      refspec: Option[String],
+      target: File
+  ): Unit =
+    if (target.exists) updateRepository(logger, target, refspec)
+    else cloneRepository(logger, repo, refspec, target)
+
+  def updateRepository(logger: Logger, base: File, refspec: Option[String]) = {
+    logger.info(s"Updating schema repository at $base")
+    val git = JGit(base)
+    git.porcelain.fetch().call()
+    switchBranch(logger, git, refspec)
+  }
+
+  def cloneRepository(
+      logger: Logger,
+      repo: String,
+      refspec: Option[String],
+      target: File
+  ): Unit = {
+    logger.info(s"Downloading repository $repo to $target")
+    val jgit = JGit.clone(repo, target)
+    switchBranch(logger, jgit, refspec)
+  }
+
+  def switchBranch(logger: Logger, git: JGit, refspec: Option[String]) =
+    refspec match {
+      case Some(ref)
+          if ref != git.branch && !git.currentTags.contains(ref) && !git.headCommitSha
+            .contains(ref) =>
+        logger.info(s"Changing to $ref")
+        val cmd = git.porcelain.reset()
+        cmd.setMode(ResetType.HARD)
+        cmd.setRef(ref)
+        val res = cmd.call()
+        logger.info(s"Repository now on $res")
+
+      case _ => ()
+    }
+
+  def evilHackAddPackage(logger: Logger, dir: File, pkg: String): Seq[File] = {
+    val pkgLine = s"package $pkg"
+
+    def prependPackage(file: File) = {
+      val content = IO.read(file)
+      if (!content.startsWith("package ")) {
+        logger.info(s"Add package to: $file")
+        IO.write(file, s"$pkgLine;\n\n") // scala & java ...
+        IO.append(file, content)
+      }
+      file
+    }
+
+    (dir ** "*.scala").get().map(prependPackage) ++
+      (dir ** "*.java").get().map(prependPackage)
+  }
+}