-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: Downloading avro schema files at build (#20)
- download avro schemas from renku-schemas to generate code - patch it to add a package name here for now - trivially fixing compile errors resulting from new generated code
- Loading branch information
Showing
7 changed files
with
159 additions
and
59 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,129 @@ | ||
| import sbt._ | ||
| import sbt.Keys._ | ||
| import com.github.sbt.git._ | ||
| import sbtavrohugger.SbtAvrohugger | ||
| import sbtavrohugger.SbtAvrohugger.autoImport.* | ||
| import java.io.File | ||
| import org.eclipse.jgit.api.ResetCommand.ResetType | ||
| import avrohugger.filesorter.AvscFileSorter | ||
|
|
||
| object AvroSchemaDownload extends AutoPlugin { | ||
|
|
||
| override def requires = GitPlugin && SbtAvrohugger | ||
|
|
||
| object autoImport { | ||
| val schemaRepository = settingKey[String]("The repository to download") | ||
| val schemaRef = | ||
| settingKey[Option[String]]("The branch, tag or commit sha to checkout") | ||
| val schemaTargetDirectory = settingKey[File]("The directory to download into") | ||
| val schemaDownloadRepository = taskKey[Seq[File]]("Download the repository") | ||
| val schemaClearDownload = taskKey[Unit]("Removes all downloaded files") | ||
| val schemaEnableDownload = | ||
| settingKey[Boolean]("Whether to enable downloading schema repository") | ||
| } | ||
|
|
||
| import autoImport._ | ||
|
|
||
| override def projectSettings = AvroCodeGen.avroHuggerSettings ++ Seq( | ||
| schemaEnableDownload := true, | ||
| schemaRepository := "https://github.com/SwissDataScienceCenter/renku-schema", | ||
| schemaRef := Some("main"), | ||
| schemaTargetDirectory := (Compile / target).value / "renku-avro-schemas", | ||
| schemaClearDownload := { | ||
| val target = schemaTargetDirectory.value | ||
| IO.delete(target) | ||
| }, | ||
| schemaDownloadRepository := { | ||
| val logger = streams.value.log | ||
| val repo = schemaRepository.value | ||
| val refspec = schemaRef.value | ||
| val output = schemaTargetDirectory.value | ||
| val enabled = schemaEnableDownload.value | ||
| if (enabled) { | ||
| synchronizeSchemaFiles(logger, repo, refspec, output) | ||
| } else { | ||
| logger.info("Downloading avro schema files is disabled.") | ||
| } | ||
| Seq(output) | ||
| }, | ||
| Compile / avroSourceDirectories := | ||
| // need to do this custom correct ordering of inputs so the files are | ||
| // evaluated in correct "dependency order" (unfortunately, the plugin doesn't sort it out) | ||
| // must be constant values, because settingKeys are evaluated at project load time | ||
| Seq( | ||
| schemaTargetDirectory.value / "common", | ||
| schemaTargetDirectory.value / "project" | ||
| ), | ||
| Compile / sourceGenerators += Def | ||
| .sequential( | ||
| schemaDownloadRepository, | ||
| Compile / avroScalaGenerate, | ||
| Def.task { | ||
| val out = (Compile / avroScalaSource).value | ||
| val pkg = "io.renku.messages" | ||
| val logger = streams.value.log | ||
| evilHackAddPackage(logger, out, pkg) | ||
| } | ||
| ) | ||
| .taskValue | ||
| ) | ||
|
|
||
| def synchronizeSchemaFiles( | ||
| logger: Logger, | ||
| repo: String, | ||
| refspec: Option[String], | ||
| target: File | ||
| ): Unit = | ||
| if (target.exists) updateRepository(logger, target, refspec) | ||
| else cloneRepository(logger, repo, refspec, target) | ||
|
|
||
| def updateRepository(logger: Logger, base: File, refspec: Option[String]) = { | ||
| logger.info(s"Updating schema repository at $base") | ||
| val git = JGit(base) | ||
| git.porcelain.fetch().call() | ||
| switchBranch(logger, git, refspec) | ||
| } | ||
|
|
||
| def cloneRepository( | ||
| logger: Logger, | ||
| repo: String, | ||
| refspec: Option[String], | ||
| target: File | ||
| ): Unit = { | ||
| logger.info(s"Downloading repository $repo to $target") | ||
| val jgit = JGit.clone(repo, target) | ||
| switchBranch(logger, jgit, refspec) | ||
| } | ||
|
|
||
| def switchBranch(logger: Logger, git: JGit, refspec: Option[String]) = | ||
| refspec match { | ||
| case Some(ref) | ||
| if ref != git.branch && !git.currentTags.contains(ref) && !git.headCommitSha | ||
| .contains(ref) => | ||
| logger.info(s"Changing to $ref") | ||
| val cmd = git.porcelain.reset() | ||
| cmd.setMode(ResetType.HARD) | ||
| cmd.setRef(ref) | ||
| val res = cmd.call() | ||
| logger.info(s"Repository now on $res") | ||
|
|
||
| case _ => () | ||
| } | ||
|
|
||
| def evilHackAddPackage(logger: Logger, dir: File, pkg: String): Seq[File] = { | ||
| val pkgLine = s"package $pkg" | ||
|
|
||
| def prependPackage(file: File) = { | ||
| val content = IO.read(file) | ||
| if (!content.startsWith("package ")) { | ||
| logger.info(s"Add package to: $file") | ||
| IO.write(file, s"$pkgLine;\n\n") // scala & java ... | ||
| IO.append(file, content) | ||
| } | ||
| file | ||
| } | ||
|
|
||
| (dir ** "*.scala").get().map(prependPackage) ++ | ||
| (dir ** "*.java").get().map(prependPackage) | ||
| } | ||
| } |