From 6a52f0304373661604d290a56540f8885d86cd49 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Mon, 16 Dec 2024 17:23:42 +0100 Subject: [PATCH 01/13] Overhaul transitive module handling in dep resolution This changes the way transitive dependencies, but also transitive BOM dependencies and dependency management, are passed to coursier. Before this commit, Mill was walking its own module graph, and dependencies / BOM dependencies / dep management were manually aggregated beforehand. During dependency resolution, coursier also does such aggregations. Having both Mill and coursier handle that concern could lead to discrepancies. So this commit passes non-processed dependencies / BOM dependencies / dep management as-is to coursier, and lets coursier handle / compose those. --- .../src/mill/bsp/worker/MillBuildServer.scala | 17 +- .../src/mill/contrib/bloop/BloopImpl.scala | 6 +- contrib/playlib/src/mill/playlib/Static.scala | 6 +- .../contrib/scalapblib/ScalaPBModule.scala | 7 +- .../contrib/scoverage/ScoverageModule.scala | 4 +- .../builtins/1-builtin-commands/build.mill | 1 - idea/src/mill/idea/GenIdeaImpl.scala | 5 +- .../src/DocAnnotationsTests.scala | 7 +- main/util/src/mill/util/CoursierSupport.scala | 89 +++-- .../mill/javalib/revapi/RevapiModule.scala | 2 +- .../src/mill/scalalib/CoursierModule.scala | 23 +- scalalib/src/mill/scalalib/JavaModule.scala | 311 ++++++++++++------ .../src/mill/scalalib/PublishModule.scala | 140 ++++++-- scalalib/src/mill/scalalib/ScalaModule.scala | 5 +- .../src/mill/scalalib/CrossVersionTests.scala | 117 ++++--- .../ScalaMultiModuleClasspathsTests.scala | 51 ++- .../scalanativelib/ScalaNativeModule.scala | 41 +-- .../mill/scalanativelib/ExclusionsTests.scala | 4 +- 18 files changed, 566 insertions(+), 270 deletions(-) diff --git a/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala b/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala index a062ea06fe8..85c64c99e36 100644 --- a/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala +++ b/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala @@ -296,7 +296,10 @@ private class MillBuildServer( Task.Anon { ( m.defaultResolver().resolveDeps( - m.transitiveCompileIvyDeps() ++ m.transitiveIvyDeps(), + Seq( + m.coursierDependency.withConfiguration(coursier.core.Configuration.provided), + m.coursierDependency + ), sources = true ), m.unmanagedClasspath(), @@ -332,7 +335,13 @@ private class MillBuildServer( hint = "buildTargetDependencyModules", targetIds = _ => params.getTargets.asScala.toSeq, tasks = { case m: JavaModule => - Task.Anon { (m.transitiveCompileIvyDeps(), m.transitiveIvyDeps(), m.unmanagedClasspath()) } + Task.Anon { + ( + m.compileIvyDeps(), + m.ivyDeps(), + m.unmanagedClasspath() + ) + } } ) { case ( @@ -340,9 +349,9 @@ private class MillBuildServer( state, id, m: JavaModule, - (transitiveCompileIvyDeps, transitiveIvyDeps, unmanagedClasspath) + (compileIvyDeps, ivyDeps, unmanagedClasspath) ) => - val ivy = transitiveCompileIvyDeps ++ transitiveIvyDeps + val ivy = compileIvyDeps ++ ivyDeps val deps = ivy.map { dep => // TODO: add data with "maven" data kind using a ... // MavenDependencyModule diff --git a/contrib/bloop/src/mill/contrib/bloop/BloopImpl.scala b/contrib/bloop/src/mill/contrib/bloop/BloopImpl.scala index 08861925e3b..e58d97f7c36 100644 --- a/contrib/bloop/src/mill/contrib/bloop/BloopImpl.scala +++ b/contrib/bloop/src/mill/contrib/bloop/BloopImpl.scala @@ -392,8 +392,10 @@ class BloopImpl(evs: () => Seq[Evaluator], wd: os.Path) extends ExternalModule { val bloopResolution: Task[BloopConfig.Resolution] = Task.Anon { val repos = module.repositoriesTask() // same as input of resolvedIvyDeps - val allIvyDeps = module.transitiveIvyDeps() ++ module.transitiveCompileIvyDeps() - val coursierDeps = allIvyDeps.map(_.dep).toList + val coursierDeps = Seq( + module.coursierDependency.withConfiguration(coursier.core.Configuration.provided), + module.coursierDependency + ) BloopConfig.Resolution(artifacts(repos, coursierDeps)) } diff --git a/contrib/playlib/src/mill/playlib/Static.scala b/contrib/playlib/src/mill/playlib/Static.scala index 1ddccd41158..310740ee333 100644 --- a/contrib/playlib/src/mill/playlib/Static.scala +++ b/contrib/playlib/src/mill/playlib/Static.scala @@ -44,10 +44,12 @@ trait Static extends ScalaModule { } /** - * webjar dependencies - created from transitive ivy deps + * webjar dependencies - created from ivy deps */ def webJarDeps = Task { - transitiveIvyDeps().filter(_.dep.module.organization.value == "org.webjars") + ivyDeps() + .filter(_.dep.module.organization.value == "org.webjars") + .map(bindDependency()) } /** diff --git a/contrib/scalapblib/src/mill/contrib/scalapblib/ScalaPBModule.scala b/contrib/scalapblib/src/mill/contrib/scalapblib/ScalaPBModule.scala index e28fa336f02..80580e8447f 100644 --- a/contrib/scalapblib/src/mill/contrib/scalapblib/ScalaPBModule.scala +++ b/contrib/scalapblib/src/mill/contrib/scalapblib/ScalaPBModule.scala @@ -86,7 +86,12 @@ trait ScalaPBModule extends ScalaModule { else Task { Seq.empty[PathRef] } def scalaPBProtoClasspath: T[Agg[PathRef]] = Task { - defaultResolver().resolveDeps(transitiveCompileIvyDeps() ++ transitiveIvyDeps()) + defaultResolver().resolveDeps( + Seq( + coursierDependency.withConfiguration(coursier.core.Configuration.provided), + coursierDependency + ) + ) } def scalaPBUnpackProto: T[PathRef] = Task { diff --git a/contrib/scoverage/src/mill/contrib/scoverage/ScoverageModule.scala b/contrib/scoverage/src/mill/contrib/scoverage/ScoverageModule.scala index 121899d8bb3..fc3f819e14a 100644 --- a/contrib/scoverage/src/mill/contrib/scoverage/ScoverageModule.scala +++ b/contrib/scoverage/src/mill/contrib/scoverage/ScoverageModule.scala @@ -159,7 +159,9 @@ trait ScoverageModule extends ScalaModule { outer: ScalaModule => override def sources: T[Seq[PathRef]] = Task { outer.sources() } override def resources: T[Seq[PathRef]] = Task { outer.resources() } override def scalaVersion = Task { outer.scalaVersion() } - override def repositoriesTask: Task[Seq[Repository]] = Task.Anon { outer.repositoriesTask() } + override def repositoriesTask: Task[Seq[Repository]] = Task.Anon { + internalRepositories() ++ outer.repositoriesTask() + } override def compileIvyDeps: T[Agg[Dep]] = Task { outer.compileIvyDeps() } override def ivyDeps: T[Agg[Dep]] = Task { outer.ivyDeps() ++ outer.scoverageRuntimeDeps() } diff --git a/example/cli/builtins/1-builtin-commands/build.mill b/example/cli/builtins/1-builtin-commands/build.mill index b0a306b83fb..9689c346fc4 100644 --- a/example/cli/builtins/1-builtin-commands/build.mill +++ b/example/cli/builtins/1-builtin-commands/build.mill @@ -248,7 +248,6 @@ foo.unmanagedClasspath foo.compileIvyDeps ... foo.ivyDeps -foo.transitiveIvyDeps foo.compileClasspath */ diff --git a/idea/src/mill/idea/GenIdeaImpl.scala b/idea/src/mill/idea/GenIdeaImpl.scala index d1bec365b89..b27fa643e26 100755 --- a/idea/src/mill/idea/GenIdeaImpl.scala +++ b/idea/src/mill/idea/GenIdeaImpl.scala @@ -132,7 +132,10 @@ case class GenIdeaImpl( // same as input of resolvedIvyDeps val allIvyDeps = Task.Anon { - mod.transitiveIvyDeps() ++ mod.transitiveCompileIvyDeps() + Agg( + mod.coursierDependency, + mod.coursierDependency.withConfiguration(coursier.core.Configuration.provided) + ).map(BoundDep(_, force = false)) } val scalaCompilerClasspath = mod match { diff --git a/integration/feature/docannotations/src/DocAnnotationsTests.scala b/integration/feature/docannotations/src/DocAnnotationsTests.scala index 49524852d22..132842a9150 100644 --- a/integration/feature/docannotations/src/DocAnnotationsTests.scala +++ b/integration/feature/docannotations/src/DocAnnotationsTests.scala @@ -102,7 +102,12 @@ object DocAnnotationsTests extends UtestIntegrationTestSuite { | tree. | |Inputs: - | core.transitiveIvyDeps + | core.bomIvyDeps + | core.mandatoryIvyDeps + | core.ivyDeps + | core.compileIvyDeps + | core.runIvyDeps + | core.depManagement |""".stripMargin, ivyDepsTree ) diff --git a/main/util/src/mill/util/CoursierSupport.scala b/main/util/src/mill/util/CoursierSupport.scala index 083d7f2216c..34c9e08aa18 100644 --- a/main/util/src/mill/util/CoursierSupport.scala +++ b/main/util/src/mill/util/CoursierSupport.scala @@ -1,13 +1,13 @@ package mill.util import coursier.cache.{CacheLogger, FileCache} -import coursier.core.BomDependency +import coursier.core.{ArtifactSource, BomDependency, Extension, Info, Module, Project, Publication} import coursier.error.FetchError.DownloadingArtifacts import coursier.error.ResolutionError.CantDownloadModule import coursier.params.ResolutionParams import coursier.parse.RepositoryParser import coursier.jvm.{JvmCache, JvmChannel, JvmIndex, JavaHome} -import coursier.util.Task +import coursier.util.{Artifact, EitherT, Monad, Task} import coursier.{Artifacts, Classifier, Dependency, Repository, Resolution, Resolve, Type} import mill.api.Loose.Agg import mill.api.{Ctx, PathRef, Result} @@ -32,18 +32,70 @@ trait CoursierSupport { ctx.fold(cache)(c => cache.withLogger(new TickerResolutionLogger(c))) } - private def isLocalTestDep(dep: Dependency): Option[Seq[PathRef]] = { - val org = dep.module.organization.value - val name = dep.module.name.value - val classpathKey = s"$org-$name" + private class TestOverridesRepo(root: os.ResourcePath) extends Repository { - val classpathResourceText = - try Some(os.read( - os.resource(getClass.getClassLoader) / "mill/local-test-overrides" / classpathKey - )) - catch { case e: os.ResourceNotFoundException => None } + private def listFor(mod: Module): Either[os.ResourceNotFoundException, String] = { + val classpathKey = s"${mod.organization.value}-${mod.name.value}" + val entryPath = root / classpathKey - classpathResourceText.map(_.linesIterator.map(s => PathRef(os.Path(s))).toSeq) + try Right(os.read(entryPath)) + catch { + case e: os.ResourceNotFoundException => + Left(e) + } + } + + def find[F[_]: Monad]( + module: Module, + version: String, + fetch: Repository.Fetch[F] + ): EitherT[F, String, (ArtifactSource, Project)] = + EitherT.fromEither[F] { + listFor(module) + .left.map(e => s"No test override found at ${e.path}") + .map { _ => + val proj = Project( + module, + version, + dependencies = Nil, + configurations = Map.empty, + parent = None, + dependencyManagement = Nil, + properties = Nil, + profiles = Nil, + versions = None, + snapshotVersioning = None, + packagingOpt = None, + relocated = false, + actualVersionOpt = None, + publications = Nil, + info = Info.empty + ) + (this, proj) + } + } + + def artifacts( + dependency: Dependency, + project: Project, + overrideClassifiers: Option[Seq[Classifier]] + ): Seq[(Publication, Artifact)] = + listFor(project.module) + .toTry.get + .linesIterator + .map(os.Path(_)) + .filter(os.exists) + .map { path => + val pub = Publication( + if (path.last.endsWith(".jar")) path.last.stripSuffix(".jar") else path.last, + Type.jar, + Extension.jar, + Classifier.empty + ) + val art = Artifact(path.toNIO.toUri.toASCIIString) + (pub, art) + } + .toSeq } /** @@ -66,12 +118,9 @@ trait CoursierSupport { artifactTypes: Option[Set[Type]] = None, resolutionParams: ResolutionParams = ResolutionParams() ): Result[Agg[PathRef]] = { - val (localTestDeps, remoteDeps) = - deps.iterator.toSeq.partitionMap(d => isLocalTestDep(d).toLeft(d)) - val resolutionRes = resolveDependenciesMetadataSafe( repositories, - remoteDeps, + deps, force, mapDependencies, customizer, @@ -110,7 +159,7 @@ trait CoursierSupport { .map(os.Path(_)) .filter(resolveFilter) .map(PathRef(_, quick = true)) - ) ++ localTestDeps.flatten + ) ) } } @@ -260,7 +309,6 @@ trait CoursierSupport { val rootDeps = deps.iterator .map(d => mapDependencies.fold(d)(_.apply(d))) - .filter(dep => isLocalTestDep(dep).isEmpty) .toSeq val forceVersions = force.iterator @@ -273,10 +321,13 @@ trait CoursierSupport { val resolutionParams0 = resolutionParams .addForceVersion(forceVersions.toSeq: _*) + val testOverridesRepo = + new TestOverridesRepo(os.resource(getClass.getClassLoader) / "mill/local-test-overrides") + val resolve = Resolve() .withCache(coursierCache0) .withDependencies(rootDeps) - .withRepositories(repositories) + .withRepositories(testOverridesRepo +: repositories) .withResolutionParams(resolutionParams0) .withMapDependenciesOpt(mapDependencies) .withBoms(boms.toSeq) diff --git a/scalalib/src/mill/javalib/revapi/RevapiModule.scala b/scalalib/src/mill/javalib/revapi/RevapiModule.scala index 6cf0de51f71..e04928913dd 100644 --- a/scalalib/src/mill/javalib/revapi/RevapiModule.scala +++ b/scalalib/src/mill/javalib/revapi/RevapiModule.scala @@ -82,7 +82,7 @@ trait RevapiModule extends PublishModule { Agg(jar()) ++ T.traverse(recursiveModuleDeps)(_.jar)() ++ defaultResolver().resolveDeps( - transitiveIvyDeps(), + Seq(coursierDependency), artifactTypes = Some(revapiArtifactTypes()) ) } diff --git a/scalalib/src/mill/scalalib/CoursierModule.scala b/scalalib/src/mill/scalalib/CoursierModule.scala index 196123074ed..eeccca0a3d9 100644 --- a/scalalib/src/mill/scalalib/CoursierModule.scala +++ b/scalalib/src/mill/scalalib/CoursierModule.scala @@ -1,7 +1,7 @@ package mill.scalalib import coursier.cache.FileCache -import coursier.core.{BomDependency, Resolution} +import coursier.core.{BomDependency, DependencyManagement, Resolution} import coursier.params.ResolutionParams import coursier.{Dependency, Repository, Resolve, Type} import mill.define.Task @@ -84,6 +84,8 @@ trait CoursierModule extends mill.Module { */ def mapDependencies: Task[Dependency => Dependency] = Task.Anon { (d: Dependency) => d } + def internalRepositories: Task[Seq[Repository]] = Task.Anon(Nil) + /** * The repositories used to resolved dependencies with [[resolveDeps()]]. */ @@ -93,7 +95,7 @@ trait CoursierModule extends mill.Module { resolve.finalRepositories.future()(resolve.cache.ec), Duration.Inf ) - repos + internalRepositories() ++ repos } /** @@ -249,9 +251,10 @@ object CoursierModule { deps: IterableOnce[T], resolutionParams: ResolutionParams = ResolutionParams(), boms: IterableOnce[BomDependency] = Nil - ): (Seq[coursier.core.Dependency], coursier.core.DependencyManagement.Map) = { + ): Seq[(Seq[coursier.core.Dependency], DependencyManagement.Map)] = { val deps0 = deps .map(implicitly[CoursierModule.Resolvable[T]].bind(_, bind)) + .iterator.toSeq val boms0 = boms.toSeq val res = Lib.resolveDependenciesMetadataSafe( repositories = repositories, @@ -264,7 +267,15 @@ object CoursierModule { boms = boms0 ).getOrThrow - (res.processedRootDependencies, res.bomDepMgmt) + deps0.map { dep => + ( + res.finalDependenciesCache.getOrElse(dep.dep, ???), + DependencyManagement.addDependencies( + Map.empty, + res.projectCache.get(dep.dep.moduleVersion).getOrElse(???)._2.dependencyManagement + ) + ) + } } } @@ -277,4 +288,8 @@ object CoursierModule { implicit case object ResolvableBoundDep extends Resolvable[BoundDep] { def bind(t: BoundDep, bind: Dep => BoundDep): BoundDep = t } + implicit case object ResolvableCoursierDep extends Resolvable[coursier.core.Dependency] { + def bind(t: coursier.core.Dependency, bind: Dep => BoundDep): BoundDep = + BoundDep(t, force = false) + } } diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 861199e9f4b..62c78584cae 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -1,10 +1,11 @@ package mill package scalalib +import coursier.{core => cs} import coursier.core.{BomDependency, Configuration, DependencyManagement, Resolution} import coursier.parse.JavaOrScalaModule import coursier.parse.ModuleParser -import coursier.util.ModuleMatcher +import coursier.util.{EitherT, ModuleMatcher, Monad} import coursier.{Repository, Type} import mainargs.{Flag, arg} import mill.Agg @@ -44,7 +45,9 @@ trait JavaModule override def resources = super[JavaModule].resources override def moduleDeps: Seq[JavaModule] = Seq(outer) - override def repositoriesTask: Task[Seq[Repository]] = outer.repositoriesTask + override def repositoriesTask: Task[Seq[Repository]] = Task.Anon { + internalRepositories() ++ outer.repositoriesTask() + } override def resolutionCustomizer: Task[Option[coursier.Resolution => coursier.Resolution]] = outer.resolutionCustomizer override def javacOptions: T[Seq[String]] = Task { outer.javacOptions() } @@ -209,41 +212,6 @@ trait JavaModule */ def depManagement: T[Agg[Dep]] = Task { Agg.empty[Dep] } - private def addBoms( - bomIvyDeps: Seq[coursier.core.BomDependency], - depMgmt: Seq[(DependencyManagement.Key, DependencyManagement.Values)], - overrideVersions: Boolean - ): coursier.core.Dependency => coursier.core.Dependency = { - val depMgmtMap = DependencyManagement.add(Map.empty, depMgmt) - dep => - val depMgmtKey = DependencyManagement.Key( - dep.module.organization, - dep.module.name, - coursier.core.Type.jar, - dep.publication.classifier - ) - val versionOverrideOpt = - if (dep.version.isEmpty) - depMgmtMap.get(depMgmtKey).map(_.version).filter(_.nonEmpty) - else None - val extraExclusions = depMgmtMap.get(depMgmtKey).map(_.minimizedExclusions) - dep - // add BOM coordinates - coursier will handle the rest - .addBomDependencies( - if (overrideVersions) bomIvyDeps.map(_.withForceOverrideVersions(overrideVersions)) - else bomIvyDeps - ) - // add dependency management ourselves: - // - overrides meant to apply to transitive dependencies - // - fill version if it's empty - // - add extra exclusions from dependency management - .addOverrides(depMgmt) - .withVersion(versionOverrideOpt.getOrElse(dep.version)) - .withMinimizedExclusions( - extraExclusions.fold(dep.minimizedExclusions)(dep.minimizedExclusions.join(_)) - ) - } - /** * Data from depManagement, converted to a type ready to be passed to coursier * for dependency resolution @@ -453,60 +421,140 @@ trait JavaModule def unmanagedClasspath: T[Agg[PathRef]] = Task { Agg.empty[PathRef] } /** - * Returns a function adding BOM and dependency management details of - * this module to a `coursier.core.Dependency` + * The `coursier.Dependency` to use to refer to this module */ - def processDependency( - overrideVersions: Boolean = false - ): Task[coursier.core.Dependency => coursier.core.Dependency] = Task.Anon { - val bomDeps0 = allBomDeps().toSeq.map(_.withConfig(Configuration.compile)) - val depMgmt = processedDependencyManagement( - depManagement().toSeq.map(bindDependency()).map(_.dep) + def coursierDependency: cs.Dependency = + // this is a simple def, and not a Task, as this is simple enough and needed in places + // where eval'ing a Task would be impractical or not allowed + cs.Dependency( + cs.Module( + coursier.core.Organization("mill-internal"), + coursier.core.ModuleName(millModuleSegments.parts.mkString("-")), + Map.empty + ), + "0+mill-internal" + ).withConfiguration(cs.Configuration.compile) + + /** + * The `coursier.Project` corresponding to this `JavaModule`. + * + * This provides details about this module to the coursier resolver (details such as + * dependencies, BOM dependencies, dependency management, etc.). Beyond more general + * resolution parameters (such as artifact types, etc.), this should be the only way + * we provide details about this module to coursier. + */ + def coursierProject: Task[cs.Project] = Task.Anon { + + // FIXME That's coursier.maven.MavenRepositoryInternal.defaultConfigurations (private for now) + // plus hack for provided + val mavenScopes = Map( + cs.Configuration.compile -> Seq.empty, + cs.Configuration.runtime -> Seq(cs.Configuration.compile), + cs.Configuration.default -> Seq(cs.Configuration.runtime), + cs.Configuration.test -> Seq(cs.Configuration.runtime), + // hack, so that depending on `coursierDependency.withConfiguration(Configuration.provided)` + // pulls the compile-scope of our provided dependencies (rather than nothing) + cs.Configuration.provided -> Seq(cs.Configuration.compile) ) - addBoms(bomDeps0, depMgmt, overrideVersions = overrideVersions) + val internalDependencies = + moduleDepsChecked.flatMap { modDep => + Seq( + (cs.Configuration.compile, modDep.coursierDependency), + ( + cs.Configuration.runtime, + modDep.coursierDependency.withConfiguration(cs.Configuration.runtime) + ) + ) + } ++ + compileModuleDepsChecked.map { modDep => + (cs.Configuration.provided, modDep.coursierDependency) + } ++ + runModuleDepsChecked.map { modDep => + ( + cs.Configuration.runtime, + modDep.coursierDependency.withConfiguration(cs.Configuration.runtime) + ) + } + + val dependencies = + (mandatoryIvyDeps() ++ ivyDeps()).map(bindDependency()).map(_.dep).iterator.toSeq.flatMap { + dep => + Seq( + (cs.Configuration.compile, dep), + (cs.Configuration.runtime, dep.withConfiguration(cs.Configuration.runtime)) + ) + } ++ + compileIvyDeps().map(bindDependency()).map(_.dep).map { dep => + (cs.Configuration.provided, dep) + } ++ + runIvyDeps().map(bindDependency()).map(_.dep).map { dep => + ( + cs.Configuration.runtime, + dep.withConfiguration( + if (dep.configuration.isEmpty) cs.Configuration.runtime else dep.configuration + ) + ) + } ++ + allBomDeps().map { bomDep => + val dep = + cs.Dependency(bomDep.module, bomDep.version).withConfiguration(bomDep.config) + (cs.Configuration.`import`, dep) + } + + val depMgmt = + processedDependencyManagement( + depManagement().iterator.toSeq.map(bindDependency()).map(_.dep) + ).map { + case (key, values) => + (cs.Configuration.compile, values.fakeDependency(key)) + } + + cs.Project( + module = coursierDependency.module, + version = coursierDependency.version, + dependencies = internalDependencies ++ dependencies, + configurations = mavenScopes, + parent = None, + dependencyManagement = depMgmt, + properties = Nil, + profiles = Nil, + versions = None, + snapshotVersioning = None, + packagingOpt = None, + relocated = false, + actualVersionOpt = None, + publications = Nil, + info = coursier.core.Info.empty + ) } /** - * The Ivy dependencies of this module, with BOM and dependency management details - * added to them. This should be used when propagating the dependencies transitively - * to other modules. + * Coursier project of this module and those of all its transitive module dependencies */ - def processedIvyDeps: Task[Agg[BoundDep]] = Task.Anon { - val processDependency0 = processDependency()() - allIvyDeps().map(bindDependency()).map { dep => - dep.copy(dep = processDependency0(dep.dep)) - } + def transitiveCoursierProjects: Task[Seq[cs.Project]] = Task.Anon { + val projects = Seq(coursierProject()) ++ + T.traverse(compileModuleDepsChecked)(_.transitiveCoursierProjects)().flatten ++ + T.traverse(moduleDepsChecked)(_.transitiveCoursierProjects)().flatten ++ + T.traverse(runModuleDepsChecked)(_.transitiveCoursierProjects)().flatten + projects.distinct } /** - * The transitive ivy dependencies of this module and all it's upstream modules. - * This is calculated from [[ivyDeps]], [[mandatoryIvyDeps]] and recursively from [[moduleDeps]]. + * The repository that knows about this project itself and its module dependencies */ - def transitiveIvyDeps: T[Agg[BoundDep]] = Task { - val processDependency0 = processDependency(overrideVersions = true)() - processedIvyDeps() ++ - T.traverse(moduleDepsChecked)(_.transitiveIvyDeps)().flatten.map { dep => - dep.copy(dep = processDependency0(dep.dep)) - } - } - - /** The compile-only transitive ivy dependencies of this module and all it's upstream compile-only modules. */ - def transitiveCompileIvyDeps: T[Agg[BoundDep]] = Task { - // We never include compile-only dependencies transitively, but we must include normal transitive dependencies! - compileIvyDeps().map(bindDependency()) ++ - T.traverse(compileModuleDepsChecked)(_.transitiveIvyDeps)().flatten + def internalDependenciesRepository: Task[cs.Repository] = Task.Anon { + JavaModule.InternalRepo(transitiveCoursierProjects()) } /** - * The transitive run ivy dependencies of this module and all it's upstream modules. - * This is calculated from [[runIvyDeps]], [[mandatoryIvyDeps]] and recursively from [[moduleDeps]]. + * Mill internal repositories to be used during dependency resolution + * + * These are not meant to be modified by Mill users, unless you really know what you're + * doing. */ - def transitiveRunIvyDeps: T[Agg[BoundDep]] = Task { - runIvyDeps().map(bindDependency()) ++ - T.traverse(moduleDepsChecked)(_.transitiveRunIvyDeps)().flatten ++ - T.traverse(runModuleDepsChecked)(_.transitiveIvyDeps)().flatten ++ - T.traverse(runModuleDepsChecked)(_.transitiveRunIvyDeps)().flatten + def internalRepositories: Task[Seq[cs.Repository]] = Task.Anon { + super.internalRepositories() ++ Seq(internalDependenciesRepository()) } /** @@ -773,11 +821,17 @@ trait JavaModule } /** - * Resolved dependencies based on [[transitiveIvyDeps]] and [[transitiveCompileIvyDeps]]. + * Resolved dependencies */ def resolvedIvyDeps: T[Agg[PathRef]] = Task { defaultResolver().resolveDeps( - transitiveCompileIvyDeps() ++ transitiveIvyDeps(), + Seq( + BoundDep( + coursierDependency.withConfiguration(cs.Configuration.provided), + force = false + ), + BoundDep(coursierDependency, force = false) + ), artifactTypes = Some(artifactTypes()), resolutionParamsMapOpt = Some(_.withDefaultConfiguration(coursier.core.Configuration.compile)) ) @@ -793,8 +847,14 @@ trait JavaModule def resolvedRunIvyDeps: T[Agg[PathRef]] = Task { defaultResolver().resolveDeps( - transitiveRunIvyDeps() ++ transitiveIvyDeps(), - artifactTypes = Some(artifactTypes()) + Seq( + BoundDep( + coursierDependency.withConfiguration(cs.Configuration.runtime), + force = false + ) + ), + artifactTypes = Some(artifactTypes()), + resolutionParamsMapOpt = Some(_.withDefaultConfiguration(cs.Configuration.runtime)) ) } @@ -1054,7 +1114,8 @@ trait JavaModule whatDependsOn: List[JavaOrScalaModule] ): Task[Unit] = Task.Anon { - val dependencies = (additionalDeps() ++ transitiveIvyDeps()).iterator.to(Seq) + val dependencies = + (additionalDeps() ++ Seq(BoundDep(coursierDependency, force = false))).iterator.to(Seq) val resolution: Resolution = Lib.resolveDependenciesMetadataSafe( repositoriesTask(), dependencies, @@ -1065,7 +1126,12 @@ trait JavaModule ).getOrThrow val roots = whatDependsOn match { - case List() => dependencies.map(_.dep) + case List() => + val mandatoryModules = + mandatoryIvyDeps().map(bindDependency()).iterator.map(_.dep.module).toSet + val (mandatory, main) = resolution.dependenciesOf(coursierDependency) + .partition(dep => mandatoryModules.contains(dep.module)) + additionalDeps().iterator.toSeq.map(_.dep) ++ main ++ mandatory case _ => // We don't really care what scalaVersions is set as here since the user // will be passing in `_2.13` or `._3` anyways. Or it may even be a java @@ -1081,15 +1147,20 @@ trait JavaModule .filter(dep => matchers.exists(matcher => matcher.matches(dep.module))).toSeq } - println( - coursier.util.Print.dependencyTree( - resolution = resolution, - roots = roots, - printExclusions = false, - reverse = if (whatDependsOn.isEmpty) inverse else true - ) + val tree = coursier.util.Print.dependencyTree( + resolution = resolution, + roots = roots, + printExclusions = false, + reverse = if (whatDependsOn.isEmpty) inverse else true ) + val processedTree = tree + .replace(" mill-internal:", " ") + .replace(":0+mill-internal ", " ") + .replace(":0+mill-internal" + System.lineSeparator(), System.lineSeparator()) + + println(processedTree) + Result.Success(()) } @@ -1108,20 +1179,37 @@ trait JavaModule printDepsTree( args.inverse.value, Task.Anon { - transitiveCompileIvyDeps() ++ transitiveRunIvyDeps() + Agg( + coursierDependency.withConfiguration(cs.Configuration.provided), + coursierDependency.withConfiguration(cs.Configuration.runtime) + ).map(BoundDep(_, force = false)) }, validModules )() } case (Flag(true), Flag(false)) => Task.Command { - printDepsTree(args.inverse.value, transitiveCompileIvyDeps, validModules)() + printDepsTree( + args.inverse.value, + Task.Anon { + Agg(BoundDep( + coursierDependency.withConfiguration(cs.Configuration.provided), + force = false + )) + }, + validModules + )() } case (Flag(false), Flag(true)) => Task.Command { printDepsTree( args.inverse.value, - Task.Anon { transitiveRunIvyDeps() }, + Task.Anon { + Agg(BoundDep( + coursierDependency.withConfiguration(cs.Configuration.runtime), + force = false + )) + }, validModules )() } @@ -1271,7 +1359,10 @@ trait JavaModule if (all.value) Seq( Task.Anon { defaultResolver().resolveDeps( - transitiveCompileIvyDeps() ++ transitiveIvyDeps(), + Seq( + coursierDependency.withConfiguration(cs.Configuration.provided), + coursierDependency + ), sources = true, resolutionParamsMapOpt = Some(_.withDefaultConfiguration(coursier.core.Configuration.compile)) @@ -1279,7 +1370,7 @@ trait JavaModule }, Task.Anon { defaultResolver().resolveDeps( - transitiveRunIvyDeps() ++ transitiveIvyDeps(), + Seq(coursierDependency.withConfiguration(cs.Configuration.runtime)), sources = true ) } @@ -1327,3 +1418,35 @@ trait JavaModule Some((JvmBuildTarget.dataKind, bspJvmBuildTargetTask())) } } + +object JavaModule { + final case class InternalRepo(projects: Seq[cs.Project]) + extends cs.Repository { + + private lazy val map = projects.map(proj => proj.moduleVersion -> proj).toMap + + override def toString(): String = + pprint.apply(this).toString + + def find[F[_]: Monad]( + module: cs.Module, + version: String, + fetch: cs.Repository.Fetch[F] + ): EitherT[F, String, (cs.ArtifactSource, cs.Project)] = + EitherT( + Monad[F].point { + map.get((module, version)) + .map((this, _)) + .toRight(s"Not an internal Mill module: ${module.repr}:$version") + } + ) + + def artifacts( + dependency: cs.Dependency, + project: cs.Project, + overrideClassifiers: Option[Seq[coursier.core.Classifier]] + ): Seq[(coursier.core.Publication, coursier.util.Artifact)] = + // Mill modules' artifacts are handled by Mill itself + Nil + } +} diff --git a/scalalib/src/mill/scalalib/PublishModule.scala b/scalalib/src/mill/scalalib/PublishModule.scala index 179f0a1e3c2..5ca02e6d2d5 100644 --- a/scalalib/src/mill/scalalib/PublishModule.scala +++ b/scalalib/src/mill/scalalib/PublishModule.scala @@ -12,6 +12,7 @@ import mill.scalalib.publish.SonatypeHelpers.{ } import mill.scalalib.publish.{Artifact, SonatypePublisher} import os.Path +import coursier.core.DependencyManagement /** * Configuration necessary for publishing a Scala module to Maven Central or similar @@ -69,9 +70,75 @@ trait PublishModule extends JavaModule { outer => Artifact(pomSettings().organization, artifactId(), publishVersion()) } + def publishIvyDeps + : Task[(Map[coursier.core.Module, String], DependencyManagement.Map) => Agg[Dependency]] = + Task.Anon { + (rootDepVersions: Map[coursier.core.Module, String], bomDepMgmt: DependencyManagement.Map) => + def process(dep: coursier.core.Dependency): coursier.core.Dependency = { + var dep0 = dep + + if (dep0.version.isEmpty) + for (version <- rootDepVersions.get(dep0.module)) + dep0 = dep0.withVersion(version) + + for ( + values <- bomDepMgmt.get(DependencyManagement.Key.from(dep0)) + if values.minimizedExclusions.nonEmpty + ) + dep0 = dep0.withMinimizedExclusions( + dep0.minimizedExclusions.join(values.minimizedExclusions) + ) + + dep0 + } + + val ivyPomDeps = allIvyDeps() + .map(bindDependency()) + .map(_.dep) + .map(process) + .map(BoundDep(_, force = false)) + .map(_.toDep) + .map(resolvePublishDependency.apply().apply(_)) + + val runIvyPomDeps = runIvyDeps() + .map(bindDependency()) + .map(_.dep) + .map(process) + .map(BoundDep(_, force = false)) + .map(_.toDep) + .map(resolvePublishDependency.apply().apply(_)) + .filter(!ivyPomDeps.contains(_)) + + val compileIvyPomDeps = compileIvyDeps() + .map(bindDependency()) + .map(_.dep) + .map(process) + .map(BoundDep(_, force = false)) + .map(_.toDep) + .map(resolvePublishDependency.apply().apply(_)) + .filter(!ivyPomDeps.contains(_)) + + val modulePomDeps = T.sequence(moduleDepsChecked.collect { + case m: PublishModule => m.publishSelfDependency + })() + val compileModulePomDeps = T.sequence(compileModuleDepsChecked.collect { + case m: PublishModule => m.publishSelfDependency + })() + val runModulePomDeps = T.sequence(runModuleDepsChecked.collect { + case m: PublishModule => m.publishSelfDependency + })() + + ivyPomDeps ++ + compileIvyPomDeps.map(_.copy(scope = Scope.Provided)) ++ + runIvyPomDeps.map(_.copy(scope = Scope.Runtime)) ++ + modulePomDeps.map(Dependency(_, Scope.Compile)) ++ + compileModulePomDeps.map(Dependency(_, Scope.Provided)) ++ + runModulePomDeps.map(Dependency(_, Scope.Runtime)) + } + def publishXmlDeps: Task[Agg[Dependency]] = Task.Anon { val ivyPomDeps = - processedIvyDeps().map(_.toDep) + allIvyDeps() .map(resolvePublishDependency.apply().apply(_)) val runIvyPomDeps = runIvyDeps() @@ -131,48 +198,59 @@ trait PublishModule extends JavaModule { outer => PathRef(pomPath) } - /** - * Dependencies with version placeholder filled from BOMs, alongside with BOM data - */ - def bomDetails: T[(Map[coursier.core.Module, String], coursier.core.DependencyManagement.Map)] = - Task { - val (processedDeps, depMgmt) = defaultResolver().processDeps( - processedIvyDeps(), - resolutionParams = resolutionParams(), - boms = allBomDeps().toSeq.map(_.withConfig(Configuration.compile)) - ) - (processedDeps.map(_.moduleVersion).toMap, depMgmt) - } - def ivy: T[PathRef] = Task { - val (rootDepVersions, bomDepMgmt) = bomDetails() - val publishXmlDeps0 = publishXmlDeps().map { dep => - if (dep.artifact.version.isEmpty) - dep.copy( - artifact = dep.artifact.copy( - version = rootDepVersions.getOrElse( - coursier.core.Module( - coursier.core.Organization(dep.artifact.group), - coursier.core.ModuleName(dep.artifact.id), - Map.empty - ), - "" /* throw instead? */ - ) - ) + val results = defaultResolver().processDeps( + Seq( + BoundDep( + coursierDependency.withConfiguration(Configuration.runtime), + force = false ) - else - dep + ), + resolutionParams = resolutionParams() + ) + val bomDepMgmt = results.last._2 + val publishXmlDeps0 = { + val rootDepVersions = results.last._1.map(_.moduleVersion).toMap + publishIvyDeps().apply(rootDepVersions, bomDepMgmt) } val overrides = { + val bomDepMgmt0 = { + // Ensure we don't override versions of root dependencies with overrides from the BOM + val rootDepsAdjustment = publishXmlDeps0.iterator.flatMap { dep => + val key = coursier.core.DependencyManagement.Key( + coursier.core.Organization(dep.artifact.group), + coursier.core.ModuleName(dep.artifact.id), + coursier.core.Type.jar, + coursier.core.Classifier.empty + ) + bomDepMgmt.get(key).flatMap { values => + if (values.version.nonEmpty && values.version != dep.artifact.version) + Some(key -> values.withVersion("")) + else + None + } + } + bomDepMgmt ++ rootDepsAdjustment + } + lazy val moduleSet = publishXmlDeps0.map(dep => (dep.artifact.group, dep.artifact.id)).toSet val depMgmtEntries = processedDependencyManagement( depManagement().toSeq .map(bindDependency()) .map(_.dep) .filter(_.version.nonEmpty) + .filter { depMgmt => + // Ensure we don't override versions of root dependencies with overrides from the BOM + !moduleSet.contains((depMgmt.module.organization.value, depMgmt.module.name.value)) + } ) val entries = coursier.core.DependencyManagement.add( Map.empty, - depMgmtEntries ++ bomDepMgmt + depMgmtEntries ++ bomDepMgmt0 + .filter { + case (key, _) => + // Ensure we don't override versions of root dependencies with overrides from the BOM + !moduleSet.contains((key.organization.value, key.name.value)) + } ) entries.toVector .map { diff --git a/scalalib/src/mill/scalalib/ScalaModule.scala b/scalalib/src/mill/scalalib/ScalaModule.scala index deda05f2120..5ffd55c2261 100644 --- a/scalalib/src/mill/scalalib/ScalaModule.scala +++ b/scalalib/src/mill/scalalib/ScalaModule.scala @@ -481,7 +481,10 @@ trait ScalaModule extends JavaModule with TestModule.ScalaModuleBase { outer => ) } val bind = bindDependency() - transitiveRunIvyDeps() ++ transitiveIvyDeps() ++ + Seq(BoundDep( + coursierDependency.withConfiguration(coursier.core.Configuration.runtime), + force = false + )) ++ Agg(ivy"com.lihaoyi:::ammonite:${ammVersion}").map(bind) } } diff --git a/scalalib/test/src/mill/scalalib/CrossVersionTests.scala b/scalalib/test/src/mill/scalalib/CrossVersionTests.scala index ae1f2f5fa25..4bdae52f4bd 100644 --- a/scalalib/test/src/mill/scalalib/CrossVersionTests.scala +++ b/scalalib/test/src/mill/scalalib/CrossVersionTests.scala @@ -30,18 +30,19 @@ object CrossVersionTests extends TestSuite { object JavaDependsOnScala213 extends JavaModule { val tree = - """├─ org.slf4j:slf4j-api:1.7.35 - |├─ com.lihaoyi:upickle_2.13:1.4.0 - |│ ├─ com.lihaoyi:ujson_2.13:1.4.0 - |│ │ └─ com.lihaoyi:upickle-core_2.13:1.4.0 - |│ │ └─ com.lihaoyi:geny_2.13:0.6.10 - |│ ├─ com.lihaoyi:upack_2.13:1.4.0 - |│ │ └─ com.lihaoyi:upickle-core_2.13:1.4.0 - |│ │ └─ com.lihaoyi:geny_2.13:0.6.10 - |│ └─ com.lihaoyi:upickle-implicits_2.13:1.4.0 - |│ └─ com.lihaoyi:upickle-core_2.13:1.4.0 - |│ └─ com.lihaoyi:geny_2.13:0.6.10 - |└─ org.scala-lang:scala-library:2.13.10 + """├─ StandaloneScala213 + |│ ├─ com.lihaoyi:upickle_2.13:1.4.0 + |│ │ ├─ com.lihaoyi:ujson_2.13:1.4.0 + |│ │ │ └─ com.lihaoyi:upickle-core_2.13:1.4.0 + |│ │ │ └─ com.lihaoyi:geny_2.13:0.6.10 + |│ │ ├─ com.lihaoyi:upack_2.13:1.4.0 + |│ │ │ └─ com.lihaoyi:upickle-core_2.13:1.4.0 + |│ │ │ └─ com.lihaoyi:geny_2.13:0.6.10 + |│ │ └─ com.lihaoyi:upickle-implicits_2.13:1.4.0 + |│ │ └─ com.lihaoyi:upickle-core_2.13:1.4.0 + |│ │ └─ com.lihaoyi:geny_2.13:0.6.10 + |│ └─ org.scala-lang:scala-library:2.13.10 + |└─ org.slf4j:slf4j-api:1.7.35 |""".stripMargin override def moduleDeps = Seq(StandaloneScala213) override def ivyDeps = Agg(ivy"org.slf4j:slf4j-api:1.7.35") @@ -49,20 +50,21 @@ object CrossVersionTests extends TestSuite { object Scala3DependsOnScala213 extends ScalaModule { val tree = - """├─ com.lihaoyi:sourcecode_3:0.2.7 - |├─ org.scala-lang:scala3-library_3:3.2.1 + """├─ StandaloneScala213 + |│ ├─ com.lihaoyi:upickle_2.13:1.4.0 + |│ │ ├─ com.lihaoyi:ujson_2.13:1.4.0 + |│ │ │ └─ com.lihaoyi:upickle-core_2.13:1.4.0 + |│ │ │ └─ com.lihaoyi:geny_2.13:0.6.10 + |│ │ ├─ com.lihaoyi:upack_2.13:1.4.0 + |│ │ │ └─ com.lihaoyi:upickle-core_2.13:1.4.0 + |│ │ │ └─ com.lihaoyi:geny_2.13:0.6.10 + |│ │ └─ com.lihaoyi:upickle-implicits_2.13:1.4.0 + |│ │ └─ com.lihaoyi:upickle-core_2.13:1.4.0 + |│ │ └─ com.lihaoyi:geny_2.13:0.6.10 |│ └─ org.scala-lang:scala-library:2.13.10 - |├─ com.lihaoyi:upickle_2.13:1.4.0 - |│ ├─ com.lihaoyi:ujson_2.13:1.4.0 - |│ │ └─ com.lihaoyi:upickle-core_2.13:1.4.0 - |│ │ └─ com.lihaoyi:geny_2.13:0.6.10 - |│ ├─ com.lihaoyi:upack_2.13:1.4.0 - |│ │ └─ com.lihaoyi:upickle-core_2.13:1.4.0 - |│ │ └─ com.lihaoyi:geny_2.13:0.6.10 - |│ └─ com.lihaoyi:upickle-implicits_2.13:1.4.0 - |│ └─ com.lihaoyi:upickle-core_2.13:1.4.0 - |│ └─ com.lihaoyi:geny_2.13:0.6.10 - |└─ org.scala-lang:scala-library:2.13.10 + |├─ com.lihaoyi:sourcecode_3:0.2.7 + |└─ org.scala-lang:scala3-library_3:3.2.1 + | └─ org.scala-lang:scala-library:2.13.10 |""".stripMargin override def scalaVersion = "3.2.1" override def moduleDeps = Seq(StandaloneScala213) @@ -71,21 +73,23 @@ object CrossVersionTests extends TestSuite { object JavaDependsOnScala3 extends JavaModule { val tree = - """├─ org.slf4j:slf4j-api:1.7.35 - |├─ com.lihaoyi:sourcecode_3:0.2.7 - |├─ org.scala-lang:scala3-library_3:3.2.1 - |│ └─ org.scala-lang:scala-library:2.13.10 - |├─ com.lihaoyi:upickle_2.13:1.4.0 - |│ ├─ com.lihaoyi:ujson_2.13:1.4.0 - |│ │ └─ com.lihaoyi:upickle-core_2.13:1.4.0 - |│ │ └─ com.lihaoyi:geny_2.13:0.6.10 - |│ ├─ com.lihaoyi:upack_2.13:1.4.0 - |│ │ └─ com.lihaoyi:upickle-core_2.13:1.4.0 - |│ │ └─ com.lihaoyi:geny_2.13:0.6.10 - |│ └─ com.lihaoyi:upickle-implicits_2.13:1.4.0 - |│ └─ com.lihaoyi:upickle-core_2.13:1.4.0 - |│ └─ com.lihaoyi:geny_2.13:0.6.10 - |└─ org.scala-lang:scala-library:2.13.10 + """├─ Scala3DependsOnScala213 + |│ ├─ com.lihaoyi:sourcecode_3:0.2.7 + |│ ├─ StandaloneScala213 + |│ │ ├─ com.lihaoyi:upickle_2.13:1.4.0 + |│ │ │ ├─ com.lihaoyi:ujson_2.13:1.4.0 + |│ │ │ │ └─ com.lihaoyi:upickle-core_2.13:1.4.0 + |│ │ │ │ └─ com.lihaoyi:geny_2.13:0.6.10 + |│ │ │ ├─ com.lihaoyi:upack_2.13:1.4.0 + |│ │ │ │ └─ com.lihaoyi:upickle-core_2.13:1.4.0 + |│ │ │ │ └─ com.lihaoyi:geny_2.13:0.6.10 + |│ │ │ └─ com.lihaoyi:upickle-implicits_2.13:1.4.0 + |│ │ │ └─ com.lihaoyi:upickle-core_2.13:1.4.0 + |│ │ │ └─ com.lihaoyi:geny_2.13:0.6.10 + |│ │ └─ org.scala-lang:scala-library:2.13.10 + |│ └─ org.scala-lang:scala3-library_3:3.2.1 + |│ └─ org.scala-lang:scala-library:2.13.10 + |└─ org.slf4j:slf4j-api:1.7.35 |""".stripMargin override def moduleDeps = Seq(Scala3DependsOnScala213) override def ivyDeps = Agg(ivy"org.slf4j:slf4j-api:1.7.35") @@ -101,19 +105,20 @@ object CrossVersionTests extends TestSuite { override def moduleDeps = Seq(sandwitch3) override def scalacOptions = Seq("-Ytasty-reader") val tree = - """├─ org.scala-lang:scala-library:2.13.6 - |├─ com.lihaoyi:upickle_3:1.4.0 - |│ ├─ com.lihaoyi:ujson_3:1.4.0 - |│ │ └─ com.lihaoyi:upickle-core_3:1.4.0 - |│ │ └─ com.lihaoyi:geny_3:0.6.10 - |│ ├─ com.lihaoyi:upack_3:1.4.0 - |│ │ └─ com.lihaoyi:upickle-core_3:1.4.0 - |│ │ └─ com.lihaoyi:geny_3:0.6.10 - |│ └─ com.lihaoyi:upickle-implicits_3:1.4.0 - |│ └─ com.lihaoyi:upickle-core_3:1.4.0 - |│ └─ com.lihaoyi:geny_3:0.6.10 - |└─ org.scala-lang:scala3-library_3:3.0.2 - | └─ org.scala-lang:scala-library:2.13.6 + """├─ sandwitch3 + |│ ├─ com.lihaoyi:upickle_3:1.4.0 + |│ │ ├─ com.lihaoyi:ujson_3:1.4.0 + |│ │ │ └─ com.lihaoyi:upickle-core_3:1.4.0 + |│ │ │ └─ com.lihaoyi:geny_3:0.6.10 + |│ │ ├─ com.lihaoyi:upack_3:1.4.0 + |│ │ │ └─ com.lihaoyi:upickle-core_3:1.4.0 + |│ │ │ └─ com.lihaoyi:geny_3:0.6.10 + |│ │ └─ com.lihaoyi:upickle-implicits_3:1.4.0 + |│ │ └─ com.lihaoyi:upickle-core_3:1.4.0 + |│ │ └─ com.lihaoyi:geny_3:0.6.10 + |│ └─ org.scala-lang:scala3-library_3:3.0.2 + |│ └─ org.scala-lang:scala-library:2.13.6 + |└─ org.scala-lang:scala-library:2.13.6 |""".stripMargin } @@ -144,12 +149,6 @@ object CrossVersionTests extends TestSuite { } } - val Right(deps) = eval.apply(mod.transitiveIvyDeps) - - val depNames = deps.value.toSeq.map(d => d.name).sorted - - assert(depNames == expectedDeps.sorted) - val Right(libs) = eval.apply(mod.compileClasspath) val libNames = libs.value.map(l => l.path.last).filter(_.endsWith(".jar")).toSeq.sorted diff --git a/scalalib/test/src/mill/scalalib/ScalaMultiModuleClasspathsTests.scala b/scalalib/test/src/mill/scalalib/ScalaMultiModuleClasspathsTests.scala index 39387e4028e..09949afc7c5 100644 --- a/scalalib/test/src/mill/scalalib/ScalaMultiModuleClasspathsTests.scala +++ b/scalalib/test/src/mill/scalalib/ScalaMultiModuleClasspathsTests.scala @@ -131,14 +131,13 @@ object ScalaMultiModuleClasspathsTests extends TestSuite { eval, MultiModuleClasspaths.ModMod.qux, expectedRunClasspath = List( - // We pick up the oldest version of utest 0.7.0 from the current module, because - // utest is a `runIvyDeps` and not picked up transitively - "com/lihaoyi/utest_2.13/0.8.4/utest_2.13-0.8.4.jar", + "org/scala-lang/scala-library/2.13.12/scala-library-2.13.12.jar", // We pick up the newest version of sourcecode 0.2.4 from the upstream module, because // sourcecode is a `ivyDeps` and `runIvyDeps` and those are picked up transitively "com/lihaoyi/sourcecode_2.13/0.2.2/sourcecode_2.13-0.2.2.jar", - // - "org/scala-lang/scala-library/2.13.12/scala-library-2.13.12.jar", + // We pick up the oldest version of utest 0.7.0 from the current module, because + // utest is a `runIvyDeps` and not picked up transitively + "com/lihaoyi/utest_2.13/0.8.4/utest_2.13-0.8.4.jar", "org/scala-sbt/test-interface/1.0/test-interface-1.0.jar", "org/portable-scala/portable-scala-reflect_2.13/1.1.3/portable-scala-reflect_2.13-1.1.3.jar", "org/scala-lang/scala-reflect/2.13.12/scala-reflect-2.13.12.jar", @@ -159,13 +158,12 @@ object ScalaMultiModuleClasspathsTests extends TestSuite { "out/ModMod/qux/compile.dest/classes" ), expectedCompileClasspath = List( + "org/scala-lang/scala-library/2.13.12/scala-library-2.13.12.jar", + "com/lihaoyi/sourcecode_2.13/0.2.2/sourcecode_2.13-0.2.2.jar", // Make sure we only have geny 0.6.4 from the current module, and not newer // versions pulled in by the upstream modules, because as `compileIvyDeps` it // is not picked up transitively "com/lihaoyi/geny_2.13/0.4.0/geny_2.13-0.4.0.jar", - "com/lihaoyi/sourcecode_2.13/0.2.2/sourcecode_2.13-0.2.2.jar", - // - "org/scala-lang/scala-library/2.13.12/scala-library-2.13.12.jar", // "ModMod/foo/compile-resources", "ModMod/foo/unmanaged", @@ -196,14 +194,14 @@ object ScalaMultiModuleClasspathsTests extends TestSuite { eval, MultiModuleClasspaths.ModCompile.qux, expectedRunClasspath = List( - // `utest` is a `runIvyDeps` and not picked up transitively - "com/lihaoyi/utest_2.13/0.8.4/utest_2.13-0.8.4.jar", + "org/scala-lang/scala-library/2.13.12/scala-library-2.13.12.jar", // Because `sourcecode` comes from `ivyDeps`, and the dependency from // `qux` to `bar` is a `compileModuleDeps`, we do not include its // dependencies for `qux`'s `runClasspath` "com/lihaoyi/sourcecode_2.13/0.2.0/sourcecode_2.13-0.2.0.jar", + // `utest` is a `runIvyDeps` and not picked up transitively + "com/lihaoyi/utest_2.13/0.8.4/utest_2.13-0.8.4.jar", // - "org/scala-lang/scala-library/2.13.12/scala-library-2.13.12.jar", "org/scala-sbt/test-interface/1.0/test-interface-1.0.jar", "org/portable-scala/portable-scala-reflect_2.13/1.1.3/portable-scala-reflect_2.13-1.1.3.jar", "org/scala-lang/scala-reflect/2.13.12/scala-reflect-2.13.12.jar", @@ -214,14 +212,13 @@ object ScalaMultiModuleClasspathsTests extends TestSuite { "out/ModCompile/qux/compile.dest/classes" ), expectedCompileClasspath = List( - "com/lihaoyi/geny_2.13/0.4.0/geny_2.13-0.4.0.jar", + "org/scala-lang/scala-library/2.13.12/scala-library-2.13.12.jar", // `sourcecode` is a `ivyDeps` from a `compileModuleDeps, which still // gets picked up transitively, but only for compilation. This is necessary // in order to make sure that we can correctly compile against the upstream // module's classes. "com/lihaoyi/sourcecode_2.13/0.2.2/sourcecode_2.13-0.2.2.jar", - // - "org/scala-lang/scala-library/2.13.12/scala-library-2.13.12.jar", + "com/lihaoyi/geny_2.13/0.4.0/geny_2.13-0.4.0.jar", // "ModCompile/foo/compile-resources", "ModCompile/foo/unmanaged", @@ -252,14 +249,14 @@ object ScalaMultiModuleClasspathsTests extends TestSuite { eval, MultiModuleClasspaths.CompileMod.qux, expectedRunClasspath = List( - "com/lihaoyi/utest_2.13/0.8.4/utest_2.13-0.8.4.jar", + "org/scala-lang/scala-library/2.13.12/scala-library-2.13.12.jar", // We pick up the version of `sourcecode` from `ivyDeps` from `bar` because // we have a normal `moduleDeps` from `qux` to `bar`, but do not pick it up // from `foo` because it's a `compileIvyDeps` from `bar` to `foo` and // `compileIvyDeps` are not transitive "com/lihaoyi/sourcecode_2.13/0.2.1/sourcecode_2.13-0.2.1.jar", + "com/lihaoyi/utest_2.13/0.8.4/utest_2.13-0.8.4.jar", // - "org/scala-lang/scala-library/2.13.12/scala-library-2.13.12.jar", "org/scala-sbt/test-interface/1.0/test-interface-1.0.jar", "org/portable-scala/portable-scala-reflect_2.13/1.1.3/portable-scala-reflect_2.13-1.1.3.jar", "org/scala-lang/scala-reflect/2.13.12/scala-reflect-2.13.12.jar", @@ -275,10 +272,10 @@ object ScalaMultiModuleClasspathsTests extends TestSuite { "out/CompileMod/qux/compile.dest/classes" ), expectedCompileClasspath = List( - "com/lihaoyi/geny_2.13/0.4.0/geny_2.13-0.4.0.jar", - "com/lihaoyi/sourcecode_2.13/0.2.1/sourcecode_2.13-0.2.1.jar", - // "org/scala-lang/scala-library/2.13.12/scala-library-2.13.12.jar", + // + "com/lihaoyi/sourcecode_2.13/0.2.1/sourcecode_2.13-0.2.1.jar", + "com/lihaoyi/geny_2.13/0.4.0/geny_2.13-0.4.0.jar", // We do not include `foo`s compile output here, because `foo` is a // `compileModuleDep` of `bar`, and `compileModuleDep`s are non-transitive // @@ -302,9 +299,9 @@ object ScalaMultiModuleClasspathsTests extends TestSuite { eval, MultiModuleClasspaths.ModRun.qux, expectedRunClasspath = List( - "com/lihaoyi/utest_2.13/0.8.4/utest_2.13-0.8.4.jar", - "com/lihaoyi/sourcecode_2.13/0.2.2/sourcecode_2.13-0.2.2.jar", "org/scala-lang/scala-library/2.13.12/scala-library-2.13.12.jar", + "com/lihaoyi/sourcecode_2.13/0.2.2/sourcecode_2.13-0.2.2.jar", + "com/lihaoyi/utest_2.13/0.8.4/utest_2.13-0.8.4.jar", "org/scala-sbt/test-interface/1.0/test-interface-1.0.jar", "org/portable-scala/portable-scala-reflect_2.13/1.1.3/portable-scala-reflect_2.13-1.1.3.jar", "org/scala-lang/scala-reflect/2.13.12/scala-reflect-2.13.12.jar", @@ -325,9 +322,9 @@ object ScalaMultiModuleClasspathsTests extends TestSuite { "out/ModRun/qux/compile.dest/classes" ), expectedCompileClasspath = List( - "com/lihaoyi/geny_2.13/0.4.0/geny_2.13-0.4.0.jar", - "com/lihaoyi/sourcecode_2.13/0.2.0/sourcecode_2.13-0.2.0.jar", "org/scala-lang/scala-library/2.13.12/scala-library-2.13.12.jar", + "com/lihaoyi/sourcecode_2.13/0.2.0/sourcecode_2.13-0.2.0.jar", + "com/lihaoyi/geny_2.13/0.4.0/geny_2.13-0.4.0.jar", "ModRun/qux/compile-resources", "ModRun/qux/unmanaged" ), @@ -345,9 +342,9 @@ object ScalaMultiModuleClasspathsTests extends TestSuite { eval, MultiModuleClasspaths.RunMod.qux, expectedRunClasspath = List( - "com/lihaoyi/utest_2.13/0.8.4/utest_2.13-0.8.4.jar", - "com/lihaoyi/sourcecode_2.13/0.2.2/sourcecode_2.13-0.2.2.jar", "org/scala-lang/scala-library/2.13.12/scala-library-2.13.12.jar", + "com/lihaoyi/sourcecode_2.13/0.2.2/sourcecode_2.13-0.2.2.jar", + "com/lihaoyi/utest_2.13/0.8.4/utest_2.13-0.8.4.jar", "org/scala-sbt/test-interface/1.0/test-interface-1.0.jar", "org/portable-scala/portable-scala-reflect_2.13/1.1.3/portable-scala-reflect_2.13-1.1.3.jar", "org/scala-lang/scala-reflect/2.13.12/scala-reflect-2.13.12.jar", @@ -368,9 +365,9 @@ object ScalaMultiModuleClasspathsTests extends TestSuite { "out/RunMod/qux/compile.dest/classes" ), expectedCompileClasspath = List( - "com/lihaoyi/geny_2.13/0.4.0/geny_2.13-0.4.0.jar", - "com/lihaoyi/sourcecode_2.13/0.2.1/sourcecode_2.13-0.2.1.jar", "org/scala-lang/scala-library/2.13.12/scala-library-2.13.12.jar", + "com/lihaoyi/sourcecode_2.13/0.2.1/sourcecode_2.13-0.2.1.jar", + "com/lihaoyi/geny_2.13/0.4.0/geny_2.13-0.4.0.jar", // `bar` ends up here because it's a normal `moduleDep`, but not `foo` because // it's a `runtimeModuleDep "RunMod/bar/compile-resources", diff --git a/scalanativelib/src/mill/scalanativelib/ScalaNativeModule.scala b/scalanativelib/src/mill/scalanativelib/ScalaNativeModule.scala index 3eb6e84def0..2b7657a0720 100644 --- a/scalanativelib/src/mill/scalanativelib/ScalaNativeModule.scala +++ b/scalanativelib/src/mill/scalanativelib/ScalaNativeModule.scala @@ -9,16 +9,7 @@ import mill.util.Jvm import mill.util.Util.millProjectModule import mill.scalalib.api.ZincWorkerUtil import mill.scalalib.bsp.{ScalaBuildTarget, ScalaPlatform} -import mill.scalalib.{ - BoundDep, - CrossVersion, - Dep, - DepSyntax, - Lib, - SbtModule, - ScalaModule, - TestModule -} +import mill.scalalib.{CrossVersion, Dep, DepSyntax, Lib, SbtModule, ScalaModule, TestModule} import mill.testrunner.{TestResult, TestRunner, TestRunnerUtils} import mill.scalanativelib.api._ import mill.scalanativelib.worker.{ @@ -314,7 +305,7 @@ trait ScalaNativeModule extends ScalaModule { outer => )) } - override def transitiveIvyDeps: T[Agg[BoundDep]] = Task { + def coursierProject: Task[coursier.core.Project] = Task.Anon { // Exclude cross published version dependencies leading to conflicts in Scala 3 vs 2.13 // When using Scala 3 exclude Scala 2.13 standard native libraries, @@ -331,15 +322,27 @@ trait ScalaNativeModule extends ScalaModule { outer => val nativeSuffix = platformSuffix() - val exclusions = scalaBinaryVersionToExclude.flatMap { scalaBinVersion => - nativeStandardLibraries.map(library => - "org.scala-native" -> s"$library${nativeSuffix}_$scalaBinVersion" - ) - } + val exclusions = coursier.core.MinimizedExclusions( + scalaBinaryVersionToExclude + .flatMap { scalaBinVersion => + nativeStandardLibraries.map { library => + coursier.core.Organization("org.scala-native") -> + coursier.core.ModuleName(s"$library${nativeSuffix}_$scalaBinVersion") + } + } + .toSet + ) - super.transitiveIvyDeps().map { dep => - dep.exclude(exclusions: _*) - } + val proj = super.coursierProject() + + proj.withDependencies( + proj.dependencies.map { + case (config, dep) => + config -> dep.withMinimizedExclusions( + dep.minimizedExclusions.join(exclusions) + ) + } + ) } override def prepareOffline(all: Flag): Command[Unit] = { diff --git a/scalanativelib/test/src/mill/scalanativelib/ExclusionsTests.scala b/scalanativelib/test/src/mill/scalanativelib/ExclusionsTests.scala index d3698c0c587..5ca38eb93d4 100644 --- a/scalanativelib/test/src/mill/scalanativelib/ExclusionsTests.scala +++ b/scalanativelib/test/src/mill/scalanativelib/ExclusionsTests.scala @@ -30,7 +30,7 @@ object ExclusionsTests extends TestSuite { val exclusionsEvaluator = UnitTester(Exclusions, null) val tests: Tests = Tests { - test("scala3 scala native libraries are excluded in Scala 2.13") { + test("scala3 scala native libraries are excluded in Scala 2_13") { val Right(result) = exclusionsEvaluator(Exclusions.scala213.resolvedIvyDeps) val jars = result.value.iterator.map(_.path.last).toSet assert(jars.contains("nativelib_native0.4_2.13-0.4.3.jar")) @@ -38,7 +38,7 @@ object ExclusionsTests extends TestSuite { assert(jars.contains("clib_native0.4_2.13-0.4.3.jar")) assert(!jars.contains("clib_native0.4_3-0.4.3.jar")) } - test("scala2.13 scala native libraries are excluded in Scala 3") { + test("scala2_13 scala native libraries are excluded in Scala 3") { val Right(result) = exclusionsEvaluator(Exclusions.scala3.resolvedIvyDeps) val jars = result.value.iterator.map(_.path.last).toSet assert(jars.contains("nativelib_native0.4_3-0.4.3.jar")) From a8c2378c8c4277862667c91e9bdb737b36c09991 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Tue, 17 Dec 2024 16:55:42 +0100 Subject: [PATCH 02/13] Add scope tests that already pass --- .../test/src/mill/scalalib/BomTests.scala | 141 +++++++++++++++++- 1 file changed, 136 insertions(+), 5 deletions(-) diff --git a/scalalib/test/src/mill/scalalib/BomTests.scala b/scalalib/test/src/mill/scalalib/BomTests.scala index 7df678fcecb..dfda0fac76d 100644 --- a/scalalib/test/src/mill/scalalib/BomTests.scala +++ b/scalalib/test/src/mill/scalalib/BomTests.scala @@ -272,6 +272,67 @@ object BomTests extends TestSuite { } } + object bomScope extends Module { + object provided extends JavaModule with TestPublishModule { + // This BOM has a versions for protobuf-java-util marked as provided, + // and one for scala-parallel-collections_2.13 in the default scope. + // Both should be taken into account here. + def bomIvyDeps = Agg( + ivy"org.apache.spark:spark-parent_2.13:3.5.3" + ) + def compileIvyDeps = Agg( + ivy"com.google.protobuf:protobuf-java-util", + ivy"org.scala-lang.modules:scala-parallel-collections_2.13" + ) + + object leak extends JavaModule with TestPublishModule { + // Same as above, except the dependencies are in the + // default scope for us here, so the protobuf-java-util version + // shouldn't be read, as it's in provided scope in the BOM. + def bomIvyDeps = Agg( + ivy"org.apache.spark:spark-parent_2.13:3.5.3" + ) + def ivyDeps = Agg( + ivy"com.google.protobuf:protobuf-java-util", + ivy"org.scala-lang.modules:scala-parallel-collections_2.13" + ) + } + } + + object runtimeScope extends JavaModule with TestPublishModule { + // BOM has a version for org.mvnpm.at.hpcc-js:wasm marked as runtime. + // This version should be taken into account in runtime deps here. + def bomIvyDeps = Agg( + ivy"io.quarkus:quarkus-bom:3.15.1" + ) + def runIvyDeps = Agg( + ivy"org.mvnpm.at.hpcc-js:wasm" + ) + } + + object runtimeScopeLeak extends JavaModule with TestPublishModule { + // BOM has a version for org.mvnpm.at.hpcc-js:wasm marked as runtime. + // This version shouldn't be taken into account in main deps here. + def bomIvyDeps = Agg( + ivy"io.quarkus:quarkus-bom:3.15.1" + ) + def ivyDeps = Agg( + ivy"org.mvnpm.at.hpcc-js:wasm" + ) + } + + object testScopeLeak extends JavaModule with TestPublishModule { + // BOM has a version for scalatest_2.13 marked as test scope. + // This version shouldn't be taken into account in main module here. + def bomIvyDeps = Agg( + ivy"org.apache.spark:spark-parent_2.13:3.5.3" + ) + def ivyDeps = Agg( + ivy"org.scalatest:scalatest_2.13" + ) + } + } + object bomOnModuleDependency extends JavaModule with TestPublishModule { def ivyDeps = Agg( ivy"com.google.protobuf:protobuf-java:3.23.4" @@ -301,7 +362,7 @@ object BomTests extends TestSuite { def compileClasspathContains( module: JavaModule, fileName: String, - jarCheck: Option[String => Boolean] + jarCheck: Option[String => Boolean] = None )(implicit eval: UnitTester ) = { @@ -311,10 +372,30 @@ object BomTests extends TestSuite { assert(check(fileName)) } + def runtimeClasspathFileNames(module: JavaModule)(implicit + eval: UnitTester + ): Seq[String] = + eval(module.runClasspath).toTry.get.value + .toSeq.map(_.path.last) + + def runtimeClasspathContains( + module: JavaModule, + fileName: String, + jarCheck: Option[String => Boolean] = None + )(implicit + eval: UnitTester + ) = { + val fileNames = runtimeClasspathFileNames(module) + assert(fileNames.contains(fileName)) + for (check <- jarCheck; fileName <- fileNames) + assert(check(fileName)) + } + def publishLocalAndResolve( module: PublishModule, dependencyModules: Seq[PublishModule], - scalaSuffix: String + scalaSuffix: String, + fetchRuntime: Boolean )(implicit eval: UnitTester): Seq[os.Path] = { val localIvyRepo = eval.evaluator.workspace / "ivy2Local" eval(module.publishLocal(localIvyRepo.toString)).toTry.get @@ -334,6 +415,13 @@ object BomTests extends TestSuite { .addRepositories( coursierapi.IvyRepository.of(localIvyRepo.toNIO.toUri.toASCIIString + "[defaultPattern]") ) + .withResolutionParams { + val defaultParams = coursierapi.ResolutionParams.create() + defaultParams.withDefaultConfiguration( + if (fetchRuntime) "runtime" + else defaultParams.getDefaultConfiguration + ) + } .fetch() .asScala .map(os.Path(_)) @@ -375,12 +463,17 @@ object BomTests extends TestSuite { dependencyModules: Seq[PublishModule] = Nil, jarCheck: Option[String => Boolean] = None, ivy2LocalCheck: Boolean = true, - scalaSuffix: String = "" + scalaSuffix: String = "", + runtimeOnly: Boolean = false )(implicit eval: UnitTester): Unit = { - compileClasspathContains(module, jarName, jarCheck) + if (runtimeOnly) + runtimeClasspathContains(module, jarName, jarCheck) + else + compileClasspathContains(module, jarName, jarCheck) if (ivy2LocalCheck) { - val resolvedCp = publishLocalAndResolve(module, dependencyModules, scalaSuffix) + val resolvedCp = + publishLocalAndResolve(module, dependencyModules, scalaSuffix, fetchRuntime = runtimeOnly) assert(resolvedCp.map(_.last).contains(jarName)) for (check <- jarCheck; fileName <- resolvedCp.map(_.last)) assert(check(fileName)) @@ -642,6 +735,44 @@ object BomTests extends TestSuite { } } + test("bomScope") { + test("provided") - UnitTester(modules, null).scoped { implicit eval => + // test about provided scope, nothing to see in published stuff + compileClasspathContains( + modules.bomScope.provided, + "protobuf-java-3.23.4.jar" + ) + } + test("providedFromBomRuntimeScope") - UnitTester(modules, null).scoped { implicit eval => + // test about provided scope, nothing to see in published stuff + compileClasspathContains( + modules.bomScope.provided, + "scala-parallel-collections_2.13-1.0.4.jar" + ) + } + test("leakProvidedInCompile") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.bomScope.provided.leak, + "scala-parallel-collections_2.13-1.0.4.jar" + ) + } + + test("testCheck") - UnitTester(modules, null).scoped { implicit eval => + compileClasspathContains( + modules.bomScope.testScopeLeak, + "scalatest_2.13-3.2.16.jar" + ) + } + + test("runtime") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.bomScope.runtimeScope, + "wasm-2.15.3.jar", + runtimeOnly = true + ) + } + } + test("bomOnModuleDependency") { test("check") - UnitTester(modules, null).scoped { implicit eval => isInClassPath( From e0849295adf4e565de942a05ccdb4796e2a8c462 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Tue, 17 Dec 2024 16:55:42 +0100 Subject: [PATCH 03/13] Add compile-time / runtime dep mgmt tests --- .../test/src/mill/scalalib/BomTests.scala | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/scalalib/test/src/mill/scalalib/BomTests.scala b/scalalib/test/src/mill/scalalib/BomTests.scala index dfda0fac76d..f0a51f96207 100644 --- a/scalalib/test/src/mill/scalalib/BomTests.scala +++ b/scalalib/test/src/mill/scalalib/BomTests.scala @@ -333,6 +333,29 @@ object BomTests extends TestSuite { } } + object depMgmtScope extends Module { + object provided extends JavaModule with TestPublishModule { + // Version in depManagement should be used in compileIvyDeps + def depManagement = Agg( + ivy"org.scala-lang.modules:scala-parallel-collections_2.13:1.0.4" + ) + def compileIvyDeps = Agg( + ivy"org.scala-lang.modules:scala-parallel-collections_2.13" + ) + } + + object runtimeScope extends JavaModule with TestPublishModule { + // Dep mgmt has a version for org.mvnpm.at.hpcc-js:wasm + // This version should be taken into account in runtime deps here. + def depManagement = Agg( + ivy"org.mvnpm.at.hpcc-js:wasm:2.15.3" + ) + def runIvyDeps = Agg( + ivy"org.mvnpm.at.hpcc-js:wasm" + ) + } + } + object bomOnModuleDependency extends JavaModule with TestPublishModule { def ivyDeps = Agg( ivy"com.google.protobuf:protobuf-java:3.23.4" @@ -788,5 +811,23 @@ object BomTests extends TestSuite { ) } } + + test("depMgmtScope") { + test("depManagementInProvided") - UnitTester(modules, null).scoped { implicit eval => + // test about provided scope, nothing to see in published stuff + compileClasspathContains( + modules.depMgmtScope.provided, + "scala-parallel-collections_2.13-1.0.4.jar" + ) + } + + test("runtime") - UnitTester(modules, null).scoped { implicit eval => + isInClassPath( + modules.depMgmtScope.runtimeScope, + "wasm-2.15.3.jar", + runtimeOnly = true + ) + } + } } } From cec7aadec7484f46987dccf5304e3ec7abbe029f Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Tue, 17 Dec 2024 16:55:42 +0100 Subject: [PATCH 04/13] Take BOMs into account in test modules --- scalalib/src/mill/scalalib/JavaModule.scala | 15 ++++++++++++- .../test/src/mill/scalalib/BomTests.scala | 21 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 62c78584cae..90218ecd4d9 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -60,6 +60,10 @@ trait JavaModule } } + override def extraBomIvyDeps = Task.Anon[Agg[Dep]] { + super.extraBomIvyDeps() ++ outer.bomIvyDeps() + } + /** * JavaModule and its derivatives define inner test modules. * To avoid unexpected misbehavior due to the use of the wrong inner test trait @@ -170,7 +174,7 @@ trait JavaModule def allBomDeps: Task[Agg[BomDependency]] = Task.Anon { val modVerOrMalformed = - bomIvyDeps().map(bindDependency()).map { bomDep => + (bomIvyDeps() ++ extraBomIvyDeps()).map(bindDependency()).map { bomDep => val fromModVer = coursier.core.Dependency(bomDep.dep.module, bomDep.dep.version) if (fromModVer == bomDep.dep) Right(bomDep.dep.asBomDependency) @@ -194,6 +198,15 @@ trait JavaModule ) } + /** + * BOM dependencies that are not meant to be overridden or changed by users. + * + * This is mainly used to add BOM dependencies of the main module to its test + * modules, while ensuring test dependencies of the BOM are taken into account too + * in the test module. + */ + def extraBomIvyDeps: Task[Agg[Dep]] = Task.Anon { Agg.empty[Dep] } + /** * Dependency management data * diff --git a/scalalib/test/src/mill/scalalib/BomTests.scala b/scalalib/test/src/mill/scalalib/BomTests.scala index f0a51f96207..bc5277e03af 100644 --- a/scalalib/test/src/mill/scalalib/BomTests.scala +++ b/scalalib/test/src/mill/scalalib/BomTests.scala @@ -321,6 +321,21 @@ object BomTests extends TestSuite { ) } + object testScope extends JavaModule with TestPublishModule { + // BOM has a version for scalatest_2.13 marked as test scope. + // This version should be taken into account in test modules here. + def bomIvyDeps = Agg( + ivy"org.apache.spark:spark-parent_2.13:3.5.3" + ) + object test extends JavaTests { + def testFramework = "com.novocode.junit.JUnitFramework" + def ivyDeps = Agg( + ivy"com.novocode:junit-interface:0.11", + ivy"org.scalatest:scalatest_2.13" + ) + } + } + object testScopeLeak extends JavaModule with TestPublishModule { // BOM has a version for scalatest_2.13 marked as test scope. // This version shouldn't be taken into account in main module here. @@ -780,6 +795,12 @@ object BomTests extends TestSuite { ) } + test("test") - UnitTester(modules, null).scoped { implicit eval => + compileClasspathContains( + modules.bomScope.testScope.test, + "scalatest_2.13-3.2.16.jar" + ) + } test("testCheck") - UnitTester(modules, null).scoped { implicit eval => compileClasspathContains( modules.bomScope.testScopeLeak, From 056641b893a9463b1c302646a7c7f7dac1b6a381 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Tue, 17 Dec 2024 16:55:42 +0100 Subject: [PATCH 05/13] Take depManagement of main module into account in test ones --- scalalib/src/mill/scalalib/JavaModule.scala | 14 +++++++++++- .../test/src/mill/scalalib/BomTests.scala | 22 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 90218ecd4d9..4224042cca4 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -63,6 +63,9 @@ trait JavaModule override def extraBomIvyDeps = Task.Anon[Agg[Dep]] { super.extraBomIvyDeps() ++ outer.bomIvyDeps() } + override def extraDepManagement = Task.Anon[Agg[Dep]] { + super.extraDepManagement() ++ outer.depManagement() + } /** * JavaModule and its derivatives define inner test modules. @@ -207,6 +210,14 @@ trait JavaModule */ def extraBomIvyDeps: Task[Agg[Dep]] = Task.Anon { Agg.empty[Dep] } + /** + * Dependency management entries that are not meant to be overridden or changed by users. + * + * This is mainly used to add dependency management entries of the main module to its test + * modules. + */ + def extraDepManagement: Task[Agg[Dep]] = Task.Anon { Agg.empty[Dep] } + /** * Dependency management data * @@ -517,7 +528,8 @@ trait JavaModule val depMgmt = processedDependencyManagement( - depManagement().iterator.toSeq.map(bindDependency()).map(_.dep) + (depManagement() ++ extraDepManagement()) + .iterator.toSeq.map(bindDependency()).map(_.dep) ).map { case (key, values) => (cs.Configuration.compile, values.fakeDependency(key)) diff --git a/scalalib/test/src/mill/scalalib/BomTests.scala b/scalalib/test/src/mill/scalalib/BomTests.scala index bc5277e03af..cac291a7766 100644 --- a/scalalib/test/src/mill/scalalib/BomTests.scala +++ b/scalalib/test/src/mill/scalalib/BomTests.scala @@ -369,6 +369,21 @@ object BomTests extends TestSuite { ivy"org.mvnpm.at.hpcc-js:wasm" ) } + + object testScope extends JavaModule with TestPublishModule { + // Dep mgmt in main module has a version for scalatest_2.13. + // This version should be taken into account in test modules here. + def depManagement = Agg( + ivy"org.scalatest:scalatest_2.13:3.2.16" + ) + object test extends JavaTests { + def testFramework = "com.novocode.junit.JUnitFramework" + def ivyDeps = Agg( + ivy"com.novocode:junit-interface:0.11", + ivy"org.scalatest:scalatest_2.13" + ) + } + } } object bomOnModuleDependency extends JavaModule with TestPublishModule { @@ -842,6 +857,13 @@ object BomTests extends TestSuite { ) } + test("test") - UnitTester(modules, null).scoped { implicit eval => + compileClasspathContains( + modules.depMgmtScope.testScope.test, + "scalatest_2.13-3.2.16.jar" + ) + } + test("runtime") - UnitTester(modules, null).scoped { implicit eval => isInClassPath( modules.depMgmtScope.runtimeScope, From 8f259c6e413e95973a38eaac572f859dfae73896 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Wed, 18 Dec 2024 18:23:39 +0100 Subject: [PATCH 06/13] Synchronize #4162, #4165 changes --- .../src/mill/bsp/worker/MillBuildServer.scala | 2 +- build.mill | 6 +- contrib/package.mill | 10 +- .../contrib/scoverage/ScoverageModule.scala | 3 +- .../src/DocAnnotationsTests.scala | 2 +- .../idea/mill_modules/helloideajs.test.iml | 2 +- kotlinlib/package.mill | 12 +- .../src/mill/kotlinlib/KotlinModule.scala | 3 +- main/package.mill | 2 + main/util/src/mill/util/CoursierSupport.scala | 24 ++- main/util/src/mill/util/Util.scala | 4 +- scalajslib/package.mill | 6 +- .../src/mill/scalajslib/ScalaJSModule.scala | 3 +- .../src/mill/scalalib/CoursierModule.scala | 16 +- scalalib/src/mill/scalalib/JavaModule.scala | 141 +++++++++++++++++- .../src/mill/scalalib/PublishModule.scala | 19 ++- .../ScalaMultiModuleClasspathsTests.scala | 13 +- .../mill/scalalib/bsp/BspModuleTests.scala | 2 +- scalanativelib/package.mill | 8 +- .../scalanativelib/ScalaNativeModule.scala | 7 +- .../mill/scalanativelib/ExclusionsTests.scala | 4 +- 21 files changed, 222 insertions(+), 67 deletions(-) diff --git a/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala b/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala index 85c64c99e36..33983e37bfd 100644 --- a/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala +++ b/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala @@ -338,7 +338,7 @@ private class MillBuildServer( Task.Anon { ( m.compileIvyDeps(), - m.ivyDeps(), + m.mandatoryIvyDeps() ++ m.ivyDeps(), m.unmanagedClasspath() ) } diff --git a/build.mill b/build.mill index 1af20417066..6d4fdfa8d86 100644 --- a/build.mill +++ b/build.mill @@ -116,7 +116,7 @@ object Deps { val asmTree = ivy"org.ow2.asm:asm-tree:9.7.1" val bloopConfig = ivy"ch.epfl.scala::bloop-config:1.5.5" - val coursierVersion = "2.1.21" + val coursierVersion = "2.1.22" val coursier = ivy"io.get-coursier::coursier:$coursierVersion" val coursierInterface = ivy"io.get-coursier:interface:1.0.26" val coursierJvm = ivy"io.get-coursier::coursier-jvm:$coursierVersion" @@ -571,6 +571,10 @@ trait MillStableScalaModule extends MillPublishScalaModule with Mima { "mill.main.MainModule.mill$define$BaseModule0$_setter_$evalWatchedValues_=" ), + // Overrides added for new methods, ought to be safe + ProblemFilter.exclude[ReversedMissingMethodProblem]("mill.scalalib.JavaModule.mill$scalalib$JavaModule$$super$internalRepositories"), + ProblemFilter.exclude[ReversedMissingMethodProblem]("mill.scalanativelib.ScalaNativeModule.mill$scalanativelib$ScalaNativeModule$$super$coursierProject"), + // https://github.com/com-lihaoyi/mill/pull/3503 ProblemFilter.exclude[ReversedMissingMethodProblem]( "mill.scalalib.ScalaModule#ScalaTests.mill$scalalib$ScalaModule$ScalaTests$$super$mandatoryScalacOptions" diff --git a/contrib/package.mill b/contrib/package.mill index d11cc3a0132..e5ff1c1dce3 100644 --- a/contrib/package.mill +++ b/contrib/package.mill @@ -126,11 +126,12 @@ object `package` extends RootModule { // Worker for Scoverage 2.0 object worker2 extends build.MillPublishScalaModule { - def compileModuleDeps = Seq(build.main.api) - def moduleDeps = Seq(scoverage.api) - def testDepPaths = Task { Seq(compile().classes) } + def compileModuleDeps = Seq( + build.main.api, + scoverage.api + ) def compileIvyDeps = Task { - Agg( + super.mandatoryIvyDeps() ++ Agg( // compile-time only, need to provide the correct scoverage version at runtime build.Deps.scalacScoverage2Plugin, build.Deps.scalacScoverage2Reporter, @@ -138,6 +139,7 @@ object `package` extends RootModule { build.Deps.scalacScoverage2Serializer ) } + def mandatoryIvyDeps = Agg.empty[Dep] } } diff --git a/contrib/scoverage/src/mill/contrib/scoverage/ScoverageModule.scala b/contrib/scoverage/src/mill/contrib/scoverage/ScoverageModule.scala index fc3f819e14a..750a953d334 100644 --- a/contrib/scoverage/src/mill/contrib/scoverage/ScoverageModule.scala +++ b/contrib/scoverage/src/mill/contrib/scoverage/ScoverageModule.scala @@ -125,8 +125,7 @@ trait ScoverageModule extends ScalaModule { outer: ScalaModule => millProjectModule( workerArtifact, - repositoriesTask(), - resolveFilter = _.toString.contains(workerArtifact) + repositoriesTask() ) } diff --git a/integration/feature/docannotations/src/DocAnnotationsTests.scala b/integration/feature/docannotations/src/DocAnnotationsTests.scala index 132842a9150..d85d1e4aa72 100644 --- a/integration/feature/docannotations/src/DocAnnotationsTests.scala +++ b/integration/feature/docannotations/src/DocAnnotationsTests.scala @@ -102,11 +102,11 @@ object DocAnnotationsTests extends UtestIntegrationTestSuite { | tree. | |Inputs: - | core.bomIvyDeps | core.mandatoryIvyDeps | core.ivyDeps | core.compileIvyDeps | core.runIvyDeps + | core.bomIvyDeps | core.depManagement |""".stripMargin, ivyDepsTree diff --git a/integration/ide/gen-idea/resources/hello-idea/idea/mill_modules/helloideajs.test.iml b/integration/ide/gen-idea/resources/hello-idea/idea/mill_modules/helloideajs.test.iml index 363889666d0..08e83a0baab 100644 --- a/integration/ide/gen-idea/resources/hello-idea/idea/mill_modules/helloideajs.test.iml +++ b/integration/ide/gen-idea/resources/hello-idea/idea/mill_modules/helloideajs.test.iml @@ -11,7 +11,7 @@ - + diff --git a/kotlinlib/package.mill b/kotlinlib/package.mill index 2913cb65eb2..50df85efc6f 100644 --- a/kotlinlib/package.mill +++ b/kotlinlib/package.mill @@ -45,11 +45,13 @@ object `package` extends RootModule with build.MillPublishScalaModule with Build ) object impl extends MillKotlinModule { - override def moduleDeps: Seq[PublishModule] = Seq(worker) - override def compileIvyDeps: T[Agg[Dep]] = Agg( - build.Deps.osLib, - build.Deps.kotlinCompiler - ) + override def compileModuleDeps = Seq(worker) + def mandatoryIvyDeps = Agg.empty[Dep] + override def compileIvyDeps: T[Agg[Dep]] = + super.mandatoryIvyDeps() ++ Agg( + build.Deps.osLib, + build.Deps.kotlinCompiler + ) } } } diff --git a/kotlinlib/src/mill/kotlinlib/KotlinModule.scala b/kotlinlib/src/mill/kotlinlib/KotlinModule.scala index 5ce0f2c6483..826148265d6 100644 --- a/kotlinlib/src/mill/kotlinlib/KotlinModule.scala +++ b/kotlinlib/src/mill/kotlinlib/KotlinModule.scala @@ -88,8 +88,7 @@ trait KotlinModule extends JavaModule { outer => private[kotlinlib] def kotlinWorkerClasspath = Task { millProjectModule( "mill-kotlinlib-worker-impl", - repositoriesTask(), - resolveFilter = _.toString.contains("mill-kotlinlib-worker-impl") + repositoriesTask() ) } diff --git a/main/package.mill b/main/package.mill index a55ee3d5882..f2df0e731e2 100644 --- a/main/package.mill +++ b/main/package.mill @@ -53,6 +53,8 @@ object `package` extends RootModule with build.MillStableScalaModule with BuildI // Lib.resolveDependenciesMetadataSafe( Lib.resolveDependenciesMetadata( repositories = build.dist.repositoriesTask(), + // When re-bootstraping Mill, replace the next line with + // Seq(BoundDep(coursierDependency(), force = false)), build.dist.transitiveIvyDeps(), Some(build.dist.mapDependencies()), build.dist.resolutionCustomizer(), diff --git a/main/util/src/mill/util/CoursierSupport.scala b/main/util/src/mill/util/CoursierSupport.scala index 34c9e08aa18..0b3e5f43ede 100644 --- a/main/util/src/mill/util/CoursierSupport.scala +++ b/main/util/src/mill/util/CoursierSupport.scala @@ -32,6 +32,13 @@ trait CoursierSupport { ctx.fold(cache)(c => cache.withLogger(new TickerResolutionLogger(c))) } + /** + * A `coursier.Repository` that exposes modules with hard-coded artifact list + * + * Used in Mill tests. This exposes internal workers for example, so that these + * come from the build and not from remote repositories or ~/.ivy2/local. See + * `MillJavaModule#{testTransitiveDeps,writeLocalTestOverrides}` in the Mill build. + */ private class TestOverridesRepo(root: os.ResourcePath) extends Repository { private def listFor(mod: Module): Either[os.ResourceNotFoundException, String] = { @@ -104,6 +111,10 @@ trait CoursierSupport { * We do not bother breaking this out into the separate ZincWorkerApi classpath, * because Coursier is already bundled with mill/Ammonite to support the * `import $ivy` syntax. + * + * Avoid using `deprecatedResolveFilter` if you can. As a substitute, use exclusions + * (or upfront, mark some dependencies as provided aka compile-time when you publish them), + * or as a last resort, manually filter the file sequence returned by this function. */ def resolveDependencies( repositories: Seq[Repository], @@ -114,7 +125,7 @@ trait CoursierSupport { customizer: Option[Resolution => Resolution] = None, ctx: Option[mill.api.Ctx.Log] = None, coursierCacheCustomizer: Option[FileCache[Task] => FileCache[Task]] = None, - resolveFilter: os.Path => Boolean = _ => true, + deprecatedResolveFilter: os.Path => Boolean = _ => true, artifactTypes: Option[Set[Type]] = None, resolutionParams: ResolutionParams = ResolutionParams() ): Result[Agg[PathRef]] = { @@ -157,7 +168,7 @@ trait CoursierSupport { Agg.from( res.files .map(os.Path(_)) - .filter(resolveFilter) + .filter(deprecatedResolveFilter) .map(PathRef(_, quick = true)) ) ) @@ -175,7 +186,7 @@ trait CoursierSupport { customizer: Option[Resolution => Resolution], ctx: Option[mill.api.Ctx.Log], coursierCacheCustomizer: Option[FileCache[Task] => FileCache[Task]], - resolveFilter: os.Path => Boolean, + deprecatedResolveFilter: os.Path => Boolean, artifactTypes: Option[Set[Type]] ): Result[Agg[PathRef]] = resolveDependencies( @@ -187,7 +198,7 @@ trait CoursierSupport { customizer, ctx, coursierCacheCustomizer, - resolveFilter, + deprecatedResolveFilter, artifactTypes, ResolutionParams() ) @@ -202,7 +213,7 @@ trait CoursierSupport { customizer: Option[Resolution => Resolution], ctx: Option[mill.api.Ctx.Log], coursierCacheCustomizer: Option[FileCache[Task] => FileCache[Task]], - resolveFilter: os.Path => Boolean + deprecatedResolveFilter: os.Path => Boolean ): Result[Agg[PathRef]] = resolveDependencies( repositories, @@ -213,8 +224,7 @@ trait CoursierSupport { customizer, ctx, coursierCacheCustomizer, - resolveFilter, - None + deprecatedResolveFilter ) @deprecated( diff --git a/main/util/src/mill/util/Util.scala b/main/util/src/mill/util/Util.scala index 9b7cdc60e7d..ddd51ca7827 100644 --- a/main/util/src/mill/util/Util.scala +++ b/main/util/src/mill/util/Util.scala @@ -73,7 +73,7 @@ object Util { def millProjectModule( artifact: String, repositories: Seq[Repository], - resolveFilter: os.Path => Boolean = _ => true, + deprecatedResolveFilter: os.Path => Boolean = _ => true, // this should correspond to the mill runtime Scala version artifactSuffix: String = "_2.13" ): Result[Agg[PathRef]] = { @@ -90,7 +90,7 @@ object Util { ) ), force = Nil, - resolveFilter = resolveFilter + deprecatedResolveFilter = deprecatedResolveFilter ).map(_.map(_.withRevalidateOnce)) } diff --git a/scalajslib/package.mill b/scalajslib/package.mill index 80bc08913dd..2dc9c3cb694 100644 --- a/scalajslib/package.mill +++ b/scalajslib/package.mill @@ -45,9 +45,9 @@ object `package` extends RootModule with build.MillStableScalaModule with BuildI trait WorkerModule extends build.MillPublishScalaModule with Cross.Module[String] { def scalajsWorkerVersion = crossValue def millSourcePath: os.Path = super.millSourcePath / scalajsWorkerVersion - def testDepPaths = Task { Seq(compile().classes) } - def moduleDeps = Seq(build.scalajslib.`worker-api`, build.main.client, build.main.api) - def ivyDeps = Agg( + def compileModuleDeps = Seq(build.scalajslib.`worker-api`, build.main.client, build.main.api) + def mandatoryIvyDeps = Agg.empty[Dep] + def compileIvyDeps = super.mandatoryIvyDeps() ++ Agg( build.Deps.Scalajs_1.scalajsLinker, build.Deps.Scalajs_1.scalajsSbtTestAdapter, build.Deps.Scalajs_1.scalajsEnvNodejs, diff --git a/scalajslib/src/mill/scalajslib/ScalaJSModule.scala b/scalajslib/src/mill/scalajslib/ScalaJSModule.scala index 6ecd5dc2253..3164ff532d4 100644 --- a/scalajslib/src/mill/scalajslib/ScalaJSModule.scala +++ b/scalajslib/src/mill/scalajslib/ScalaJSModule.scala @@ -52,8 +52,7 @@ trait ScalaJSModule extends scalalib.ScalaModule { outer => def scalaJSWorkerClasspath = Task { mill.util.Util.millProjectModule( artifact = s"mill-scalajslib-worker-${scalaJSWorkerVersion()}", - repositories = repositoriesTask(), - resolveFilter = _.toString.contains("mill-scalajslib-worker") + repositories = repositoriesTask() ) } diff --git a/scalalib/src/mill/scalalib/CoursierModule.scala b/scalalib/src/mill/scalalib/CoursierModule.scala index eeccca0a3d9..849474f2d0c 100644 --- a/scalalib/src/mill/scalalib/CoursierModule.scala +++ b/scalalib/src/mill/scalalib/CoursierModule.scala @@ -251,7 +251,7 @@ object CoursierModule { deps: IterableOnce[T], resolutionParams: ResolutionParams = ResolutionParams(), boms: IterableOnce[BomDependency] = Nil - ): Seq[(Seq[coursier.core.Dependency], DependencyManagement.Map)] = { + ): (Seq[coursier.core.Dependency], DependencyManagement.Map) = { val deps0 = deps .map(implicitly[CoursierModule.Resolvable[T]].bind(_, bind)) .iterator.toSeq @@ -267,15 +267,13 @@ object CoursierModule { boms = boms0 ).getOrThrow - deps0.map { dep => - ( - res.finalDependenciesCache.getOrElse(dep.dep, ???), - DependencyManagement.addDependencies( - Map.empty, - res.projectCache.get(dep.dep.moduleVersion).getOrElse(???)._2.dependencyManagement - ) + ( + res.finalDependenciesCache.getOrElse(deps0.head.dep, ???), + DependencyManagement.addDependencies( + Map.empty, + res.projectCache.get(deps0.head.dep.moduleVersion).getOrElse(???)._2.dependencyManagement ) - } + ) } } diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 4224042cca4..e8850d40725 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -469,20 +469,31 @@ trait JavaModule */ def coursierProject: Task[cs.Project] = Task.Anon { - // FIXME That's coursier.maven.MavenRepositoryInternal.defaultConfigurations (private for now) - // plus hack for provided - val mavenScopes = Map( + // Tells coursier that if something depends on a given scope of ours, we should also + // pull other scopes of our own dependencies. + // + // E.g. scope(runtime) contains compile, so depending on us as a runtime dependency + // will not only pull our runtime dependencies, but also our compile ones. + // + // This is the default scope mapping used in coursier for Maven dependencies, but for + // one scope: provided. By default in Maven, depending on a dependency as provided + // doesn't pull anything. Here, by explicitly adding provided to the values, + // we make coursier pull our own provided dependencies. + val scopes = Map( cs.Configuration.compile -> Seq.empty, cs.Configuration.runtime -> Seq(cs.Configuration.compile), cs.Configuration.default -> Seq(cs.Configuration.runtime), cs.Configuration.test -> Seq(cs.Configuration.runtime), // hack, so that depending on `coursierDependency.withConfiguration(Configuration.provided)` - // pulls the compile-scope of our provided dependencies (rather than nothing) - cs.Configuration.provided -> Seq(cs.Configuration.compile) + // pulls our provided dependencies (rather than nothing) + cs.Configuration.provided -> Seq(cs.Configuration.provided) ) val internalDependencies = moduleDepsChecked.flatMap { modDep => + // Standard dependencies + // We pull their compile scope when our compile scope is asked, + // and pull their runtime scope when our runtime scope is asked. Seq( (cs.Configuration.compile, modDep.coursierDependency), ( @@ -492,27 +503,41 @@ trait JavaModule ) } ++ compileModuleDepsChecked.map { modDep => + // Compile-only (aka provided) dependencies + // We pull their compile scope when our provided scope is asked (see scopes above) (cs.Configuration.provided, modDep.coursierDependency) } ++ runModuleDepsChecked.map { modDep => + // Runtime dependencies + // We pull their runtime scope when our runtime scope is pulled ( cs.Configuration.runtime, - modDep.coursierDependency.withConfiguration(cs.Configuration.runtime) + modDep.coursierDependency.withConfiguration( + if (modDep.coursierDependency.configuration.isEmpty) cs.Configuration.runtime + else modDep.coursierDependency.configuration + ) ) } val dependencies = (mandatoryIvyDeps() ++ ivyDeps()).map(bindDependency()).map(_.dep).iterator.toSeq.flatMap { dep => + // Standard dependencies, like above + // We pull their compile scope when our compile scope is asked, + // and pull their runtime scope when our runtime scope is asked. Seq( (cs.Configuration.compile, dep), (cs.Configuration.runtime, dep.withConfiguration(cs.Configuration.runtime)) ) } ++ compileIvyDeps().map(bindDependency()).map(_.dep).map { dep => + // Compile-only (aka provided) dependencies, like above + // We pull their compile scope when our provided scope is asked (see scopes above) (cs.Configuration.provided, dep) } ++ runIvyDeps().map(bindDependency()).map(_.dep).map { dep => + // Runtime dependencies, like above + // We pull their runtime scope when our runtime scope is pulled ( cs.Configuration.runtime, dep.withConfiguration( @@ -521,6 +546,8 @@ trait JavaModule ) } ++ allBomDeps().map { bomDep => + // BOM dependencies + // Maven has a special scope for those: "import" val dep = cs.Dependency(bomDep.module, bomDep.version).withConfiguration(bomDep.config) (cs.Configuration.`import`, dep) @@ -532,14 +559,17 @@ trait JavaModule .iterator.toSeq.map(bindDependency()).map(_.dep) ).map { case (key, values) => - (cs.Configuration.compile, values.fakeDependency(key)) + val config0 = + if (values.config.isEmpty) cs.Configuration.compile + else values.config + (config0, values.fakeDependency(key)) } cs.Project( module = coursierDependency.module, version = coursierDependency.version, dependencies = internalDependencies ++ dependencies, - configurations = mavenScopes, + configurations = scopes, parent = None, dependencyManagement = depMgmt, properties = Nil, @@ -565,10 +595,96 @@ trait JavaModule projects.distinct } + /** + * The Ivy dependencies of this module, with BOM and dependency management details + * added to them. This should be used when propagating the dependencies transitively + * to other modules. + */ + @deprecated("Unused by Mill, use allIvyDeps instead", "Mill after 0.12.4") + def processedIvyDeps: Task[Agg[BoundDep]] = Task { + allIvyDeps().map(bindDependency()) + } + + /** + * Returns a function adding BOM and dependency management details of + * this module to a `coursier.core.Dependency` + */ + @deprecated("Unused by Mill", "Mill after 0.12.4") + def processDependency( + overrideVersions: Boolean = false + ): Task[coursier.core.Dependency => coursier.core.Dependency] = + Task.Anon((x: coursier.core.Dependency) => x) + + /** + * The transitive ivy dependencies of this module and all it's upstream modules. + * This is calculated from [[ivyDeps]], [[mandatoryIvyDeps]] and recursively from [[moduleDeps]]. + * + * This isn't used by Mill anymore. Instead of this, consider using either: + * * `coursierDependency`, which will pull all this module's dependencies transitively + * * `allIvyDeps`, which contains the full list of direct (external) dependencies of this module + */ + @deprecated("Unused by Mill, use coursierDependency or allIvyDeps instead", "Mill after 0.12.4") + def transitiveIvyDeps: T[Agg[BoundDep]] = Task { + allIvyDeps().map(bindDependency()) ++ + T.traverse(moduleDepsChecked)(_.transitiveIvyDeps)().flatten + } + + /** + * The compile-only transitive ivy dependencies of this module and all its upstream compile-only modules. + * + * This isn't used by Mill anymore. Instead of this, consider using either: + * * `coursierDependency().withConfiguration(Configuration.provided`), which will pull all + * this module's compile-only dependencies transitively + * * `compileIvyDeps`, which contains the full list of direct (external) compile-only + * dependencies of this module + */ + @deprecated( + "Unused by Mill, use coursierDependency().withConfiguration(Configuration.provided) or compileIvyDeps instead", + "Mill after 0.12.4" + ) + def transitiveCompileIvyDeps: T[Agg[BoundDep]] = Task { + compileIvyDeps().map(bindDependency()) ++ + T.traverse(moduleDepsChecked)(_.transitiveCompileIvyDeps)().flatten + } + + /** + * The transitive run ivy dependencies of this module and all it's upstream modules. + * This is calculated from [[runIvyDeps]], [[mandatoryIvyDeps]] and recursively from [[moduleDeps]]. + * + * This isn't used by Mill anymore. Instead of this, consider using either: + * * `coursierDependency().withConfiguration(Configuration.runtime`), which will pull all + * this module's runtime dependencies transitively + * * `runIvyDeps`, which contains the full list of direct (external) runtime + * dependencies of this module + */ + @deprecated( + "Unused by Mill, use coursierDependency().withConfiguration(Configuration.runtime) or runIvyDeps instead", + "Mill after 0.12.4" + ) + def transitiveRunIvyDeps: T[Agg[BoundDep]] = Task { + runIvyDeps().map(bindDependency()) ++ + T.traverse(moduleDepsChecked)(_.transitiveRunIvyDeps)().flatten + } + /** * The repository that knows about this project itself and its module dependencies */ def internalDependenciesRepository: Task[cs.Repository] = Task.Anon { + // This is the main point of contact between the coursier resolver and Mill. + // Basically, all relevant Mill modules are aggregated and converted to a + // coursier.Project (provided by JavaModule#coursierProject). + // + // Dependencies, both external ones (like ivyDeps, bomIvyDeps, etc.) and internal ones + // (like moduleDeps) are put in coursier.Project#dependencies. The coursier.Dependency + // used to represent each module is built by JavaModule#coursierDependency. So we put + // JavaModule#coursierDependency in the dependencies field of other modules' + // JavaModule#coursierProject to represent links between them. + // + // coursier.Project#dependencies accepts (coursier.Configuration, coursier.Dependency) tuples. + // The configuration is different for compile-time only / runtime / BOM dependencies + // (it's respectively provided, runtime, import). The configuration is compile for + // standard ivyDeps / moduleDeps. + // JavaModule.InternalRepo(transitiveCoursierProjects()) } @@ -1445,6 +1561,15 @@ trait JavaModule } object JavaModule { + + /** + * An in-memory [[coursier.Repository]] that exposes the passed projects + * + * Doesn't generate artifacts for these. These are assumed to be managed + * externally for now. + * + * @param projects + */ final case class InternalRepo(projects: Seq[cs.Project]) extends cs.Repository { diff --git a/scalalib/src/mill/scalalib/PublishModule.scala b/scalalib/src/mill/scalalib/PublishModule.scala index 5ca02e6d2d5..3eaca99eddb 100644 --- a/scalalib/src/mill/scalalib/PublishModule.scala +++ b/scalalib/src/mill/scalalib/PublishModule.scala @@ -198,8 +198,22 @@ trait PublishModule extends JavaModule { outer => PathRef(pomPath) } + /** + * Dependencies with version placeholder filled from BOMs, alongside with BOM data + */ + @deprecated("Unused by Mill", "Mill after 0.12.4") + def bomDetails: T[(Map[coursier.core.Module, String], coursier.core.DependencyManagement.Map)] = + Task { + val (processedDeps, depMgmt) = defaultResolver().processDeps( + processedIvyDeps(), + resolutionParams = resolutionParams(), + boms = allBomDeps().toSeq.map(_.withConfig(Configuration.compile)) + ) + (processedDeps.map(_.moduleVersion).toMap, depMgmt) + } + def ivy: T[PathRef] = Task { - val results = defaultResolver().processDeps( + val (results, bomDepMgmt) = defaultResolver().processDeps( Seq( BoundDep( coursierDependency.withConfiguration(Configuration.runtime), @@ -208,9 +222,8 @@ trait PublishModule extends JavaModule { outer => ), resolutionParams = resolutionParams() ) - val bomDepMgmt = results.last._2 val publishXmlDeps0 = { - val rootDepVersions = results.last._1.map(_.moduleVersion).toMap + val rootDepVersions = results.map(_.moduleVersion).toMap publishIvyDeps().apply(rootDepVersions, bomDepMgmt) } val overrides = { diff --git a/scalalib/test/src/mill/scalalib/ScalaMultiModuleClasspathsTests.scala b/scalalib/test/src/mill/scalalib/ScalaMultiModuleClasspathsTests.scala index 09949afc7c5..b06a89e3ce4 100644 --- a/scalalib/test/src/mill/scalalib/ScalaMultiModuleClasspathsTests.scala +++ b/scalalib/test/src/mill/scalalib/ScalaMultiModuleClasspathsTests.scala @@ -158,12 +158,12 @@ object ScalaMultiModuleClasspathsTests extends TestSuite { "out/ModMod/qux/compile.dest/classes" ), expectedCompileClasspath = List( - "org/scala-lang/scala-library/2.13.12/scala-library-2.13.12.jar", - "com/lihaoyi/sourcecode_2.13/0.2.2/sourcecode_2.13-0.2.2.jar", // Make sure we only have geny 0.6.4 from the current module, and not newer // versions pulled in by the upstream modules, because as `compileIvyDeps` it // is not picked up transitively "com/lihaoyi/geny_2.13/0.4.0/geny_2.13-0.4.0.jar", + "org/scala-lang/scala-library/2.13.12/scala-library-2.13.12.jar", + "com/lihaoyi/sourcecode_2.13/0.2.2/sourcecode_2.13-0.2.2.jar", // "ModMod/foo/compile-resources", "ModMod/foo/unmanaged", @@ -212,13 +212,13 @@ object ScalaMultiModuleClasspathsTests extends TestSuite { "out/ModCompile/qux/compile.dest/classes" ), expectedCompileClasspath = List( + "com/lihaoyi/geny_2.13/0.4.0/geny_2.13-0.4.0.jar", "org/scala-lang/scala-library/2.13.12/scala-library-2.13.12.jar", // `sourcecode` is a `ivyDeps` from a `compileModuleDeps, which still // gets picked up transitively, but only for compilation. This is necessary // in order to make sure that we can correctly compile against the upstream // module's classes. "com/lihaoyi/sourcecode_2.13/0.2.2/sourcecode_2.13-0.2.2.jar", - "com/lihaoyi/geny_2.13/0.4.0/geny_2.13-0.4.0.jar", // "ModCompile/foo/compile-resources", "ModCompile/foo/unmanaged", @@ -272,10 +272,9 @@ object ScalaMultiModuleClasspathsTests extends TestSuite { "out/CompileMod/qux/compile.dest/classes" ), expectedCompileClasspath = List( + "com/lihaoyi/geny_2.13/0.4.0/geny_2.13-0.4.0.jar", "org/scala-lang/scala-library/2.13.12/scala-library-2.13.12.jar", - // "com/lihaoyi/sourcecode_2.13/0.2.1/sourcecode_2.13-0.2.1.jar", - "com/lihaoyi/geny_2.13/0.4.0/geny_2.13-0.4.0.jar", // We do not include `foo`s compile output here, because `foo` is a // `compileModuleDep` of `bar`, and `compileModuleDep`s are non-transitive // @@ -322,9 +321,9 @@ object ScalaMultiModuleClasspathsTests extends TestSuite { "out/ModRun/qux/compile.dest/classes" ), expectedCompileClasspath = List( + "com/lihaoyi/geny_2.13/0.4.0/geny_2.13-0.4.0.jar", "org/scala-lang/scala-library/2.13.12/scala-library-2.13.12.jar", "com/lihaoyi/sourcecode_2.13/0.2.0/sourcecode_2.13-0.2.0.jar", - "com/lihaoyi/geny_2.13/0.4.0/geny_2.13-0.4.0.jar", "ModRun/qux/compile-resources", "ModRun/qux/unmanaged" ), @@ -365,9 +364,9 @@ object ScalaMultiModuleClasspathsTests extends TestSuite { "out/RunMod/qux/compile.dest/classes" ), expectedCompileClasspath = List( + "com/lihaoyi/geny_2.13/0.4.0/geny_2.13-0.4.0.jar", "org/scala-lang/scala-library/2.13.12/scala-library-2.13.12.jar", "com/lihaoyi/sourcecode_2.13/0.2.1/sourcecode_2.13-0.2.1.jar", - "com/lihaoyi/geny_2.13/0.4.0/geny_2.13-0.4.0.jar", // `bar` ends up here because it's a normal `moduleDep`, but not `foo` because // it's a `runtimeModuleDep "RunMod/bar/compile-resources", diff --git a/scalalib/test/src/mill/scalalib/bsp/BspModuleTests.scala b/scalalib/test/src/mill/scalalib/bsp/BspModuleTests.scala index b42cbc03dc3..599ac32f8fc 100644 --- a/scalalib/test/src/mill/scalalib/bsp/BspModuleTests.scala +++ b/scalalib/test/src/mill/scalalib/bsp/BspModuleTests.scala @@ -101,7 +101,7 @@ object BspModuleTests extends TestSuite { } test("index 1 (no deps)") { runNoBsp(1, 5000) } test("index 10") { runNoBsp(10, 30000) } - test("index 15") { runNoBsp(15, 30000) } + test("index 15") { runNoBsp(15, 60000) } } def run(entry: Int, maxTime: Int) = retry(3) { UnitTester(InterDeps, null).scoped { eval => diff --git a/scalanativelib/package.mill b/scalanativelib/package.mill index 0233b4a6c65..a0c2c525a9e 100644 --- a/scalanativelib/package.mill +++ b/scalanativelib/package.mill @@ -18,11 +18,10 @@ object `package` extends RootModule with build.MillStableScalaModule { trait WorkerModule extends build.MillPublishScalaModule with Cross.Module[String] { def scalaNativeWorkerVersion = crossValue def millSourcePath: os.Path = super.millSourcePath / scalaNativeWorkerVersion - def testDepPaths = Task { Seq(compile().classes) } - def moduleDeps = Seq(`worker-api`) - def ivyDeps = scalaNativeWorkerVersion match { + def compileModuleDeps = Seq(`worker-api`) + def compileIvyDeps = scalaNativeWorkerVersion match { case "0.5" => - Agg( + super.mandatoryIvyDeps() ++ Agg( build.Deps.osLib, build.Deps.Scalanative_0_5.scalanativeTools, build.Deps.Scalanative_0_5.scalanativeUtil, @@ -30,5 +29,6 @@ object `package` extends RootModule with build.MillStableScalaModule { build.Deps.Scalanative_0_5.scalanativeTestRunner ) } + def mandatoryIvyDeps = Agg.empty[mill.scalalib.Dep] } } diff --git a/scalanativelib/src/mill/scalanativelib/ScalaNativeModule.scala b/scalanativelib/src/mill/scalanativelib/ScalaNativeModule.scala index 2b7657a0720..530b5200cbe 100644 --- a/scalanativelib/src/mill/scalanativelib/ScalaNativeModule.scala +++ b/scalanativelib/src/mill/scalanativelib/ScalaNativeModule.scala @@ -43,8 +43,7 @@ trait ScalaNativeModule extends ScalaModule { outer => def scalaNativeWorkerClasspath = Task { millProjectModule( s"mill-scalanativelib-worker-${scalaNativeWorkerVersion()}", - repositoriesTask(), - resolveFilter = _.toString.contains("mill-scalanativelib-worker") + repositoriesTask() ) } @@ -305,6 +304,10 @@ trait ScalaNativeModule extends ScalaModule { outer => )) } + // override added for binary compatibility + override def transitiveIvyDeps: T[Agg[mill.scalalib.BoundDep]] = + super.transitiveIvyDeps + def coursierProject: Task[coursier.core.Project] = Task.Anon { // Exclude cross published version dependencies leading to conflicts in Scala 3 vs 2.13 diff --git a/scalanativelib/test/src/mill/scalanativelib/ExclusionsTests.scala b/scalanativelib/test/src/mill/scalanativelib/ExclusionsTests.scala index 5ca38eb93d4..d3698c0c587 100644 --- a/scalanativelib/test/src/mill/scalanativelib/ExclusionsTests.scala +++ b/scalanativelib/test/src/mill/scalanativelib/ExclusionsTests.scala @@ -30,7 +30,7 @@ object ExclusionsTests extends TestSuite { val exclusionsEvaluator = UnitTester(Exclusions, null) val tests: Tests = Tests { - test("scala3 scala native libraries are excluded in Scala 2_13") { + test("scala3 scala native libraries are excluded in Scala 2.13") { val Right(result) = exclusionsEvaluator(Exclusions.scala213.resolvedIvyDeps) val jars = result.value.iterator.map(_.path.last).toSet assert(jars.contains("nativelib_native0.4_2.13-0.4.3.jar")) @@ -38,7 +38,7 @@ object ExclusionsTests extends TestSuite { assert(jars.contains("clib_native0.4_2.13-0.4.3.jar")) assert(!jars.contains("clib_native0.4_3-0.4.3.jar")) } - test("scala2_13 scala native libraries are excluded in Scala 3") { + test("scala2.13 scala native libraries are excluded in Scala 3") { val Right(result) = exclusionsEvaluator(Exclusions.scala3.resolvedIvyDeps) val jars = result.value.iterator.map(_.path.last).toSet assert(jars.contains("nativelib_native0.4_3-0.4.3.jar")) From 37d623c80f61551044a32fbb1066fbbc9423b8c9 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Tue, 7 Jan 2025 19:53:28 +0100 Subject: [PATCH 07/13] Tweaking --- .../src/mill/bsp/worker/MillBuildServer.scala | 19 +++++++----- build.mill | 2 ++ .../out-dir/1-out-files/build.mill | 6 ++-- .../src/mill/scalalib/CoursierModule.scala | 26 ++++++++++++++++ scalalib/src/mill/scalalib/JavaModule.scala | 30 +++++++++++-------- 5 files changed, 60 insertions(+), 23 deletions(-) diff --git a/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala b/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala index 33983e37bfd..0bac5eff5c9 100644 --- a/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala +++ b/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala @@ -337,8 +337,13 @@ private class MillBuildServer( tasks = { case m: JavaModule => Task.Anon { ( - m.compileIvyDeps(), - m.mandatoryIvyDeps() ++ m.ivyDeps(), + // full list of dependencies, including transitive ones + m.defaultResolver().allDeps( + Seq( + m.coursierDependency.withConfiguration(coursier.core.Configuration.provided), + m.coursierDependency + ) + ), m.unmanagedClasspath() ) } @@ -349,14 +354,14 @@ private class MillBuildServer( state, id, m: JavaModule, - (compileIvyDeps, ivyDeps, unmanagedClasspath) + (ivyDeps, unmanagedClasspath) ) => - val ivy = compileIvyDeps ++ ivyDeps - val deps = ivy.map { dep => - // TODO: add data with "maven" data kind using a ... + val deps = ivyDeps.collect { + case dep if dep.module.organization != JavaModule.internalOrg => + // TODO: add data with "maven" data kind using a ... // MavenDependencyModule - new DependencyModule(dep.dep.module.repr, dep.dep.version) + new DependencyModule(dep.module.repr, dep.version) } val unmanaged = unmanagedClasspath.map { dep => new DependencyModule(s"unmanaged-${dep.path.last}", "") diff --git a/build.mill b/build.mill index eb2c1aeec27..25c33159282 100644 --- a/build.mill +++ b/build.mill @@ -590,6 +590,8 @@ trait MillStableScalaModule extends MillPublishScalaModule with Mima { // Overrides added for new methods, ought to be safe ProblemFilter.exclude[ReversedMissingMethodProblem]("mill.scalalib.JavaModule.mill$scalalib$JavaModule$$super$internalRepositories"), ProblemFilter.exclude[ReversedMissingMethodProblem]("mill.scalanativelib.ScalaNativeModule.mill$scalanativelib$ScalaNativeModule$$super$coursierProject"), + ProblemFilter.exclude[ReversedMissingMethodProblem]("mill.scalalib.JavaModule#JavaModuleTests.mill$scalalib$JavaModule$JavaModuleTests$$super$extraBomIvyDeps"), + ProblemFilter.exclude[ReversedMissingMethodProblem]("mill.scalalib.JavaModule#JavaModuleTests.mill$scalalib$JavaModule$JavaModuleTests$$super$extraDepManagement"), // https://github.com/com-lihaoyi/mill/pull/3503 ProblemFilter.exclude[ReversedMissingMethodProblem]( diff --git a/example/fundamentals/out-dir/1-out-files/build.mill b/example/fundamentals/out-dir/1-out-files/build.mill index e67d719df6a..8935e6df366 100644 --- a/example/fundamentals/out-dir/1-out-files/build.mill +++ b/example/fundamentals/out-dir/1-out-files/build.mill @@ -243,8 +243,7 @@ out/mill-server > cat out/mill-build/methodCodeHashSignatures.dest/current/spanningInvalidationTree.json { - "call scala.runtime.BoxesRunTime.boxToInteger(int)java.lang.Integer": {}, - "call scala.Predef$#println(java.lang.Object)void": { + "call scala.runtime.BoxesRunTime.boxToInteger(int)java.lang.Integer": { "def build_.package_$foo$#(build_.package_)void": { "call build_.package_$foo$!(build_.package_)void": { "def build_.package_#foo$lzycompute$1()void": { @@ -254,7 +253,8 @@ out/mill-server } } } - } + }, + "call scala.Predef$#println(java.lang.Object)void": {} } */ diff --git a/scalalib/src/mill/scalalib/CoursierModule.scala b/scalalib/src/mill/scalalib/CoursierModule.scala index 849474f2d0c..04f04210fb9 100644 --- a/scalalib/src/mill/scalalib/CoursierModule.scala +++ b/scalalib/src/mill/scalalib/CoursierModule.scala @@ -275,6 +275,32 @@ object CoursierModule { ) ) } + + /** + * All dependencies pulled by the passed dependencies + * + * @param deps root dependencies + * @return full - ordered - list of dependencies pulled by `deps` + */ + def allDeps[T: CoursierModule.Resolvable]( + deps: IterableOnce[T] + ): Seq[coursier.core.Dependency] = { + val deps0 = deps + .map(implicitly[CoursierModule.Resolvable[T]].bind(_, bind)) + .iterator.toSeq + val res = Lib.resolveDependenciesMetadataSafe( + repositories = repositories, + deps = deps0, + mapDependencies = mapDependencies, + customizer = customizer, + coursierCacheCustomizer = coursierCacheCustomizer, + ctx = ctx, + resolutionParams = ResolutionParams(), + boms = Nil + ).getOrThrow + + res.orderedDependencies + } } sealed trait Resolvable[T] { diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index e8850d40725..baec3bdc044 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -452,11 +452,11 @@ trait JavaModule // where eval'ing a Task would be impractical or not allowed cs.Dependency( cs.Module( - coursier.core.Organization("mill-internal"), + JavaModule.internalOrg, coursier.core.ModuleName(millModuleSegments.parts.mkString("-")), Map.empty ), - "0+mill-internal" + JavaModule.internalVersion ).withConfiguration(cs.Configuration.compile) /** @@ -495,7 +495,10 @@ trait JavaModule // We pull their compile scope when our compile scope is asked, // and pull their runtime scope when our runtime scope is asked. Seq( - (cs.Configuration.compile, modDep.coursierDependency), + ( + cs.Configuration.compile, + modDep.coursierDependency.withConfiguration(cs.Configuration.compile) + ), ( cs.Configuration.runtime, modDep.coursierDependency.withConfiguration(cs.Configuration.runtime) @@ -505,17 +508,17 @@ trait JavaModule compileModuleDepsChecked.map { modDep => // Compile-only (aka provided) dependencies // We pull their compile scope when our provided scope is asked (see scopes above) - (cs.Configuration.provided, modDep.coursierDependency) + ( + cs.Configuration.provided, + modDep.coursierDependency.withConfiguration(cs.Configuration.compile) + ) } ++ runModuleDepsChecked.map { modDep => // Runtime dependencies // We pull their runtime scope when our runtime scope is pulled ( cs.Configuration.runtime, - modDep.coursierDependency.withConfiguration( - if (modDep.coursierDependency.configuration.isEmpty) cs.Configuration.runtime - else modDep.coursierDependency.configuration - ) + modDep.coursierDependency.withConfiguration(cs.Configuration.runtime) ) } @@ -526,23 +529,21 @@ trait JavaModule // We pull their compile scope when our compile scope is asked, // and pull their runtime scope when our runtime scope is asked. Seq( - (cs.Configuration.compile, dep), + (cs.Configuration.compile, dep.withConfiguration(cs.Configuration.compile)), (cs.Configuration.runtime, dep.withConfiguration(cs.Configuration.runtime)) ) } ++ compileIvyDeps().map(bindDependency()).map(_.dep).map { dep => // Compile-only (aka provided) dependencies, like above // We pull their compile scope when our provided scope is asked (see scopes above) - (cs.Configuration.provided, dep) + (cs.Configuration.provided, dep.withConfiguration(cs.Configuration.compile)) } ++ runIvyDeps().map(bindDependency()).map(_.dep).map { dep => // Runtime dependencies, like above // We pull their runtime scope when our runtime scope is pulled ( cs.Configuration.runtime, - dep.withConfiguration( - if (dep.configuration.isEmpty) cs.Configuration.runtime else dep.configuration - ) + dep.withConfiguration(cs.Configuration.runtime) ) } ++ allBomDeps().map { bomDep => @@ -1599,4 +1600,7 @@ object JavaModule { // Mill modules' artifacts are handled by Mill itself Nil } + + private[mill] def internalOrg = coursier.core.Organization("mill-internal") + private[mill] def internalVersion = "0+mill-internal" } From 59e95dcb110226778a3861744e3b0d9c1773ec5c Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Tue, 7 Jan 2025 21:14:48 +0100 Subject: [PATCH 08/13] Update BspServerTests test data --- .../resources/snapshots/build-targets-dependency-modules.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-dependency-modules.json b/integration/ide/bsp-server/resources/snapshots/build-targets-dependency-modules.json index b188222db81..66e431aedf0 100644 --- a/integration/ide/bsp-server/resources/snapshots/build-targets-dependency-modules.json +++ b/integration/ide/bsp-server/resources/snapshots/build-targets-dependency-modules.json @@ -8,6 +8,10 @@ { "name": "org.jetbrains.kotlin:kotlin-stdlib", "version": "" + }, + { + "name": "org.jetbrains:annotations", + "version": "13.0" } ] }, From 3e07734c3aa6af6787f56eb9f5ffac3666d76f1f Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Wed, 8 Jan 2025 11:34:10 +0100 Subject: [PATCH 09/13] Add "Overhaul transitive module handling in dependency resolution" PR changes --- main/util/src/mill/util/CoursierSupport.scala | 32 ++++++++++++++----- .../src/mill/scalalib/CoursierModule.scala | 16 ++++++++-- 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/main/util/src/mill/util/CoursierSupport.scala b/main/util/src/mill/util/CoursierSupport.scala index 0b3e5f43ede..a4b347d7bd1 100644 --- a/main/util/src/mill/util/CoursierSupport.scala +++ b/main/util/src/mill/util/CoursierSupport.scala @@ -12,6 +12,8 @@ import coursier.{Artifacts, Classifier, Dependency, Repository, Resolution, Reso import mill.api.Loose.Agg import mill.api.{Ctx, PathRef, Result} +import java.util.concurrent.ConcurrentHashMap + import scala.collection.mutable import scala.util.chaining.scalaUtilChainingOps import coursier.cache.ArchiveCache @@ -39,17 +41,31 @@ trait CoursierSupport { * come from the build and not from remote repositories or ~/.ivy2/local. See * `MillJavaModule#{testTransitiveDeps,writeLocalTestOverrides}` in the Mill build. */ - private class TestOverridesRepo(root: os.ResourcePath) extends Repository { + private final class TestOverridesRepo(root: os.ResourcePath) extends Repository { + + private val map = new ConcurrentHashMap[Module, Option[String]] private def listFor(mod: Module): Either[os.ResourceNotFoundException, String] = { - val classpathKey = s"${mod.organization.value}-${mod.name.value}" - val entryPath = root / classpathKey - try Right(os.read(entryPath)) - catch { - case e: os.ResourceNotFoundException => - Left(e) - } + def entryPath = root / s"${mod.organization.value}-${mod.name.value}" + + val inCacheOpt = Option(map.get(mod)) + + inCacheOpt + .getOrElse { + + val computedOpt = + try Some(os.read(entryPath)) + catch { + case _: os.ResourceNotFoundException => + None + } + val concurrentOpt = Option(map.putIfAbsent(mod, computedOpt)) + concurrentOpt.getOrElse(computedOpt) + } + .toRight { + new os.ResourceNotFoundException(entryPath) + } } def find[F[_]: Monad]( diff --git a/scalalib/src/mill/scalalib/CoursierModule.scala b/scalalib/src/mill/scalalib/CoursierModule.scala index 04f04210fb9..57da3a7bdce 100644 --- a/scalalib/src/mill/scalalib/CoursierModule.scala +++ b/scalalib/src/mill/scalalib/CoursierModule.scala @@ -268,10 +268,22 @@ object CoursierModule { ).getOrThrow ( - res.finalDependenciesCache.getOrElse(deps0.head.dep, ???), + res.finalDependenciesCache.getOrElse( + deps0.head.dep, + sys.error( + s"Should not happen - could not find root dependency ${deps0.head.dep} in Resolution#finalDependenciesCache" + ) + ), DependencyManagement.addDependencies( Map.empty, - res.projectCache.get(deps0.head.dep.moduleVersion).getOrElse(???)._2.dependencyManagement + res.projectCache + .get(deps0.head.dep.moduleVersion) + .map(_._2.dependencyManagement) + .getOrElse { + sys.error( + s"Should not happen - could not find root dependency ${deps0.head.dep.moduleVersion} in Resolution#projectCache" + ) + } ) ) } From cedf050e95a892f89b55b2402c67d3a6c3b14b9e Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Wed, 8 Jan 2025 20:54:56 +0100 Subject: [PATCH 10/13] Report other PR changes --- .../src/mill/scalalib/CoursierModule.scala | 19 +++++++--------- scalalib/src/mill/scalalib/JavaModule.scala | 9 ++++---- .../src/mill/scalalib/JsonFormatters.scala | 22 +++++++++++++++++++ 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/scalalib/src/mill/scalalib/CoursierModule.scala b/scalalib/src/mill/scalalib/CoursierModule.scala index 57da3a7bdce..1bec48b06a2 100644 --- a/scalalib/src/mill/scalalib/CoursierModule.scala +++ b/scalalib/src/mill/scalalib/CoursierModule.scala @@ -274,17 +274,14 @@ object CoursierModule { s"Should not happen - could not find root dependency ${deps0.head.dep} in Resolution#finalDependenciesCache" ) ), - DependencyManagement.addDependencies( - Map.empty, - res.projectCache - .get(deps0.head.dep.moduleVersion) - .map(_._2.dependencyManagement) - .getOrElse { - sys.error( - s"Should not happen - could not find root dependency ${deps0.head.dep.moduleVersion} in Resolution#projectCache" - ) - } - ) + res.projectCache + .get(deps0.head.dep.moduleVersion) + .map(_._2.overrides.flatten.toMap) + .getOrElse { + sys.error( + s"Should not happen - could not find root dependency ${deps0.head.dep.moduleVersion} in Resolution#projectCache" + ) + } ) } diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index baec3bdc044..1c4d55e6b90 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -467,7 +467,7 @@ trait JavaModule * resolution parameters (such as artifact types, etc.), this should be the only way * we provide details about this module to coursier. */ - def coursierProject: Task[cs.Project] = Task.Anon { + def coursierProject: Task[cs.Project] = Task { // Tells coursier that if something depends on a given scope of ours, we should also // pull other scopes of our own dependencies. @@ -588,12 +588,11 @@ trait JavaModule /** * Coursier project of this module and those of all its transitive module dependencies */ - def transitiveCoursierProjects: Task[Seq[cs.Project]] = Task.Anon { - val projects = Seq(coursierProject()) ++ + def transitiveCoursierProjects: Task[Seq[cs.Project]] = Task { + Seq(coursierProject()) ++ T.traverse(compileModuleDepsChecked)(_.transitiveCoursierProjects)().flatten ++ T.traverse(moduleDepsChecked)(_.transitiveCoursierProjects)().flatten ++ T.traverse(runModuleDepsChecked)(_.transitiveCoursierProjects)().flatten - projects.distinct } /** @@ -686,7 +685,7 @@ trait JavaModule // (it's respectively provided, runtime, import). The configuration is compile for // standard ivyDeps / moduleDeps. // - JavaModule.InternalRepo(transitiveCoursierProjects()) + JavaModule.InternalRepo(transitiveCoursierProjects().distinctBy(_.module.name.value)) } /** diff --git a/scalalib/src/mill/scalalib/JsonFormatters.scala b/scalalib/src/mill/scalalib/JsonFormatters.scala index 04fc8109a24..3b2d879e6e0 100644 --- a/scalalib/src/mill/scalalib/JsonFormatters.scala +++ b/scalalib/src/mill/scalalib/JsonFormatters.scala @@ -32,6 +32,28 @@ trait JsonFormatters { upickle.default.macroRW implicit lazy val depMgmtValuesFormat: RW[coursier.core.DependencyManagement.Values] = upickle.default.macroRW + implicit lazy val activationOsFormat: RW[coursier.core.Activation.Os] = upickle.default.macroRW + implicit lazy val infoDeveloperFormat: RW[coursier.core.Info.Developer] = upickle.default.macroRW + implicit lazy val infoScmFormat: RW[coursier.core.Info.Scm] = upickle.default.macroRW + implicit lazy val infoLicenseFormat: RW[coursier.core.Info.License] = upickle.default.macroRW + implicit lazy val infoFormat: RW[coursier.core.Info] = upickle.default.macroRW + implicit lazy val snapshotVersionFormat: RW[coursier.core.SnapshotVersion] = + upickle.default.macroRW + implicit lazy val versionInternalFormat: RW[coursier.core.VersionInterval] = + upickle.default.macroRW + implicit lazy val versionFormat: RW[coursier.core.Version] = + implicitly[RW[String]].bimap( + _.repr, + coursier.core.Version(_) + ) + implicit lazy val snapshotVersioningFormat: RW[coursier.core.SnapshotVersioning] = + upickle.default.macroRW + implicit lazy val versionsFormat: RW[coursier.core.Versions] = upickle.default.macroRW + implicit lazy val versionsDateTimeFormat: RW[coursier.core.Versions.DateTime] = + upickle.default.macroRW + implicit lazy val activationFormat: RW[coursier.core.Activation] = upickle.default.macroRW + implicit lazy val profileFormat: RW[coursier.core.Profile] = upickle.default.macroRW + implicit lazy val projectFormat: RW[coursier.core.Project] = upickle.default.macroRW } object JsonFormatters extends JsonFormatters From 113270569d815d53eef14facc738ce560245730c Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Wed, 8 Jan 2025 23:37:42 +0100 Subject: [PATCH 11/13] Update BSP test snapshots --- .../resources/snapshots/build-targets-dependency-modules.json | 4 ++++ .../snapshots/build-targets-jvm-run-environments.json | 2 +- .../snapshots/build-targets-jvm-test-environments.json | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-dependency-modules.json b/integration/ide/bsp-server/resources/snapshots/build-targets-dependency-modules.json index f838752520f..50415727b78 100644 --- a/integration/ide/bsp-server/resources/snapshots/build-targets-dependency-modules.json +++ b/integration/ide/bsp-server/resources/snapshots/build-targets-dependency-modules.json @@ -27,6 +27,10 @@ { "name": "org.slf4j:slf4j-api", "version": "2.0.16" + }, + { + "name": "org.hamcrest:hamcrest-core", + "version": "1.3" } ] }, diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-jvm-run-environments.json b/integration/ide/bsp-server/resources/snapshots/build-targets-jvm-run-environments.json index 4c424898ebc..477e0bc5bfd 100644 --- a/integration/ide/bsp-server/resources/snapshots/build-targets-jvm-run-environments.json +++ b/integration/ide/bsp-server/resources/snapshots/build-targets-jvm-run-environments.json @@ -5,8 +5,8 @@ "uri": "file:///workspace/app" }, "classpath": [ - "file:///coursier-cache/https/repo1.maven.org/maven2/com/mysql/mysql-connector-j/9.1.0/mysql-connector-j-9.1.0.jar", "file:///coursier-cache/https/repo1.maven.org/maven2/ch/qos/logback/logback-core/1.5.15/logback-core-1.5.15.jar", + "file:///coursier-cache/https/repo1.maven.org/maven2/com/mysql/mysql-connector-j/9.1.0/mysql-connector-j-9.1.0.jar", "file:///coursier-cache/https/repo1.maven.org/maven2/org/slf4j/slf4j-api/2.0.16/slf4j-api-2.0.16.jar", "file:///coursier-cache/https/repo1.maven.org/maven2/com/google/protobuf/protobuf-java/4.26.1/protobuf-java-4.26.1.jar", "file:///workspace/lib/compile-resources", diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-jvm-test-environments.json b/integration/ide/bsp-server/resources/snapshots/build-targets-jvm-test-environments.json index 4c424898ebc..477e0bc5bfd 100644 --- a/integration/ide/bsp-server/resources/snapshots/build-targets-jvm-test-environments.json +++ b/integration/ide/bsp-server/resources/snapshots/build-targets-jvm-test-environments.json @@ -5,8 +5,8 @@ "uri": "file:///workspace/app" }, "classpath": [ - "file:///coursier-cache/https/repo1.maven.org/maven2/com/mysql/mysql-connector-j/9.1.0/mysql-connector-j-9.1.0.jar", "file:///coursier-cache/https/repo1.maven.org/maven2/ch/qos/logback/logback-core/1.5.15/logback-core-1.5.15.jar", + "file:///coursier-cache/https/repo1.maven.org/maven2/com/mysql/mysql-connector-j/9.1.0/mysql-connector-j-9.1.0.jar", "file:///coursier-cache/https/repo1.maven.org/maven2/org/slf4j/slf4j-api/2.0.16/slf4j-api-2.0.16.jar", "file:///coursier-cache/https/repo1.maven.org/maven2/com/google/protobuf/protobuf-java/4.26.1/protobuf-java-4.26.1.jar", "file:///workspace/lib/compile-resources", From 74ef92de65e4c61a76632c9fc62edd6f3570fd99 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Mon, 20 Jan 2025 13:30:38 +0100 Subject: [PATCH 12/13] Work around possible bin compat issues --- build.mill | 7 ------- scalalib/src/mill/scalalib/JavaModule.scala | 8 ++++++-- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/build.mill b/build.mill index 2ed9fe4ce58..ef58d37bced 100644 --- a/build.mill +++ b/build.mill @@ -596,13 +596,6 @@ trait MillStableScalaModule extends MillPublishScalaModule with Mima { "mill.main.MainModule.mill$define$BaseModule0$_setter_$evalWatchedValues_=" ), - ProblemFilter.exclude[ReversedMissingMethodProblem]( - "mill.scalalib.JavaModule#JavaModuleTests.mill$scalalib$JavaModule$JavaModuleTests$$super$extraBomIvyDeps" - ), - ProblemFilter.exclude[ReversedMissingMethodProblem]( - "mill.scalalib.JavaModule#JavaModuleTests.mill$scalalib$JavaModule$JavaModuleTests$$super$extraDepManagement" - ), - // https://github.com/com-lihaoyi/mill/pull/3503 ProblemFilter.exclude[ReversedMissingMethodProblem]( "mill.scalalib.ScalaModule#ScalaTests.mill$scalalib$ScalaModule$ScalaTests$$super$mandatoryScalacOptions" diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index e05ae5a0423..0488889fe80 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -64,10 +64,14 @@ trait JavaModule } override def extraBomIvyDeps = Task.Anon[Agg[Dep]] { - super.extraBomIvyDeps() ++ outer.bomIvyDeps() + // FIXME Add that back when we can break bin-compat + // super.extraBomIvyDeps() ++ + outer.bomIvyDeps() } override def extraDepManagement = Task.Anon[Agg[Dep]] { - super.extraDepManagement() ++ outer.depManagement() + // FIXME Add that back when we can break bin-compat + // super.extraDepManagement() ++ + outer.depManagement() } /** From 7ffc08bfbe10256f95ce31639fe60819e3acc98f Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Mon, 20 Jan 2025 13:57:34 +0100 Subject: [PATCH 13/13] Remove extraBomIvyDeps / extraDepManagement stuff --- scalalib/src/mill/scalalib/JavaModule.scala | 30 +++++---------------- 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 0488889fe80..440fd996e25 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -63,14 +63,14 @@ trait JavaModule } } - override def extraBomIvyDeps = Task.Anon[Agg[Dep]] { + override def bomIvyDeps = Task.Anon[Agg[Dep]] { // FIXME Add that back when we can break bin-compat - // super.extraBomIvyDeps() ++ + // super.bomIvyDeps() ++ outer.bomIvyDeps() } - override def extraDepManagement = Task.Anon[Agg[Dep]] { + override def depManagement = Task.Anon[Agg[Dep]] { // FIXME Add that back when we can break bin-compat - // super.extraDepManagement() ++ + // super.depManagement() ++ outer.depManagement() } @@ -184,7 +184,7 @@ trait JavaModule def allBomDeps: Task[Agg[BomDependency]] = Task.Anon { val modVerOrMalformed = - (bomIvyDeps() ++ extraBomIvyDeps()).map(bindDependency()).map { bomDep => + bomIvyDeps().map(bindDependency()).map { bomDep => val fromModVer = coursier.core.Dependency(bomDep.dep.module, bomDep.dep.version) if (fromModVer == bomDep.dep) Right(bomDep.dep.asBomDependency) @@ -208,23 +208,6 @@ trait JavaModule ) } - /** - * BOM dependencies that are not meant to be overridden or changed by users. - * - * This is mainly used to add BOM dependencies of the main module to its test - * modules, while ensuring test dependencies of the BOM are taken into account too - * in the test module. - */ - def extraBomIvyDeps: Task[Agg[Dep]] = Task.Anon { Agg.empty[Dep] } - - /** - * Dependency management entries that are not meant to be overridden or changed by users. - * - * This is mainly used to add dependency management entries of the main module to its test - * modules. - */ - def extraDepManagement: Task[Agg[Dep]] = Task.Anon { Agg.empty[Dep] } - /** * Dependency management data * @@ -571,8 +554,7 @@ trait JavaModule val depMgmt = processedDependencyManagement( - (depManagement() ++ extraDepManagement()) - .iterator.toSeq.map(bindDependency()).map(_.dep) + depManagement().iterator.toSeq.map(bindDependency()).map(_.dep) ).map { case (key, values) => val config0 =