diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae23acc..bf95e62 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,19 +15,19 @@ jobs: target: - os: linux cpu: amd64 - nim_branch: devel + nim_branch: version-2-0 # - os: linux # cpu: i386 # nim_branch: devel - os: macos cpu: amd64 - nim_branch: devel + nim_branch: version-2-0 - os: windows cpu: amd64 - nim_branch: devel + nim_branch: version-2-0 - os: windows cpu: i386 - nim_branch: devel + nim_branch: version-2-0 include: - target: os: linux @@ -52,6 +52,7 @@ jobs: git config --global init.defaultBranch master git config --global user.email "atlasbot@nimlang.com" git config --global user.name "atlasbot" + git config --global gc.auto 0 - name: Checkout atlas uses: actions/checkout@v4 @@ -128,6 +129,10 @@ jobs: version: ${{ matrix.target.nim_branch }} architecture: ${{ matrix.target.cpu }} + - name: Nim Version + run: | + nim -v + - name: Run tests run: | cd atlas diff --git a/.gitignore b/.gitignore index 5fb5da2..ce37c7e 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ tests/* !tests/*.* atlas deps.png +nim.cfg diff --git a/config.nims b/config.nims index 2b42bb2..6bc5dbe 100644 --- a/config.nims +++ b/config.nims @@ -1,6 +1,6 @@ task build, "Build local atlas": - exec "nim c -d:debug -o:./atlas src/atlas.nim" + exec "nim c -d:debug -o:./bin/atlas src/atlas.nim" task unitTests, "Runs unit tests": exec "nim c -d:debug -r tests/unittests.nim" diff --git a/src/atlas.nim b/src/atlas.nim index b39f85a..ba8a66d 100644 --- a/src/atlas.nim +++ b/src/atlas.nim @@ -219,9 +219,12 @@ proc installDependencies(c: var AtlasContext; nc: var NimbleContext; nimbleFile: let (dir, pkgname, _) = splitFile(nimbleFile) info c, pkgname, "installing dependencies for " & pkgname & ".nimble" var g = createGraph(c, createUrlSkipPatterns(dir)) + trace c, pkgname, "traversing depency loop" let paths = traverseLoop(c, nc, g) + trace c, pkgname, "done traversing depencies" let cfgPath = if CfgHere in c.flags: CfgPath c.currentDir else: findCfgDir(c) patchNimCfg(c, paths, cfgPath) + trace c, pkgname, "executing post install actions" afterGraphActions c, g proc updateDir(c: var AtlasContext; dir, filter: string) = @@ -341,10 +344,6 @@ proc main(c: var AtlasContext) = if c.projectDir == c.workspace or c.projectDir == c.depsDir: fatal action & " command must be executed in a project, not in the workspace" - proc findCurrentNimble(): string = - for x in walkPattern("*.nimble"): - return x - var autoinit = false var explicitProjectOverride = false var explicitDepsDirOverride = false @@ -459,23 +458,21 @@ proc main(c: var AtlasContext) = patchNimCfg c, deps, cfgPath of "use": singleArg() - #fillPackageLookupTable(c.nimbleContext, c, ) - var amb = false - var nimbleFile = findNimbleFile(c, c.workspace, amb) + var nimbleFile = findNimbleFile(c, c.workspace) var nc = createNimbleContext(c, c.depsDir) - if nimbleFile.len == 0: - nimbleFile = c.workspace / extractProjectName(c.workspace) & ".nimble" - writeFile(nimbleFile, "") - patchNimbleFile(nc, c, c.overrides, nimbleFile, args[0]) + if nimbleFile.isNone: + trace c, getCurrentDir().relativePath(c.workspace), "no nimble file found for project" + nimbleFile = some c.workspace / extractProjectName(c.workspace) & ".nimble" + writeFile(nimbleFile.get, "") + nimbleFile = findNimbleFile(c, c.workspace) + trace c, getCurrentDir().relativePath(c.workspace), "wrote new nimble file" + + patchNimbleFile(nc, c, c.overrides, nimbleFile.get(), args[0]) if c.errors > 0: discard "don't continue for 'cannot resolve'" - elif nimbleFile.len > 0 and not amb: - installDependencies(c, nc, nimbleFile) - elif amb: - error c, args[0], "ambiguous .nimble file" else: - error c, args[0], "cannot find .nimble file" + installDependencies(c, nc, nimbleFile.get()) of "pin": optSingleArg(LockFileName) @@ -500,16 +497,16 @@ proc main(c: var AtlasContext) = # projectCmd() if args.len > 1: fatal "install command takes a single argument" - var nimbleFile = "" + var nimbleFile: Option[string] if args.len == 1: - nimbleFile = args[0] + nimbleFile = some args[0] else: - nimbleFile = findCurrentNimble() - if nimbleFile.len == 0: + nimbleFile = findNimbleFile(c, getCurrentDir()) + if nimbleFile.isNone: fatal "could not find a .nimble file" else: var nc = createNimbleContext(c, c.depsDir) - installDependencies(c, nc, nimbleFile) + installDependencies(c, nc, nimbleFile.get()) of "refresh": noArgs() updatePackages(c, c.depsDir) diff --git a/src/cloner.nim b/src/cloner.nim index cf882c5..21f23da 100644 --- a/src/cloner.nim +++ b/src/cloner.nim @@ -53,9 +53,11 @@ proc cloneUrl*(c: var AtlasContext, infoNow c, url.projectName, "Cloning url: " & modurl # Checking repo with git + trace c, "atlas cloner", "checking repo " & $url let gitCmdStr = "git ls-remote --quiet --tags " & modurl var success = execCmdEx(gitCmdStr)[1] == QuitSuccess if not success and isGitHub: + trace c, "atlas cloner", "failed check ls-remote..." # retry multiple times to avoid annoying GitHub timeouts: success = retryUrl(gitCmdStr, modurl, c, url.projectName, false) @@ -74,215 +76,8 @@ proc cloneUrl*(c: var AtlasContext, (OtherError, "exernal program failed: " & hgCmdStr) else: if gitops.clone(c, url.url, dest, fullClones=true): # gitops.clone has buit-in retrying + infoNow c, url.projectName, "Cloning success" (Ok, "") else: + infoNow c, url.projectName, "Cloning failed: " & modurl (OtherError, "exernal program failed: " & $GitClone) - -when false: - proc updatePackages*(c: var AtlasContext) = - if dirExists(c.depsDir / DefaultPackagesSubDir): - withDir(c, c.depsDir / DefaultPackagesSubDir): - gitPull(c, DefaultPackagesSubDir) - else: - withDir c, c.depsDir: - let (status, err) = cloneUrl(c, PkgUrl"https://github.com/nim-lang/packages", DefaultPackagesSubDir, false) - if status != Ok: - error c, DefaultPackagesSubDir, err - - proc fillPackageLookupTable(c: var AtlasContext) = - if not c.hasPackageList: - c.hasPackageList = true - if not fileExists(c.depsDir / DefaultPackagesSubDir / "packages.json"): - updatePackages(c) - let plist = getPackageInfos(c.depsDir) - debug c, "fillPackageLookupTable", "initializing..." - for entry in plist: - let url = getUrl(entry.url) - let pkg = Package(name: PackageName unicode.toLower entry.name, - repo: PackageRepo lastPathComponent($url), - url: url) - c.urlMapping["name:" & pkg.name.string] = pkg - -when false: - proc dependencyDir*(c: var AtlasContext; pkg: Package): PackageDir = - template checkDir(dir: string) = - if dir.len > 0 and dirExists(dir): - debug c, pkg, "dependencyDir: found: " & dir - return PackageDir dir - else: - debug c, pkg, "dependencyDir: not found: " & dir - - debug c, pkg, "dependencyDir: check: pth: " & pkg.path.string & " cd: " & getCurrentDir() & " ws: " & c.workspace - if pkg.exists: - debug c, pkg, "dependencyDir: exists: " & pkg.path.string - return PackageDir pkg.path.string.absolutePath - if c.workspace.lastPathComponent == pkg.repo.string: - debug c, pkg, "dependencyDir: workspace: " & c.workspace - return PackageDir c.workspace - - if pkg.path.string.len > 0: - checkDir pkg.path.string - checkDir c.workspace / pkg.path.string - checkDir c.depsDir / pkg.path.string - - checkDir c.workspace / pkg.repo.string - checkDir c.depsDir / pkg.repo.string - checkDir c.workspace / pkg.name.string - checkDir c.depsDir / pkg.name.string - result = PackageDir c.depsDir / pkg.repo.string - trace c, pkg, "dependency not found using default" - - proc findNimbleFile*(c: var AtlasContext; pkg: Package; depDir = PackageDir""): string = - let dir = if depDir.string.len == 0: dependencyDir(c, pkg).string - else: depDir.string - result = dir / (pkg.name.string & ".nimble") - debug c, pkg, "findNimbleFile: searching: " & pkg.repo.string & " path: " & pkg.path.string & " dir: " & dir & " curr: " & result - if not fileExists(result): - debug c, pkg, "findNimbleFile: not found: " & result - result = "" - for file in walkFiles(dir / "*.nimble"): - if result.len == 0: - result = file - trace c, pkg, "nimble file found " & result - else: - error c, pkg, "ambiguous .nimble file " & result - return "" - else: - trace c, pkg, "nimble file found " & result - -when false: - proc resolvePackageUrl(c: var AtlasContext; url: string, checkOverrides = true): Package = - result = Package(url: getUrl(url), - name: url.toRepo().PackageName, - repo: url.toRepo()) - - debug c, result, "resolvePackageUrl: search: " & url - - let isFile = result.url.scheme == "file" - var isUrlOverriden = false - if not isFile and checkOverrides and UsesOverrides in c.flags: - let url = c.overrides.substitute($result.url) - if url.len > 0: - warn c, result, "resolvePackageUrl: url override found: " & $url - result.url = url.getUrl() - isUrlOverriden = true - - let namePkg = c.urlMapping.getOrDefault("name:" & result.name.string, nil) - let repoPkg = c.urlMapping.getOrDefault("repo:" & result.repo.string, nil) - - if not namePkg.isNil: - debug c, result, "resolvePackageUrl: found by name: " & $result.name.string - if namePkg.url != result.url and isUrlOverriden: - namePkg.url = result.url # update package url to match - result = namePkg - elif namePkg.url != result.url: - # package conflicts - # change package repo to `repo.user.host` - let purl = result.url - let host = purl.hostname - let org = purl.path.parentDir.lastPathPart - let rname = purl.path.lastPathPart - let pname = [rname, org, host].join(".") - warn c, result, - "conflicting url's for package; renaming package: " & - result.name.string & " to " & pname - result.repo = PackageRepo pname - c.urlMapping["name:" & result.name.string] = result - else: - result = namePkg - elif not repoPkg.isNil: - debug c, result, "resolvePackageUrl: found by repo: " & $result.repo.string - result = repoPkg - else: - # package doesn't exit and doesn't conflict - # set the url with package name as url name - c.urlMapping["repo:" & result.name.string] = result - trace c, result, "resolvePackageUrl: not found; set pkg: " & $result.repo.string - - #if result.url.scheme == "file": - # result.path = PackageDir result.url.hostname & result.url.path - # trace c, result, "resolvePackageUrl: setting manual path: " & $result.path.string - - proc resolvePackageName(c: var AtlasContext; name: string): Package = - result = Package(name: PackageName name, - repo: PackageRepo name) - - # the project name can be overwritten too! - if UsesOverrides in c.flags: - let name = c.overrides.substitute(name) - if name.len > 0: - if name.isUrl(): - return c.resolvePackageUrl(name, checkOverrides=false) - - # echo "URL MAP: ", repr c.urlMapping.keys().toSeq() - let namePkg = c.urlMapping.getOrDefault("name:" & result.name.string, nil) - let repoPkg = c.urlMapping.getOrDefault("repo:" & result.name.string, nil) - - debug c, result, "resolvePackageName: searching for package name: " & result.name.string - if not namePkg.isNil: - # great, found package! - debug c, result, "resolvePackageName: found!" - result = namePkg - result.inPackages = true - elif not repoPkg.isNil: - # check if rawHandle is a package repo name - debug c, result, "resolvePackageName: found by repo!" - result = repoPkg - result.inPackages = true - - if UsesOverrides in c.flags: - let newUrl = c.overrides.substitute($result.url) - if newUrl.len > 0: - trace c, result, "resolvePackageName: not url: UsesOverrides: " & $newUrl - result.url = getUrl newUrl - - proc resolvePackage*(c: var AtlasContext; rawHandle: string): Package = - ## Takes a raw handle which can be a name, a repo name, or a url - ## and resolves it into a package. If not found it will create - ## a new one. - ## - ## Note that Package should be unique globally. This happens - ## by updating the packages list when new packages are added or - ## loaded from a packages.json. - ## - result = Package() - - fillPackageLookupTable(c) - - trace c, rawHandle, "resolving package" - - if rawHandle.isUrl: - result = c.resolvePackageUrl(rawHandle) - else: - result = c.resolvePackageName(unicode.toLower(rawHandle)) - - result.path = dependencyDir(c, result) - let res = c.findNimbleFile(result, result.path) - if res.len > 0: - let nimble = PackageNimble res - result.exists = true - result.nimble = nimble - # the nimble package name is .nimble - result.name = PackageName nimble.string.splitFile().name - debug c, result, "resolvePackageName: nimble: found: " & $result - else: - debug c, result, "resolvePackageName: nimble: not found: " & $result - - - proc resolveNimble*(c: var AtlasContext; pkg: Package) = - ## Try to resolve the nimble file for the given package. - ## - ## This should be done after cloning a new repo. - ## - if pkg.exists: return - - pkg.path = dependencyDir(c, pkg) - let res = c.findNimbleFile(pkg) - if res.len > 0: - let nimble = PackageNimble res - # let path = PackageDir res.parentDir() - pkg.exists = true - pkg.nimble = nimble - info c, pkg, "resolvePackageName: nimble: found: " & $pkg - else: - info c, pkg, "resolvePackageName: nimble: not found: " & $pkg diff --git a/src/confighandler.nim b/src/confighandler.nim index ae06b7f..3eedfd0 100644 --- a/src/confighandler.nim +++ b/src/confighandler.nim @@ -52,7 +52,11 @@ type graph: JsonNode proc writeDefaultConfigFile*(c: var AtlasContext) = - let config = JsonConfig(resolver: $SemVer, graph: newJNull()) + let config = JsonConfig( + resolver: $SemVer, + graph: newJNull(), + deps: c.depsDir + ) let configFile = c.workspace / AtlasWorkspace writeFile(configFile, pretty %*config) diff --git a/src/context.nim b/src/context.nim index e54ab6b..ca8ceaa 100644 --- a/src/context.nim +++ b/src/context.nim @@ -67,7 +67,7 @@ template projectFromCurrentDir*(): untyped = c.currentDir.absolutePath template withDir*(c: var AtlasContext; dir: string; body: untyped) = let oldDir = getCurrentDir() - debug c, dir, "Current directory is now: " & dir + debug c, dir.relativePath(c.workspace), "Current directory is now: " & dir try: setCurrentDir(dir) body diff --git a/src/depgraphs.nim b/src/depgraphs.nim index 95757bf..128e4f2 100644 --- a/src/depgraphs.nim +++ b/src/depgraphs.nim @@ -6,7 +6,7 @@ # distribution, for details about the copyright. # -import std / [sets, tables, os, strutils, streams, json, jsonutils, algorithm] +import std / [os, sets, sequtils, tables, strutils, streams, json, jsonutils, algorithm, options] import context, gitops, runners, reporters, nimbleparser, pkgurls, cloner, versions @@ -32,6 +32,7 @@ type status: CloneStatus activeVersion*: int ondisk*: string + nimbleFile*: Option[string] DepGraph* = object nodes: seq[Dependency] @@ -64,21 +65,29 @@ proc readOnDisk(c: var AtlasContext; result: var DepGraph) = if n.isRoot: if not result.packageToDependency.hasKey(n.pkg): result.packageToDependency[n.pkg] = result.nodes.len - result.nodes.add Dependency(pkg: n.pkg, versions: @[], isRoot: true, isTopLevel: n.isTopLevel, activeVersion: -1) + result.nodes.add Dependency( + pkg: n.pkg, + versions: @[], + isRoot: true, + isTopLevel: n.isTopLevel, + activeVersion: -1, + nimbleFile: c.findNimbleFile(n.pkg, n.ondisk) + ) except: error c, configFile, "cannot read: " & configFile -proc createGraph*(c: var AtlasContext; s: PkgUrl): DepGraph = - result = DepGraph(nodes: @[], - reqs: defaultReqs()) +proc createGraph*(c: var AtlasContext; s: PkgUrl, readConfig = true): DepGraph = + result = DepGraph(nodes: @[], reqs: defaultReqs()) result.packageToDependency[s] = result.nodes.len result.nodes.add Dependency(pkg: s, versions: @[], isRoot: true, isTopLevel: true, activeVersion: -1) - readOnDisk(c, result) + if readConfig: + readOnDisk(c, result) proc toJson*(d: DepGraph): JsonNode = result = newJObject() - result["nodes"] = toJson(d.nodes) - result["reqs"] = toJson(d.reqs) + let jopts = ToJsonOptions(enumMode: joptEnumSymbol) + result["nodes"] = toJson(d.nodes, jopts) + result["reqs"] = toJson(d.reqs, jopts) proc createGraphFromWorkspace*(c: var AtlasContext): DepGraph = result = DepGraph(nodes: @[], reqs: defaultReqs()) @@ -109,10 +118,14 @@ type CommitOrigin = enum FromHead, FromGitTag, FromDep, FromNimbleFile -iterator releases(c: var AtlasContext; m: TraversalMode; versions: seq[DependencyVersion]; +iterator releases(c: var AtlasContext; + m: TraversalMode; + pkg: PkgUrl; + versions: seq[DependencyVersion]; nimbleCommits: seq[string]): (CommitOrigin, Commit) = - let (cc, status) = exec(c, GitCurrentCommit, []) - if status == 0: + let cc = c.getCurrentCommit() + trace c, pkg.projectName, "iterating commit " & $cc + if cc.isSome: case m of AllReleases: try: @@ -141,23 +154,25 @@ iterator releases(c: var AtlasContext; m: TraversalMode; versions: seq[Dependenc if produced == 0: yield (FromHead, Commit(h: "", v: Version"#head")) + except Exception as err: + echo "Atlas traverseDependency error: " + error c, pkg.projectName, "error: " & $err.msg + except Defect as err: + echo "Atlas traverseDependency Defect: " + error c, pkg.projectName, "error: " & $err.msg + except CatchableError as err: + echo "Atlas traverseDependency catchable error: " + error c, pkg.projectName, "error: " & $err.msg + finally: - discard exec(c, GitCheckout, [cc]) + # discard exec(c, GitCheckout, [cc.get()]) + trace c, pkg.projectName, "attempt to checkout out " & cc.get() + c.checkoutGitCommit(pkg.projectName, cc.get()) of CurrentCommit: yield (FromHead, Commit(h: "", v: Version"#head")) else: yield (FromHead, Commit(h: "", v: Version"#head")) -proc findNimbleFile(g: DepGraph; idx: int): (string, int) = - var nimbleFile = g.nodes[idx].pkg.projectName & ".nimble" - var found = 0 - if fileExists(nimbleFile): - inc found - else: - for file in walkFiles("*.nimble"): - nimbleFile = file - inc found - result = (ensureMove nimbleFile, found) proc enrichVersionsViaExplicitHash(versions: var seq[DependencyVersion]; x: VersionInterval) = let commit = extractSpecificCommit(x) @@ -167,28 +182,31 @@ proc enrichVersionsViaExplicitHash(versions: var seq[DependencyVersion]; x: Vers versions.add DependencyVersion(version: Version"", commit: commit, req: EmptyReqs, v: NoVar) -proc collectNimbleVersions*(c: var AtlasContext; nc: NimbleContext; g: var DepGraph; idx: int): seq[string] = - let (outerNimbleFile, found) = findNimbleFile(g, idx) +proc collectNimbleVersions*(c: var AtlasContext; dep: var Dependency): seq[string] = result = @[] - if found == 1: - let (outp, status) = exec(c, GitLog, [outerNimbleFile]) - if status == 0: - for line in splitLines(outp): - if line.len > 0 and not line.endsWith("^{}"): - result.add line - result.reverse() + if dep.nimbleFile.isSome: + result = gitops.lookupFileHashes(c, dep.nimbleFile.get()) + debug c, dep.pkg.projectName, "nimble commit versions: " & $result proc traverseRelease(c: var AtlasContext; nc: NimbleContext; g: var DepGraph; idx: int; origin: CommitOrigin; r: Commit; lastNimbleContents: var string) = - let (nimbleFile, found) = findNimbleFile(g, idx) + let pkg = g.nodes[idx].pkg + let nimbleFile = g.nodes[idx].nimbleFile + debug c, pkg.projectName, "traverseRelease: origin: " & $origin & " commit: " & $r var pv = DependencyVersion( version: r.v, commit: r.h, - req: EmptyReqs, v: NoVar) + req: EmptyReqs, + v: NoVar + ) var badNimbleFile = false - if found != 1: + if nimbleFile.isNone: + pv.req = UnknownReqs + elif not nimbleFile.get().fileExists(): + badNimbleFile = false pv.req = UnknownReqs else: + let nimbleFile = nimbleFile.get() when (NimMajor, NimMinor, NimPatch) == (2, 0, 0): # bug #110; make it compatible with Nim 2.0.0 # ensureMove requires mutable places when version < 2.0.2 @@ -234,10 +252,12 @@ proc traverseDependency(c: var AtlasContext; nc: NimbleContext; g: var DepGraph; var lastNimbleContents = "" let versions = move g.nodes[idx].versions - let nimbleVersions = collectNimbleVersions(c, nc, g, idx) + let nimbleVersions = collectNimbleVersions(c, g.nodes[idx]) - for (origin, r) in releases(c, m, versions, nimbleVersions): + for (origin, r) in releases(c, m, g.nodes[idx].pkg, versions, nimbleVersions): + debug c, g.nodes[idx].pkg.projectName, "traverseDependency: " & $c.getCurrentCommit() traverseRelease c, nc, g, idx, origin, r, lastNimbleContents + debug c, g.nodes[idx].pkg.projectName, "traverseDependencies: " & $g.nodes[idx].versions.mapIt($it.version & " <- " & ($(it.commit & "000000")[0..5])) const FileWorkspace = "file://./" @@ -264,9 +284,10 @@ type PackageAction = enum DoNothing, DoClone -proc pkgUrlToDirname(c: var AtlasContext; g: var DepGraph; d: Dependency): (string, PackageAction) = +proc pkgUrlToDirname(c: var AtlasContext; g: var DepGraph; d: var Dependency): (string, PackageAction) = # XXX implement namespace support here var dest = g.ondisk.getOrDefault(d.pkg.url) + trace c, d.pkg.projectName, "using dirname: " & $dest & " for url: " & $d.pkg.url if dest.len == 0: if d.isTopLevel: dest = c.workspace @@ -285,6 +306,7 @@ proc expand*(c: var AtlasContext; g: var DepGraph; nc: NimbleContext; m: Travers while i < g.nodes.len: if not processed.containsOrIncl(g.nodes[i].pkg): let (dest, todo) = pkgUrlToDirname(c, g, g.nodes[i]) + trace c, $g.nodes[i].pkg.projectName, "expanded destination dir: " & $dest g.nodes[i].ondisk = dest if todo == DoClone: let (status, _) = @@ -295,6 +317,7 @@ proc expand*(c: var AtlasContext; g: var DepGraph; nc: NimbleContext; m: Travers g.nodes[i].status = status if g.nodes[i].status == Ok: + g.nodes[i].nimbleFile = c.findNimbleFile(g.nodes[i].pkg, dest) withDir c, dest: traverseDependency(c, nc, g, i, m) inc i @@ -455,13 +478,15 @@ proc runBuildSteps(c: var AtlasContext; g: var DepGraph) = let activeVersion = g.nodes[i].activeVersion let r = if g.nodes[i].versions.len == 0: -1 else: g.nodes[i].versions[activeVersion].req if r >= 0 and r < g.reqs.len and g.reqs[r].hasInstallHooks: - let (nf, found) = findNimbleFile(g, i) - if found == 1: - runNimScriptInstallHook c, nf, pkg.projectName + let nf = c.findNimbleFile(g.nodes[i].pkg, getCurrentDir()) + if nf.isSome: + trace c, pkg.projectName, "running Nimble install hook" + runNimScriptInstallHook c, nf.get, pkg.projectName # check for nim script builders for p in mitems c.plugins.builderPatterns: let f = p[0] % pkg.projectName if fileExists(f): + trace c, pkg.projectName, "running NimScript builder" runNimScriptBuilder c, p, pkg.projectName proc debugFormular(c: var AtlasContext; g: var DepGraph; f: Form; s: Solution) = @@ -478,9 +503,17 @@ proc debugFormular(c: var AtlasContext; g: var DepGraph; f: Form; s: Solution) = proc solve*(c: var AtlasContext; g: var DepGraph; f: Form) = let m = f.idgen var s = createSolution(m) - #debugFormular c, g, f, s - if satisfiable(f.f, s): + var status = + try: satisfiable(f.f, s) + except SatOverflowError as err: + echo "\n" + echo "SAT MaxIterationLimitError: " + debugFormular c, g, f, s + echo "\n" + raise err + + if status: for n in mitems g.nodes: if n.isRoot: n.active = true for i in 0 ..< m: diff --git a/src/gitops.nim b/src/gitops.nim index c9aeadc..be9f939 100644 --- a/src/gitops.nim +++ b/src/gitops.nim @@ -6,7 +6,7 @@ # distribution, for details about the copyright. # -import std/[os, osproc, sequtils, strutils] +import std/[os, osproc, sequtils, strutils, options, algorithm] import reporters, osutils, versions type @@ -83,15 +83,30 @@ proc clone*(c: var Reporter; url, dest: string; retries = 5; fullClones=false): else: "" let cmd = $GitClone & " " & extraArgs & " " & quoteShell(url) & " " & dest + debug c, url, "cloning using command: " & cmd for i in 1..retries: - if execShellCmd(cmd) == 0: + let res = execShellCmd(cmd) + debug c, url, "cloning status: " & $res + if res == 0 and dirExists(dest): return true - os.sleep(i*2_000) + trace c, url, "trying to clone again" + sleep(i*2_000) proc gitDescribeRefTag*(c: var Reporter; commit: string): string = let (lt, status) = exec(c, GitDescribe, ["--tags", commit]) result = if status == 0: strutils.strip(lt) else: "" +proc lookupFileHashes*(c: var Reporter; nimbleFile: string): seq[string] = + result = @[] + let (outp, status) = exec(c, GitLog, [nimbleFile]) + if status == 0: + for line in splitLines(outp): + if line.len > 0 and not line.endsWith("^{}"): + result.add line + else: + warn c, nimbleFile, "error looking up git hash history: " & outp + result.reverse() + proc getLastTaggedCommit*(c: var Reporter): string = let (ltr, status) = exec(c, GitLastTaggedRef, []) if status == 0: @@ -129,18 +144,23 @@ proc listFiles*(c: var Reporter): seq[string] = result = @[] proc checkoutGitCommit*(c: var Reporter; p, commit: string) = + debug(c, p, "checking out commit " & commit) let (currentCommit, statusA) = exec(c, GitCurrentCommit, []) - if statusA == 0 and currentCommit.strip() == commit: return + if statusA == 0 and currentCommit.strip() == commit: + info(c, p, "updated package to " & commit) + return - let (_, statusB) = exec(c, GitCheckout, [commit]) + let (outp, statusB) = exec(c, GitCheckout, [commit]) if statusB != 0: - error(c, p, "could not checkout commit " & commit) + error(c, p, "could not checkout commit " & commit & " error: " & $outp) else: info(c, p, "updated package to " & commit) proc checkoutGitCommitFull*(c: var Reporter; p, commit: string; fullClones: bool) = var smExtraArgs: seq[string] = @[] + trace(c, p, "attempt to checkout out package commit " & commit) + if not fullClones and commit.len == 40: smExtraArgs.add "--depth=1" @@ -207,6 +227,7 @@ proc incrementTag*(c: var Reporter; displayName, lastTag: string; field: Natural lastTag[0.. 0 and nimblePath.fileExists(): + let nimblePath = findNimbleFile(c, startPkg) + if nimblePath.isSome: lf.nimbleFile = LockedNimbleFile( - filename: nimblePath.relativePath(c.currentDir), - content: readFile(nimblePath).splitLines()) + filename: nimblePath.get().relativePath(c.currentDir), + content: readFile(nimblePath.get()).splitLines()) if not exportNimble: write lf, lockFilePath @@ -278,7 +279,7 @@ proc listChanged*(c: var AtlasContext; lockFilePath: string) = " found: " & url & " lockfile has: " & v.url - let commit = gitops.getCurrentCommit() + let commit = c.getCurrentCommit().get() if commit != v.commit: #let info = parseNimble(c, pkg.nimble) warn c, dir, "commit differs;" & diff --git a/src/nimbleparser.nim b/src/nimbleparser.nim index 53aaec2..ebbe5dd 100644 --- a/src/nimbleparser.nim +++ b/src/nimbleparser.nim @@ -6,9 +6,11 @@ # distribution, for details about the copyright. # -import std / [os, strutils, tables, unicode, hashes] +import std / [os, strutils, tables, unicode, hashes, options] import versions, packagesjson, reporters, gitops, parse_requires, pkgurls, compiledpatterns +export options + when defined(nimAtlasBootstrap): import ../dist/sat/src/sat/satvars else: @@ -49,9 +51,13 @@ proc `==`*(a, b: Requirements): bool = proc updatePackages*(c: var Reporter; depsDir: string) = if dirExists(depsDir / DefaultPackagesSubDir): withDir(c, depsDir / DefaultPackagesSubDir): + trace c, getCurrentDir(), "updating packages" + echo "TRY UPDATING PACKAGES" gitPull(c, DefaultPackagesSubDir) else: withDir c, depsDir: + trace c, getCurrentDir(), "cloning packages" + echo "TRY CLONING PACKAGES" let success = clone(c, "https://github.com/nim-lang/packages", DefaultPackagesSubDir) if not success: error c, DefaultPackagesSubDir, "cannot clone packages repo" @@ -115,20 +121,43 @@ proc parseNimbleFile*(c: NimbleContext; nimbleFile: string; p: Patterns): Requir else: result.deps.add (createUrlSkipPatterns(u), query) -proc findNimbleFile*(c: var Reporter; dir: string; ambiguous: var bool): string = - result = "" - var counter = 0 - for x in walkFiles(dir / "*.nimble"): - inc counter - if result.len == 0: - result = x - if counter > 1: - ambiguous = true - result = "" - -# if counter > 1: -# warn c, dir, "cannot determine `.nimble` file; there are multiple to choose from" -# result = "" +proc findNimbleFile*(c: var Reporter, dir: string): Option[string] = + ## Search the given directory for a Nimble file. + ## + ## An error is reported if there are multiple Nimble files + ## which results in an ambiguity. + var nimbleFile = "" + var found = 0 + + for file in walkFiles(dir / "*.nimble"): + nimbleFile = file + found.inc + + if found > 1: + error c, dir, "ambiguous .nimble files; found: " & $found & " options" + result = string.none() + elif found == 0: + warn c, dir, "no nimble file found" + result = string.none() + else: + result = some(nimbleFile.absolutePath()) + debug c, dir, "findNimbleFile: found: " & nimbleFile + +proc findNimbleFile*(c: var Reporter, pkg: PkgUrl, dir: string): Option[string] = + ## Search the given directory for a Nimble file starting with the + ## name from the package's URL. + ## + ## An error is reported if there are multiple Nimble files + ## which results in an ambiguity. + var nimbleFile = pkg.projectName & ".nimble" + debug c, pkg.projectName, "findNimbleFile: searching: " & pkg.projectName & + " path: " & dir + assert dir != "" + if fileExists(dir / nimbleFile): + debug c, pkg.projectName, "findNimbleFile: found: " & nimbleFile + some((dir / nimbleFile).absolutePath()) + else: + findNimbleFile(c, dir) proc genRequiresLine(u: string): string = "requires \"$1\"\n" % u.escape("", "") diff --git a/src/parse_requires.nim b/src/parse_requires.nim index 5ede51f..5aa0b8b 100644 --- a/src/parse_requires.nim +++ b/src/parse_requires.nim @@ -32,7 +32,7 @@ proc extract(n: PNode; conf: ConfigRef; result: var NimbleFileInfo) = if ch.kind in {nkStrLit..nkTripleStrLit}: result.requires.add ch.strVal else: - localError(conf, ch.info, "'requires' takes string literals") + localError(conf, ch.info, warnUser, "'requires' takes string literals") result.hasErrors = true of "task": if n.len >= 3 and n[1].kind == nkIdent and n[2].kind in {nkStrLit..nkTripleStrLit}: @@ -54,13 +54,13 @@ proc extract(n: PNode; conf: ConfigRef; result: var NimbleFileInfo) = if n[1].kind in {nkStrLit..nkTripleStrLit}: result.srcDir = n[1].strVal else: - localError(conf, n[1].info, "assignments to 'srcDir' must be string literals") + localError(conf, n[1].info, warnUser, "assignments to 'srcDir' must be string literals") result.hasErrors = true elif n[0].kind == nkIdent and eqIdent(n[0].ident.s, "version"): if n[1].kind in {nkStrLit..nkTripleStrLit}: result.version = n[1].strVal else: - localError(conf, n[1].info, "assignments to 'version' must be string literals") + localError(conf, n[1].info, warnUser, "assignments to 'version' must be string literals") result.hasErrors = true else: discard diff --git a/src/pkgurls.nim b/src/pkgurls.nim index 9183a51..f9d88dc 100644 --- a/src/pkgurls.nim +++ b/src/pkgurls.nim @@ -56,9 +56,3 @@ proc `==`*(a, b: PkgUrl): bool {.inline.} = a.u == b.u proc hash*(a: PkgUrl): Hash {.inline.} = hash(a.u) proc isFileProtocol*(s: PkgUrl): bool = s.u.startsWith("file://") - -proc dir*(s: PkgUrl): string = - if isFileProtocol(s): - result = substr(s.u, len("file://")) - else: - result = s.projectName diff --git a/tests/nim.cfg b/tests/nim.cfg deleted file mode 100644 index 3982b12..0000000 --- a/tests/nim.cfg +++ /dev/null @@ -1,10 +0,0 @@ -############# begin Atlas config section ########## ---noNimblePath ---path:"../balls" ---path:"../grok" ---path:"../ups" ---path:"../sync" ---path:"../npeg/src" ---path:"../testes" ---path:"../nim-bytes2human/src" -############# end Atlas config section ########## diff --git a/tests/setups.nim b/tests/setups.nim index cdde3ca..668ffc6 100644 --- a/tests/setups.nim +++ b/tests/setups.nim @@ -1,5 +1,8 @@ import std / os +import std / tempfiles +export tempfiles +export os proc exec*(cmd: string) = if execShellCmd(cmd) != 0: @@ -13,6 +16,38 @@ template withDir*(dir: string; body: untyped) = finally: setCurrentDir(old) +const removeTempDirs {.booldefine.} = true + +template withTempTestDirFull*(name: string, remove: bool, blk: untyped) = + ## creates test dir and optionally remove dir afterwords + let old = getCurrentDir() + let dir {.inject.} = createTempDir("atlas_test_", "_" & name) + echo "Creating temp test directory: ", dir + try: + setCurrentDir(dir) + `blk` + finally: + setCurrentDir(old) + if remove: + removeDir(dir) + +template withTempTestDir*(name: string, blk: untyped) = + withTempTestDirFull(name, removeTempDirs, blk) + +template withTempTestDirFrom*(name: string, src: string, blk: untyped) = + ## creates test dir by copying from a template folder + let old = getCurrentDir() + let dir {.inject.} = genTempPath("atlas_test_", "_" & name) + echo "Creating temp test directory: ", dir + copyDir(src, dir) + try: + setCurrentDir(dir) + `blk` + finally: + setCurrentDir(old) + if removeTempDirs: + removeDir(dir) + proc buildGraph* = createDir "source" withDir "source": diff --git a/tests/tester.nim b/tests/tester.nim index ae7343e..efd1c7f 100644 --- a/tests/tester.nim +++ b/tests/tester.nim @@ -9,11 +9,10 @@ if execShellCmd("nim c -d:debug -r tests/unittests.nim") != 0: var failures = 0 -let atlasExe = absolutePath("bin" / "atlas".addFileExt(ExeExt)) -if execShellCmd("nim c -o:$# -d:release src/atlas.nim" % [atlasExe]) != 0: +var atlasExe = absolutePath("bin" / "atlas".addFileExt(ExeExt)) +if execShellCmd("nim c -o:$# -d:debug src/atlas.nim" % [atlasExe]) != 0: quit("FAILURE: compilation of atlas failed") - proc sameDirContents(expected, given: string): bool = result = true for _, e in walkDir(expected): @@ -70,24 +69,31 @@ const end of selection """ +template withTestDir(dir: string, desc: string, blk: untyped) = + echo "\n# Running test ", desc, " in dir: ", dir, "\n" + withDir dir: + `blk` + echo "\n## Finished running test in dir: ", dir, "\n" + proc testSemVer2(expected: string) = createDir "semproject" withDir "semproject": - let cmd = atlasExe & " --full --keepWorkspace --resolver=SemVer --colors:off --list use proj_a" + echo "## Running test in subdir `semproject`" + let cmd = atlasExe & " --verbosity:trace --full --keepWorkspace --resolver=SemVer --colors:off --list use proj_a" let (outp, status) = execCmdEx(cmd) - if status == 0: - if outp.contains expected: - discard "fine" - else: - echo "expected ", expected, " but got ", outp - raise newException(AssertionDefect, "Test failed!") + if outp.contains expected: + echo "# Test success" + echo outp else: - echo "\n\n<<<<<<<<<<<<<<<< failed " - echo "testSemVer2:command: ", cmd - echo "testSemVer2:pwd: ", getCurrentDir() - echo "testSemVer2:failed command:\n", outp - echo ">>>>>>>>>>>>>>>> failed\n" - assert false, "testSemVer2" + echo "" + echo "# Failure error code: ", status + echo "## Expected:\n", expected + echo "" + echo "## Actual:\n", outp + echo "" + echo "" + raise newException(AssertionDefect, "Test failed!") + echo "### Done Running test in subdir `semproject`" proc testMinVer() = buildGraph() @@ -103,7 +109,7 @@ proc testMinVer() = else: assert false, outp -withDir "tests/ws_semver2": +withTestDir "tests/ws_semver2", "semver2 plain": try: buildGraph() testSemVer2(SemVerExpectedResult) @@ -117,7 +123,7 @@ withDir "tests/ws_semver2": removeDir "proj_c" removeDir "proj_d" -withDir "tests/ws_semver2": +withTestDir "tests/ws_semver2", "semver2 no git tags": try: buildGraphNoGitTags() testSemVer2(SemVerExpectedResultNoGitTags) diff --git a/tests/unit_tests/config.nims b/tests/unit_tests/config.nims new file mode 100644 index 0000000..b656e8b --- /dev/null +++ b/tests/unit_tests/config.nims @@ -0,0 +1,5 @@ +--path:"../../src" +--path:"$nim" +--d:atlasStandAlone +--d:atlasUnitTests +--d:atlasNoUnitTestFiles diff --git a/tests/unit_tests/testCollectVersions.nim b/tests/unit_tests/testCollectVersions.nim new file mode 100644 index 0000000..5de33ee --- /dev/null +++ b/tests/unit_tests/testCollectVersions.nim @@ -0,0 +1,48 @@ + +import std/unittest +import std/strutils +import std/os +import std/tempfiles +import std/options + +import ../setups + +import context, reporters, nimbleparser, pkgurls +import compiledpatterns +import compiledpatterns +import pkgurls +import depgraphs + +proc toDirSep(s: string): string = + result = s.replace("/", $DirSep) + +template setupDepsAndGraph(dir: string) = + var + p {.inject.} = initPatterns() + u {.inject.} = createUrl("file://" & dir, p) + c {.inject.} = AtlasContext() + g {.inject.} = createGraph(c, u, readConfig = false) + + c.depsDir = "source" + c.workspace = dir.toDirSep + c.projectDir = dir.toDirSep + c.verbosity = 3 + +suite "test pkgurls": + + test "basic url": + withTempTestDir "basic_url": + buildGraphNoGitTags() + echo "\n" + setupDepsAndGraph(dir) + var d = Dependency() + let depDir = "source" / "proj_a/" + ## TODOX: how to handle this relative or not thing? + let nimble = "proj_a.nimble" + setCurrentDir(depDir) + d.pkg = createUrl("file://" & $depDir, p) + d.nimbleFile = some nimble + echo "D: ", d + let versions = collectNimbleVersions(c, d) + echo "VERSIONS: ", versions + diff --git a/tests/unit_tests/testCollectVersions.nims b/tests/unit_tests/testCollectVersions.nims new file mode 100644 index 0000000..cfda6f2 --- /dev/null +++ b/tests/unit_tests/testCollectVersions.nims @@ -0,0 +1,9 @@ +--noNimblePath +--path:"../../src" +--path:"$nim" +--d:atlasStandAlone +--d:atlasUnitTests +--d:atlasNoUnitTestFiles + +## todo remove? +--d:"removeTempDirs:false" \ No newline at end of file diff --git a/tests/unit_tests/testpkgurls.nim b/tests/unit_tests/testpkgurls.nim new file mode 100644 index 0000000..4d89a0f --- /dev/null +++ b/tests/unit_tests/testpkgurls.nim @@ -0,0 +1,108 @@ + +import std/unittest +import std/strutils +import std/paths +import std/options + +import ../setups + +import context, reporters, nimbleparser +import compiledpatterns +import pkgurls +import depgraphs + +proc toDirSep(s: string): string = + result = s.replace("/", $DirSep) + +template setupDepsAndGraph(url: string) = + var + p {.inject.} = initPatterns() + u {.inject.} = createUrl(url, p) + c {.inject.} = AtlasContext() + g {.inject.} = createGraph(c, u, readConfig = false) + d {.inject.} = Dependency() + + c.depsDir = "fakeDeps" + c.workspace = "/workspace/".toDirSep + c.projectDir = "/workspace".toDirSep + +template withProjectDir*(names: varargs[string], blk: untyped) = + createDir "workspace" + setCurrentDir("workspace") + c.workspace = os.getCurrentDir() + for name in names: + createDir name + setCurrentDir(name) + c.projectDir = os.getCurrentDir() + `blk` + discard + when false: + echo "\n<<<<<< CWD: ", os.getCurrentDir() + echo "workspace: ", c.workspace + echo "projectDir: ", c.projectDir, "\n" + discard execShellCmd("find " & dir) + echo ">>>>>>\n\n" + +suite "test pkgurls": + + test "basic url": + setupDepsAndGraph("https://github.com/example/proj.git") + check $u == "https://github.com/example/proj.git" + check u.projectName == "proj" + + test "basic url no git": + setupDepsAndGraph("https://github.com/example/proj") + check $u == "https://github.com/example/proj" + check u.projectName == "proj" + + test "basic url prefix": + setupDepsAndGraph("https://github.com/example/nim-proj") + check $u == "https://github.com/example/nim-proj" + check u.projectName == "nim-proj" + +# var testTemplateDir: string +# withTempTestDirFull("test_template", remove=false): +# buildGraphNoGitTags() +# testTemplateDir = dir + +suite "nimble stuff": + setup: + setupDepsAndGraph("https://github.com/example/nim-testProj1") + + test "find nimble in project dir from project dir": + withTempTestDir "test": + withProjectDir "fakeDeps", "testProj1": + writeFile("testProj1.nimble", "") + let projDir = "workspace" / "fakeDeps" / "testProj1" + let res1 = findNimbleFile(c, u, dir / projDir) + check res1.get().relativePath(dir) == projDir / "testProj1.nimble" + + setCurrentDir(dir / "workspace") + let res2 = findNimbleFile(c, u, dir / projDir) + check res2.get().relativePath(dir) == projDir / "testProj1.nimble" + + test "find nimble in project dir with other name": + withTempTestDir "test": + withProjectDir "fakeDeps", "nim-testProj1": + writeFile("testProj1.nimble", "") + let projDir = "workspace" / "fakeDeps" / "nim-testProj1" + let res = findNimbleFile(c, u, dir / projDir) + check res.get().relativePath(dir) == projDir / "testProj1.nimble" + + test "missing": + withTempTestDir "basic_url": + withProjectDir "fakeDeps", "testProj1": + let projDir = "workspace" / "fakeDeps" / "testProj1" + let res1 = findNimbleFile(c, u, dir / projDir) + check res1.isNone() + + test "ambiguous": + withTempTestDir "basic_url": + withProjectDir "fakeDeps", "testProj1": + writeFile("testProj1.nimble", "") + writeFile("testProj2.nimble", "") + let projDir = "workspace" / "fakeDeps" / "testProj1" + let res = findNimbleFile(c, u, dir / projDir) + check res == string.none + check c.errors == 1 + diff --git a/tests/unittests.nim b/tests/unittests.nim index e32a446..8c44fce 100644 --- a/tests/unittests.nim +++ b/tests/unittests.nim @@ -4,8 +4,7 @@ import std/[unittest, os, strutils] import context, osutils, versions -when false: - from nameresolver import resolvePackage +import unit_tests/testpkgurls let basicExamples = { diff --git a/tests/unittests.nims b/tests/unittests.nims index 5c99662..224d42f 100644 --- a/tests/unittests.nims +++ b/tests/unittests.nims @@ -1,4 +1,3 @@ ---noNimblePath --path:"../src" --path:"$nim" --d:atlasStandAlone