From 4882b252092af2aff91a3edfdb50d5370a2ec86c Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Sat, 30 Mar 2024 14:46:18 +0200 Subject: [PATCH 1/8] Fixes various Nimble and Dependency handling issues This PR started to work on #117 but uncovered a few other items which needed cleaned up or fixed. Changes: Combine multiple ways to search for Nimble files into one place fixes some edge cases in handling finding Nimble files fixes issue with writing default config file and handling the deps cli option osutils.nim provides shims for unit testing proc's using a subset of file operations unit tests for various nimble file states unit tests for parsing PkgUrl's fixes some adhoc git execs to use the gitops module, restoring logging etc --- .github/workflows/ci.yml | 13 +- .gitignore | 1 + config.nims | 2 +- src/atlas.nim | 39 ++-- src/cloner.nim | 213 +--------------------- src/confighandler.nim | 6 +- src/context.nim | 2 +- src/depgraphs.nim | 117 +++++++----- src/gitops.nim | 41 ++++- src/lockfiles.nim | 23 +-- src/nimbleparser.nim | 59 ++++-- src/parse_requires.nim | 6 +- src/pkgurls.nim | 6 - tests/nim.cfg | 10 - tests/setups.nim | 35 ++++ tests/tester.nim | 42 +++-- tests/unit_tests/config.nims | 5 + tests/unit_tests/testCollectVersions.nim | 48 +++++ tests/unit_tests/testCollectVersions.nims | 9 + tests/unit_tests/testpkgurls.nim | 108 +++++++++++ tests/unittests.nim | 3 +- tests/unittests.nims | 1 - 22 files changed, 436 insertions(+), 353 deletions(-) delete mode 100644 tests/nim.cfg create mode 100644 tests/unit_tests/config.nims create mode 100644 tests/unit_tests/testCollectVersions.nim create mode 100644 tests/unit_tests/testCollectVersions.nims create mode 100644 tests/unit_tests/testpkgurls.nim diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae23acc..bf95e62 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,19 +15,19 @@ jobs: target: - os: linux cpu: amd64 - nim_branch: devel + nim_branch: version-2-0 # - os: linux # cpu: i386 # nim_branch: devel - os: macos cpu: amd64 - nim_branch: devel + nim_branch: version-2-0 - os: windows cpu: amd64 - nim_branch: devel + nim_branch: version-2-0 - os: windows cpu: i386 - nim_branch: devel + nim_branch: version-2-0 include: - target: os: linux @@ -52,6 +52,7 @@ jobs: git config --global init.defaultBranch master git config --global user.email "atlasbot@nimlang.com" git config --global user.name "atlasbot" + git config --global gc.auto 0 - name: Checkout atlas uses: actions/checkout@v4 @@ -128,6 +129,10 @@ jobs: version: ${{ matrix.target.nim_branch }} architecture: ${{ matrix.target.cpu }} + - name: Nim Version + run: | + nim -v + - name: Run tests run: | cd atlas diff --git a/.gitignore b/.gitignore index 5fb5da2..ce37c7e 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ tests/* !tests/*.* atlas deps.png +nim.cfg diff --git a/config.nims b/config.nims index 2b42bb2..6bc5dbe 100644 --- a/config.nims +++ b/config.nims @@ -1,6 +1,6 @@ task build, "Build local atlas": - exec "nim c -d:debug -o:./atlas src/atlas.nim" + exec "nim c -d:debug -o:./bin/atlas src/atlas.nim" task unitTests, "Runs unit tests": exec "nim c -d:debug -r tests/unittests.nim" diff --git a/src/atlas.nim b/src/atlas.nim index b39f85a..ba8a66d 100644 --- a/src/atlas.nim +++ b/src/atlas.nim @@ -219,9 +219,12 @@ proc installDependencies(c: var AtlasContext; nc: var NimbleContext; nimbleFile: let (dir, pkgname, _) = splitFile(nimbleFile) info c, pkgname, "installing dependencies for " & pkgname & ".nimble" var g = createGraph(c, createUrlSkipPatterns(dir)) + trace c, pkgname, "traversing depency loop" let paths = traverseLoop(c, nc, g) + trace c, pkgname, "done traversing depencies" let cfgPath = if CfgHere in c.flags: CfgPath c.currentDir else: findCfgDir(c) patchNimCfg(c, paths, cfgPath) + trace c, pkgname, "executing post install actions" afterGraphActions c, g proc updateDir(c: var AtlasContext; dir, filter: string) = @@ -341,10 +344,6 @@ proc main(c: var AtlasContext) = if c.projectDir == c.workspace or c.projectDir == c.depsDir: fatal action & " command must be executed in a project, not in the workspace" - proc findCurrentNimble(): string = - for x in walkPattern("*.nimble"): - return x - var autoinit = false var explicitProjectOverride = false var explicitDepsDirOverride = false @@ -459,23 +458,21 @@ proc main(c: var AtlasContext) = patchNimCfg c, deps, cfgPath of "use": singleArg() - #fillPackageLookupTable(c.nimbleContext, c, ) - var amb = false - var nimbleFile = findNimbleFile(c, c.workspace, amb) + var nimbleFile = findNimbleFile(c, c.workspace) var nc = createNimbleContext(c, c.depsDir) - if nimbleFile.len == 0: - nimbleFile = c.workspace / extractProjectName(c.workspace) & ".nimble" - writeFile(nimbleFile, "") - patchNimbleFile(nc, c, c.overrides, nimbleFile, args[0]) + if nimbleFile.isNone: + trace c, getCurrentDir().relativePath(c.workspace), "no nimble file found for project" + nimbleFile = some c.workspace / extractProjectName(c.workspace) & ".nimble" + writeFile(nimbleFile.get, "") + nimbleFile = findNimbleFile(c, c.workspace) + trace c, getCurrentDir().relativePath(c.workspace), "wrote new nimble file" + + patchNimbleFile(nc, c, c.overrides, nimbleFile.get(), args[0]) if c.errors > 0: discard "don't continue for 'cannot resolve'" - elif nimbleFile.len > 0 and not amb: - installDependencies(c, nc, nimbleFile) - elif amb: - error c, args[0], "ambiguous .nimble file" else: - error c, args[0], "cannot find .nimble file" + installDependencies(c, nc, nimbleFile.get()) of "pin": optSingleArg(LockFileName) @@ -500,16 +497,16 @@ proc main(c: var AtlasContext) = # projectCmd() if args.len > 1: fatal "install command takes a single argument" - var nimbleFile = "" + var nimbleFile: Option[string] if args.len == 1: - nimbleFile = args[0] + nimbleFile = some args[0] else: - nimbleFile = findCurrentNimble() - if nimbleFile.len == 0: + nimbleFile = findNimbleFile(c, getCurrentDir()) + if nimbleFile.isNone: fatal "could not find a .nimble file" else: var nc = createNimbleContext(c, c.depsDir) - installDependencies(c, nc, nimbleFile) + installDependencies(c, nc, nimbleFile.get()) of "refresh": noArgs() updatePackages(c, c.depsDir) diff --git a/src/cloner.nim b/src/cloner.nim index cf882c5..21f23da 100644 --- a/src/cloner.nim +++ b/src/cloner.nim @@ -53,9 +53,11 @@ proc cloneUrl*(c: var AtlasContext, infoNow c, url.projectName, "Cloning url: " & modurl # Checking repo with git + trace c, "atlas cloner", "checking repo " & $url let gitCmdStr = "git ls-remote --quiet --tags " & modurl var success = execCmdEx(gitCmdStr)[1] == QuitSuccess if not success and isGitHub: + trace c, "atlas cloner", "failed check ls-remote..." # retry multiple times to avoid annoying GitHub timeouts: success = retryUrl(gitCmdStr, modurl, c, url.projectName, false) @@ -74,215 +76,8 @@ proc cloneUrl*(c: var AtlasContext, (OtherError, "exernal program failed: " & hgCmdStr) else: if gitops.clone(c, url.url, dest, fullClones=true): # gitops.clone has buit-in retrying + infoNow c, url.projectName, "Cloning success" (Ok, "") else: + infoNow c, url.projectName, "Cloning failed: " & modurl (OtherError, "exernal program failed: " & $GitClone) - -when false: - proc updatePackages*(c: var AtlasContext) = - if dirExists(c.depsDir / DefaultPackagesSubDir): - withDir(c, c.depsDir / DefaultPackagesSubDir): - gitPull(c, DefaultPackagesSubDir) - else: - withDir c, c.depsDir: - let (status, err) = cloneUrl(c, PkgUrl"https://github.com/nim-lang/packages", DefaultPackagesSubDir, false) - if status != Ok: - error c, DefaultPackagesSubDir, err - - proc fillPackageLookupTable(c: var AtlasContext) = - if not c.hasPackageList: - c.hasPackageList = true - if not fileExists(c.depsDir / DefaultPackagesSubDir / "packages.json"): - updatePackages(c) - let plist = getPackageInfos(c.depsDir) - debug c, "fillPackageLookupTable", "initializing..." - for entry in plist: - let url = getUrl(entry.url) - let pkg = Package(name: PackageName unicode.toLower entry.name, - repo: PackageRepo lastPathComponent($url), - url: url) - c.urlMapping["name:" & pkg.name.string] = pkg - -when false: - proc dependencyDir*(c: var AtlasContext; pkg: Package): PackageDir = - template checkDir(dir: string) = - if dir.len > 0 and dirExists(dir): - debug c, pkg, "dependencyDir: found: " & dir - return PackageDir dir - else: - debug c, pkg, "dependencyDir: not found: " & dir - - debug c, pkg, "dependencyDir: check: pth: " & pkg.path.string & " cd: " & getCurrentDir() & " ws: " & c.workspace - if pkg.exists: - debug c, pkg, "dependencyDir: exists: " & pkg.path.string - return PackageDir pkg.path.string.absolutePath - if c.workspace.lastPathComponent == pkg.repo.string: - debug c, pkg, "dependencyDir: workspace: " & c.workspace - return PackageDir c.workspace - - if pkg.path.string.len > 0: - checkDir pkg.path.string - checkDir c.workspace / pkg.path.string - checkDir c.depsDir / pkg.path.string - - checkDir c.workspace / pkg.repo.string - checkDir c.depsDir / pkg.repo.string - checkDir c.workspace / pkg.name.string - checkDir c.depsDir / pkg.name.string - result = PackageDir c.depsDir / pkg.repo.string - trace c, pkg, "dependency not found using default" - - proc findNimbleFile*(c: var AtlasContext; pkg: Package; depDir = PackageDir""): string = - let dir = if depDir.string.len == 0: dependencyDir(c, pkg).string - else: depDir.string - result = dir / (pkg.name.string & ".nimble") - debug c, pkg, "findNimbleFile: searching: " & pkg.repo.string & " path: " & pkg.path.string & " dir: " & dir & " curr: " & result - if not fileExists(result): - debug c, pkg, "findNimbleFile: not found: " & result - result = "" - for file in walkFiles(dir / "*.nimble"): - if result.len == 0: - result = file - trace c, pkg, "nimble file found " & result - else: - error c, pkg, "ambiguous .nimble file " & result - return "" - else: - trace c, pkg, "nimble file found " & result - -when false: - proc resolvePackageUrl(c: var AtlasContext; url: string, checkOverrides = true): Package = - result = Package(url: getUrl(url), - name: url.toRepo().PackageName, - repo: url.toRepo()) - - debug c, result, "resolvePackageUrl: search: " & url - - let isFile = result.url.scheme == "file" - var isUrlOverriden = false - if not isFile and checkOverrides and UsesOverrides in c.flags: - let url = c.overrides.substitute($result.url) - if url.len > 0: - warn c, result, "resolvePackageUrl: url override found: " & $url - result.url = url.getUrl() - isUrlOverriden = true - - let namePkg = c.urlMapping.getOrDefault("name:" & result.name.string, nil) - let repoPkg = c.urlMapping.getOrDefault("repo:" & result.repo.string, nil) - - if not namePkg.isNil: - debug c, result, "resolvePackageUrl: found by name: " & $result.name.string - if namePkg.url != result.url and isUrlOverriden: - namePkg.url = result.url # update package url to match - result = namePkg - elif namePkg.url != result.url: - # package conflicts - # change package repo to `repo.user.host` - let purl = result.url - let host = purl.hostname - let org = purl.path.parentDir.lastPathPart - let rname = purl.path.lastPathPart - let pname = [rname, org, host].join(".") - warn c, result, - "conflicting url's for package; renaming package: " & - result.name.string & " to " & pname - result.repo = PackageRepo pname - c.urlMapping["name:" & result.name.string] = result - else: - result = namePkg - elif not repoPkg.isNil: - debug c, result, "resolvePackageUrl: found by repo: " & $result.repo.string - result = repoPkg - else: - # package doesn't exit and doesn't conflict - # set the url with package name as url name - c.urlMapping["repo:" & result.name.string] = result - trace c, result, "resolvePackageUrl: not found; set pkg: " & $result.repo.string - - #if result.url.scheme == "file": - # result.path = PackageDir result.url.hostname & result.url.path - # trace c, result, "resolvePackageUrl: setting manual path: " & $result.path.string - - proc resolvePackageName(c: var AtlasContext; name: string): Package = - result = Package(name: PackageName name, - repo: PackageRepo name) - - # the project name can be overwritten too! - if UsesOverrides in c.flags: - let name = c.overrides.substitute(name) - if name.len > 0: - if name.isUrl(): - return c.resolvePackageUrl(name, checkOverrides=false) - - # echo "URL MAP: ", repr c.urlMapping.keys().toSeq() - let namePkg = c.urlMapping.getOrDefault("name:" & result.name.string, nil) - let repoPkg = c.urlMapping.getOrDefault("repo:" & result.name.string, nil) - - debug c, result, "resolvePackageName: searching for package name: " & result.name.string - if not namePkg.isNil: - # great, found package! - debug c, result, "resolvePackageName: found!" - result = namePkg - result.inPackages = true - elif not repoPkg.isNil: - # check if rawHandle is a package repo name - debug c, result, "resolvePackageName: found by repo!" - result = repoPkg - result.inPackages = true - - if UsesOverrides in c.flags: - let newUrl = c.overrides.substitute($result.url) - if newUrl.len > 0: - trace c, result, "resolvePackageName: not url: UsesOverrides: " & $newUrl - result.url = getUrl newUrl - - proc resolvePackage*(c: var AtlasContext; rawHandle: string): Package = - ## Takes a raw handle which can be a name, a repo name, or a url - ## and resolves it into a package. If not found it will create - ## a new one. - ## - ## Note that Package should be unique globally. This happens - ## by updating the packages list when new packages are added or - ## loaded from a packages.json. - ## - result = Package() - - fillPackageLookupTable(c) - - trace c, rawHandle, "resolving package" - - if rawHandle.isUrl: - result = c.resolvePackageUrl(rawHandle) - else: - result = c.resolvePackageName(unicode.toLower(rawHandle)) - - result.path = dependencyDir(c, result) - let res = c.findNimbleFile(result, result.path) - if res.len > 0: - let nimble = PackageNimble res - result.exists = true - result.nimble = nimble - # the nimble package name is .nimble - result.name = PackageName nimble.string.splitFile().name - debug c, result, "resolvePackageName: nimble: found: " & $result - else: - debug c, result, "resolvePackageName: nimble: not found: " & $result - - - proc resolveNimble*(c: var AtlasContext; pkg: Package) = - ## Try to resolve the nimble file for the given package. - ## - ## This should be done after cloning a new repo. - ## - if pkg.exists: return - - pkg.path = dependencyDir(c, pkg) - let res = c.findNimbleFile(pkg) - if res.len > 0: - let nimble = PackageNimble res - # let path = PackageDir res.parentDir() - pkg.exists = true - pkg.nimble = nimble - info c, pkg, "resolvePackageName: nimble: found: " & $pkg - else: - info c, pkg, "resolvePackageName: nimble: not found: " & $pkg diff --git a/src/confighandler.nim b/src/confighandler.nim index ae06b7f..3eedfd0 100644 --- a/src/confighandler.nim +++ b/src/confighandler.nim @@ -52,7 +52,11 @@ type graph: JsonNode proc writeDefaultConfigFile*(c: var AtlasContext) = - let config = JsonConfig(resolver: $SemVer, graph: newJNull()) + let config = JsonConfig( + resolver: $SemVer, + graph: newJNull(), + deps: c.depsDir + ) let configFile = c.workspace / AtlasWorkspace writeFile(configFile, pretty %*config) diff --git a/src/context.nim b/src/context.nim index e54ab6b..ca8ceaa 100644 --- a/src/context.nim +++ b/src/context.nim @@ -67,7 +67,7 @@ template projectFromCurrentDir*(): untyped = c.currentDir.absolutePath template withDir*(c: var AtlasContext; dir: string; body: untyped) = let oldDir = getCurrentDir() - debug c, dir, "Current directory is now: " & dir + debug c, dir.relativePath(c.workspace), "Current directory is now: " & dir try: setCurrentDir(dir) body diff --git a/src/depgraphs.nim b/src/depgraphs.nim index 95757bf..128e4f2 100644 --- a/src/depgraphs.nim +++ b/src/depgraphs.nim @@ -6,7 +6,7 @@ # distribution, for details about the copyright. # -import std / [sets, tables, os, strutils, streams, json, jsonutils, algorithm] +import std / [os, sets, sequtils, tables, strutils, streams, json, jsonutils, algorithm, options] import context, gitops, runners, reporters, nimbleparser, pkgurls, cloner, versions @@ -32,6 +32,7 @@ type status: CloneStatus activeVersion*: int ondisk*: string + nimbleFile*: Option[string] DepGraph* = object nodes: seq[Dependency] @@ -64,21 +65,29 @@ proc readOnDisk(c: var AtlasContext; result: var DepGraph) = if n.isRoot: if not result.packageToDependency.hasKey(n.pkg): result.packageToDependency[n.pkg] = result.nodes.len - result.nodes.add Dependency(pkg: n.pkg, versions: @[], isRoot: true, isTopLevel: n.isTopLevel, activeVersion: -1) + result.nodes.add Dependency( + pkg: n.pkg, + versions: @[], + isRoot: true, + isTopLevel: n.isTopLevel, + activeVersion: -1, + nimbleFile: c.findNimbleFile(n.pkg, n.ondisk) + ) except: error c, configFile, "cannot read: " & configFile -proc createGraph*(c: var AtlasContext; s: PkgUrl): DepGraph = - result = DepGraph(nodes: @[], - reqs: defaultReqs()) +proc createGraph*(c: var AtlasContext; s: PkgUrl, readConfig = true): DepGraph = + result = DepGraph(nodes: @[], reqs: defaultReqs()) result.packageToDependency[s] = result.nodes.len result.nodes.add Dependency(pkg: s, versions: @[], isRoot: true, isTopLevel: true, activeVersion: -1) - readOnDisk(c, result) + if readConfig: + readOnDisk(c, result) proc toJson*(d: DepGraph): JsonNode = result = newJObject() - result["nodes"] = toJson(d.nodes) - result["reqs"] = toJson(d.reqs) + let jopts = ToJsonOptions(enumMode: joptEnumSymbol) + result["nodes"] = toJson(d.nodes, jopts) + result["reqs"] = toJson(d.reqs, jopts) proc createGraphFromWorkspace*(c: var AtlasContext): DepGraph = result = DepGraph(nodes: @[], reqs: defaultReqs()) @@ -109,10 +118,14 @@ type CommitOrigin = enum FromHead, FromGitTag, FromDep, FromNimbleFile -iterator releases(c: var AtlasContext; m: TraversalMode; versions: seq[DependencyVersion]; +iterator releases(c: var AtlasContext; + m: TraversalMode; + pkg: PkgUrl; + versions: seq[DependencyVersion]; nimbleCommits: seq[string]): (CommitOrigin, Commit) = - let (cc, status) = exec(c, GitCurrentCommit, []) - if status == 0: + let cc = c.getCurrentCommit() + trace c, pkg.projectName, "iterating commit " & $cc + if cc.isSome: case m of AllReleases: try: @@ -141,23 +154,25 @@ iterator releases(c: var AtlasContext; m: TraversalMode; versions: seq[Dependenc if produced == 0: yield (FromHead, Commit(h: "", v: Version"#head")) + except Exception as err: + echo "Atlas traverseDependency error: " + error c, pkg.projectName, "error: " & $err.msg + except Defect as err: + echo "Atlas traverseDependency Defect: " + error c, pkg.projectName, "error: " & $err.msg + except CatchableError as err: + echo "Atlas traverseDependency catchable error: " + error c, pkg.projectName, "error: " & $err.msg + finally: - discard exec(c, GitCheckout, [cc]) + # discard exec(c, GitCheckout, [cc.get()]) + trace c, pkg.projectName, "attempt to checkout out " & cc.get() + c.checkoutGitCommit(pkg.projectName, cc.get()) of CurrentCommit: yield (FromHead, Commit(h: "", v: Version"#head")) else: yield (FromHead, Commit(h: "", v: Version"#head")) -proc findNimbleFile(g: DepGraph; idx: int): (string, int) = - var nimbleFile = g.nodes[idx].pkg.projectName & ".nimble" - var found = 0 - if fileExists(nimbleFile): - inc found - else: - for file in walkFiles("*.nimble"): - nimbleFile = file - inc found - result = (ensureMove nimbleFile, found) proc enrichVersionsViaExplicitHash(versions: var seq[DependencyVersion]; x: VersionInterval) = let commit = extractSpecificCommit(x) @@ -167,28 +182,31 @@ proc enrichVersionsViaExplicitHash(versions: var seq[DependencyVersion]; x: Vers versions.add DependencyVersion(version: Version"", commit: commit, req: EmptyReqs, v: NoVar) -proc collectNimbleVersions*(c: var AtlasContext; nc: NimbleContext; g: var DepGraph; idx: int): seq[string] = - let (outerNimbleFile, found) = findNimbleFile(g, idx) +proc collectNimbleVersions*(c: var AtlasContext; dep: var Dependency): seq[string] = result = @[] - if found == 1: - let (outp, status) = exec(c, GitLog, [outerNimbleFile]) - if status == 0: - for line in splitLines(outp): - if line.len > 0 and not line.endsWith("^{}"): - result.add line - result.reverse() + if dep.nimbleFile.isSome: + result = gitops.lookupFileHashes(c, dep.nimbleFile.get()) + debug c, dep.pkg.projectName, "nimble commit versions: " & $result proc traverseRelease(c: var AtlasContext; nc: NimbleContext; g: var DepGraph; idx: int; origin: CommitOrigin; r: Commit; lastNimbleContents: var string) = - let (nimbleFile, found) = findNimbleFile(g, idx) + let pkg = g.nodes[idx].pkg + let nimbleFile = g.nodes[idx].nimbleFile + debug c, pkg.projectName, "traverseRelease: origin: " & $origin & " commit: " & $r var pv = DependencyVersion( version: r.v, commit: r.h, - req: EmptyReqs, v: NoVar) + req: EmptyReqs, + v: NoVar + ) var badNimbleFile = false - if found != 1: + if nimbleFile.isNone: + pv.req = UnknownReqs + elif not nimbleFile.get().fileExists(): + badNimbleFile = false pv.req = UnknownReqs else: + let nimbleFile = nimbleFile.get() when (NimMajor, NimMinor, NimPatch) == (2, 0, 0): # bug #110; make it compatible with Nim 2.0.0 # ensureMove requires mutable places when version < 2.0.2 @@ -234,10 +252,12 @@ proc traverseDependency(c: var AtlasContext; nc: NimbleContext; g: var DepGraph; var lastNimbleContents = "" let versions = move g.nodes[idx].versions - let nimbleVersions = collectNimbleVersions(c, nc, g, idx) + let nimbleVersions = collectNimbleVersions(c, g.nodes[idx]) - for (origin, r) in releases(c, m, versions, nimbleVersions): + for (origin, r) in releases(c, m, g.nodes[idx].pkg, versions, nimbleVersions): + debug c, g.nodes[idx].pkg.projectName, "traverseDependency: " & $c.getCurrentCommit() traverseRelease c, nc, g, idx, origin, r, lastNimbleContents + debug c, g.nodes[idx].pkg.projectName, "traverseDependencies: " & $g.nodes[idx].versions.mapIt($it.version & " <- " & ($(it.commit & "000000")[0..5])) const FileWorkspace = "file://./" @@ -264,9 +284,10 @@ type PackageAction = enum DoNothing, DoClone -proc pkgUrlToDirname(c: var AtlasContext; g: var DepGraph; d: Dependency): (string, PackageAction) = +proc pkgUrlToDirname(c: var AtlasContext; g: var DepGraph; d: var Dependency): (string, PackageAction) = # XXX implement namespace support here var dest = g.ondisk.getOrDefault(d.pkg.url) + trace c, d.pkg.projectName, "using dirname: " & $dest & " for url: " & $d.pkg.url if dest.len == 0: if d.isTopLevel: dest = c.workspace @@ -285,6 +306,7 @@ proc expand*(c: var AtlasContext; g: var DepGraph; nc: NimbleContext; m: Travers while i < g.nodes.len: if not processed.containsOrIncl(g.nodes[i].pkg): let (dest, todo) = pkgUrlToDirname(c, g, g.nodes[i]) + trace c, $g.nodes[i].pkg.projectName, "expanded destination dir: " & $dest g.nodes[i].ondisk = dest if todo == DoClone: let (status, _) = @@ -295,6 +317,7 @@ proc expand*(c: var AtlasContext; g: var DepGraph; nc: NimbleContext; m: Travers g.nodes[i].status = status if g.nodes[i].status == Ok: + g.nodes[i].nimbleFile = c.findNimbleFile(g.nodes[i].pkg, dest) withDir c, dest: traverseDependency(c, nc, g, i, m) inc i @@ -455,13 +478,15 @@ proc runBuildSteps(c: var AtlasContext; g: var DepGraph) = let activeVersion = g.nodes[i].activeVersion let r = if g.nodes[i].versions.len == 0: -1 else: g.nodes[i].versions[activeVersion].req if r >= 0 and r < g.reqs.len and g.reqs[r].hasInstallHooks: - let (nf, found) = findNimbleFile(g, i) - if found == 1: - runNimScriptInstallHook c, nf, pkg.projectName + let nf = c.findNimbleFile(g.nodes[i].pkg, getCurrentDir()) + if nf.isSome: + trace c, pkg.projectName, "running Nimble install hook" + runNimScriptInstallHook c, nf.get, pkg.projectName # check for nim script builders for p in mitems c.plugins.builderPatterns: let f = p[0] % pkg.projectName if fileExists(f): + trace c, pkg.projectName, "running NimScript builder" runNimScriptBuilder c, p, pkg.projectName proc debugFormular(c: var AtlasContext; g: var DepGraph; f: Form; s: Solution) = @@ -478,9 +503,17 @@ proc debugFormular(c: var AtlasContext; g: var DepGraph; f: Form; s: Solution) = proc solve*(c: var AtlasContext; g: var DepGraph; f: Form) = let m = f.idgen var s = createSolution(m) - #debugFormular c, g, f, s - if satisfiable(f.f, s): + var status = + try: satisfiable(f.f, s) + except SatOverflowError as err: + echo "\n" + echo "SAT MaxIterationLimitError: " + debugFormular c, g, f, s + echo "\n" + raise err + + if status: for n in mitems g.nodes: if n.isRoot: n.active = true for i in 0 ..< m: diff --git a/src/gitops.nim b/src/gitops.nim index c9aeadc..be9f939 100644 --- a/src/gitops.nim +++ b/src/gitops.nim @@ -6,7 +6,7 @@ # distribution, for details about the copyright. # -import std/[os, osproc, sequtils, strutils] +import std/[os, osproc, sequtils, strutils, options, algorithm] import reporters, osutils, versions type @@ -83,15 +83,30 @@ proc clone*(c: var Reporter; url, dest: string; retries = 5; fullClones=false): else: "" let cmd = $GitClone & " " & extraArgs & " " & quoteShell(url) & " " & dest + debug c, url, "cloning using command: " & cmd for i in 1..retries: - if execShellCmd(cmd) == 0: + let res = execShellCmd(cmd) + debug c, url, "cloning status: " & $res + if res == 0 and dirExists(dest): return true - os.sleep(i*2_000) + trace c, url, "trying to clone again" + sleep(i*2_000) proc gitDescribeRefTag*(c: var Reporter; commit: string): string = let (lt, status) = exec(c, GitDescribe, ["--tags", commit]) result = if status == 0: strutils.strip(lt) else: "" +proc lookupFileHashes*(c: var Reporter; nimbleFile: string): seq[string] = + result = @[] + let (outp, status) = exec(c, GitLog, [nimbleFile]) + if status == 0: + for line in splitLines(outp): + if line.len > 0 and not line.endsWith("^{}"): + result.add line + else: + warn c, nimbleFile, "error looking up git hash history: " & outp + result.reverse() + proc getLastTaggedCommit*(c: var Reporter): string = let (ltr, status) = exec(c, GitLastTaggedRef, []) if status == 0: @@ -129,18 +144,23 @@ proc listFiles*(c: var Reporter): seq[string] = result = @[] proc checkoutGitCommit*(c: var Reporter; p, commit: string) = + debug(c, p, "checking out commit " & commit) let (currentCommit, statusA) = exec(c, GitCurrentCommit, []) - if statusA == 0 and currentCommit.strip() == commit: return + if statusA == 0 and currentCommit.strip() == commit: + info(c, p, "updated package to " & commit) + return - let (_, statusB) = exec(c, GitCheckout, [commit]) + let (outp, statusB) = exec(c, GitCheckout, [commit]) if statusB != 0: - error(c, p, "could not checkout commit " & commit) + error(c, p, "could not checkout commit " & commit & " error: " & $outp) else: info(c, p, "updated package to " & commit) proc checkoutGitCommitFull*(c: var Reporter; p, commit: string; fullClones: bool) = var smExtraArgs: seq[string] = @[] + trace(c, p, "attempt to checkout out package commit " & commit) + if not fullClones and commit.len == 40: smExtraArgs.add "--depth=1" @@ -207,6 +227,7 @@ proc incrementTag*(c: var Reporter; displayName, lastTag: string; field: Natural lastTag[0.. 0 and nimblePath.fileExists(): + let nimblePath = findNimbleFile(c, startPkg) + if nimblePath.isSome: lf.nimbleFile = LockedNimbleFile( - filename: nimblePath.relativePath(c.currentDir), - content: readFile(nimblePath).splitLines()) + filename: nimblePath.get().relativePath(c.currentDir), + content: readFile(nimblePath.get()).splitLines()) if not exportNimble: write lf, lockFilePath @@ -278,7 +279,7 @@ proc listChanged*(c: var AtlasContext; lockFilePath: string) = " found: " & url & " lockfile has: " & v.url - let commit = gitops.getCurrentCommit() + let commit = c.getCurrentCommit().get() if commit != v.commit: #let info = parseNimble(c, pkg.nimble) warn c, dir, "commit differs;" & diff --git a/src/nimbleparser.nim b/src/nimbleparser.nim index 53aaec2..ebbe5dd 100644 --- a/src/nimbleparser.nim +++ b/src/nimbleparser.nim @@ -6,9 +6,11 @@ # distribution, for details about the copyright. # -import std / [os, strutils, tables, unicode, hashes] +import std / [os, strutils, tables, unicode, hashes, options] import versions, packagesjson, reporters, gitops, parse_requires, pkgurls, compiledpatterns +export options + when defined(nimAtlasBootstrap): import ../dist/sat/src/sat/satvars else: @@ -49,9 +51,13 @@ proc `==`*(a, b: Requirements): bool = proc updatePackages*(c: var Reporter; depsDir: string) = if dirExists(depsDir / DefaultPackagesSubDir): withDir(c, depsDir / DefaultPackagesSubDir): + trace c, getCurrentDir(), "updating packages" + echo "TRY UPDATING PACKAGES" gitPull(c, DefaultPackagesSubDir) else: withDir c, depsDir: + trace c, getCurrentDir(), "cloning packages" + echo "TRY CLONING PACKAGES" let success = clone(c, "https://github.com/nim-lang/packages", DefaultPackagesSubDir) if not success: error c, DefaultPackagesSubDir, "cannot clone packages repo" @@ -115,20 +121,43 @@ proc parseNimbleFile*(c: NimbleContext; nimbleFile: string; p: Patterns): Requir else: result.deps.add (createUrlSkipPatterns(u), query) -proc findNimbleFile*(c: var Reporter; dir: string; ambiguous: var bool): string = - result = "" - var counter = 0 - for x in walkFiles(dir / "*.nimble"): - inc counter - if result.len == 0: - result = x - if counter > 1: - ambiguous = true - result = "" - -# if counter > 1: -# warn c, dir, "cannot determine `.nimble` file; there are multiple to choose from" -# result = "" +proc findNimbleFile*(c: var Reporter, dir: string): Option[string] = + ## Search the given directory for a Nimble file. + ## + ## An error is reported if there are multiple Nimble files + ## which results in an ambiguity. + var nimbleFile = "" + var found = 0 + + for file in walkFiles(dir / "*.nimble"): + nimbleFile = file + found.inc + + if found > 1: + error c, dir, "ambiguous .nimble files; found: " & $found & " options" + result = string.none() + elif found == 0: + warn c, dir, "no nimble file found" + result = string.none() + else: + result = some(nimbleFile.absolutePath()) + debug c, dir, "findNimbleFile: found: " & nimbleFile + +proc findNimbleFile*(c: var Reporter, pkg: PkgUrl, dir: string): Option[string] = + ## Search the given directory for a Nimble file starting with the + ## name from the package's URL. + ## + ## An error is reported if there are multiple Nimble files + ## which results in an ambiguity. + var nimbleFile = pkg.projectName & ".nimble" + debug c, pkg.projectName, "findNimbleFile: searching: " & pkg.projectName & + " path: " & dir + assert dir != "" + if fileExists(dir / nimbleFile): + debug c, pkg.projectName, "findNimbleFile: found: " & nimbleFile + some((dir / nimbleFile).absolutePath()) + else: + findNimbleFile(c, dir) proc genRequiresLine(u: string): string = "requires \"$1\"\n" % u.escape("", "") diff --git a/src/parse_requires.nim b/src/parse_requires.nim index 5ede51f..5aa0b8b 100644 --- a/src/parse_requires.nim +++ b/src/parse_requires.nim @@ -32,7 +32,7 @@ proc extract(n: PNode; conf: ConfigRef; result: var NimbleFileInfo) = if ch.kind in {nkStrLit..nkTripleStrLit}: result.requires.add ch.strVal else: - localError(conf, ch.info, "'requires' takes string literals") + localError(conf, ch.info, warnUser, "'requires' takes string literals") result.hasErrors = true of "task": if n.len >= 3 and n[1].kind == nkIdent and n[2].kind in {nkStrLit..nkTripleStrLit}: @@ -54,13 +54,13 @@ proc extract(n: PNode; conf: ConfigRef; result: var NimbleFileInfo) = if n[1].kind in {nkStrLit..nkTripleStrLit}: result.srcDir = n[1].strVal else: - localError(conf, n[1].info, "assignments to 'srcDir' must be string literals") + localError(conf, n[1].info, warnUser, "assignments to 'srcDir' must be string literals") result.hasErrors = true elif n[0].kind == nkIdent and eqIdent(n[0].ident.s, "version"): if n[1].kind in {nkStrLit..nkTripleStrLit}: result.version = n[1].strVal else: - localError(conf, n[1].info, "assignments to 'version' must be string literals") + localError(conf, n[1].info, warnUser, "assignments to 'version' must be string literals") result.hasErrors = true else: discard diff --git a/src/pkgurls.nim b/src/pkgurls.nim index 9183a51..f9d88dc 100644 --- a/src/pkgurls.nim +++ b/src/pkgurls.nim @@ -56,9 +56,3 @@ proc `==`*(a, b: PkgUrl): bool {.inline.} = a.u == b.u proc hash*(a: PkgUrl): Hash {.inline.} = hash(a.u) proc isFileProtocol*(s: PkgUrl): bool = s.u.startsWith("file://") - -proc dir*(s: PkgUrl): string = - if isFileProtocol(s): - result = substr(s.u, len("file://")) - else: - result = s.projectName diff --git a/tests/nim.cfg b/tests/nim.cfg deleted file mode 100644 index 3982b12..0000000 --- a/tests/nim.cfg +++ /dev/null @@ -1,10 +0,0 @@ -############# begin Atlas config section ########## ---noNimblePath ---path:"../balls" ---path:"../grok" ---path:"../ups" ---path:"../sync" ---path:"../npeg/src" ---path:"../testes" ---path:"../nim-bytes2human/src" -############# end Atlas config section ########## diff --git a/tests/setups.nim b/tests/setups.nim index cdde3ca..668ffc6 100644 --- a/tests/setups.nim +++ b/tests/setups.nim @@ -1,5 +1,8 @@ import std / os +import std / tempfiles +export tempfiles +export os proc exec*(cmd: string) = if execShellCmd(cmd) != 0: @@ -13,6 +16,38 @@ template withDir*(dir: string; body: untyped) = finally: setCurrentDir(old) +const removeTempDirs {.booldefine.} = true + +template withTempTestDirFull*(name: string, remove: bool, blk: untyped) = + ## creates test dir and optionally remove dir afterwords + let old = getCurrentDir() + let dir {.inject.} = createTempDir("atlas_test_", "_" & name) + echo "Creating temp test directory: ", dir + try: + setCurrentDir(dir) + `blk` + finally: + setCurrentDir(old) + if remove: + removeDir(dir) + +template withTempTestDir*(name: string, blk: untyped) = + withTempTestDirFull(name, removeTempDirs, blk) + +template withTempTestDirFrom*(name: string, src: string, blk: untyped) = + ## creates test dir by copying from a template folder + let old = getCurrentDir() + let dir {.inject.} = genTempPath("atlas_test_", "_" & name) + echo "Creating temp test directory: ", dir + copyDir(src, dir) + try: + setCurrentDir(dir) + `blk` + finally: + setCurrentDir(old) + if removeTempDirs: + removeDir(dir) + proc buildGraph* = createDir "source" withDir "source": diff --git a/tests/tester.nim b/tests/tester.nim index ae7343e..efd1c7f 100644 --- a/tests/tester.nim +++ b/tests/tester.nim @@ -9,11 +9,10 @@ if execShellCmd("nim c -d:debug -r tests/unittests.nim") != 0: var failures = 0 -let atlasExe = absolutePath("bin" / "atlas".addFileExt(ExeExt)) -if execShellCmd("nim c -o:$# -d:release src/atlas.nim" % [atlasExe]) != 0: +var atlasExe = absolutePath("bin" / "atlas".addFileExt(ExeExt)) +if execShellCmd("nim c -o:$# -d:debug src/atlas.nim" % [atlasExe]) != 0: quit("FAILURE: compilation of atlas failed") - proc sameDirContents(expected, given: string): bool = result = true for _, e in walkDir(expected): @@ -70,24 +69,31 @@ const end of selection """ +template withTestDir(dir: string, desc: string, blk: untyped) = + echo "\n# Running test ", desc, " in dir: ", dir, "\n" + withDir dir: + `blk` + echo "\n## Finished running test in dir: ", dir, "\n" + proc testSemVer2(expected: string) = createDir "semproject" withDir "semproject": - let cmd = atlasExe & " --full --keepWorkspace --resolver=SemVer --colors:off --list use proj_a" + echo "## Running test in subdir `semproject`" + let cmd = atlasExe & " --verbosity:trace --full --keepWorkspace --resolver=SemVer --colors:off --list use proj_a" let (outp, status) = execCmdEx(cmd) - if status == 0: - if outp.contains expected: - discard "fine" - else: - echo "expected ", expected, " but got ", outp - raise newException(AssertionDefect, "Test failed!") + if outp.contains expected: + echo "# Test success" + echo outp else: - echo "\n\n<<<<<<<<<<<<<<<< failed " - echo "testSemVer2:command: ", cmd - echo "testSemVer2:pwd: ", getCurrentDir() - echo "testSemVer2:failed command:\n", outp - echo ">>>>>>>>>>>>>>>> failed\n" - assert false, "testSemVer2" + echo "" + echo "# Failure error code: ", status + echo "## Expected:\n", expected + echo "" + echo "## Actual:\n", outp + echo "" + echo "" + raise newException(AssertionDefect, "Test failed!") + echo "### Done Running test in subdir `semproject`" proc testMinVer() = buildGraph() @@ -103,7 +109,7 @@ proc testMinVer() = else: assert false, outp -withDir "tests/ws_semver2": +withTestDir "tests/ws_semver2", "semver2 plain": try: buildGraph() testSemVer2(SemVerExpectedResult) @@ -117,7 +123,7 @@ withDir "tests/ws_semver2": removeDir "proj_c" removeDir "proj_d" -withDir "tests/ws_semver2": +withTestDir "tests/ws_semver2", "semver2 no git tags": try: buildGraphNoGitTags() testSemVer2(SemVerExpectedResultNoGitTags) diff --git a/tests/unit_tests/config.nims b/tests/unit_tests/config.nims new file mode 100644 index 0000000..b656e8b --- /dev/null +++ b/tests/unit_tests/config.nims @@ -0,0 +1,5 @@ +--path:"../../src" +--path:"$nim" +--d:atlasStandAlone +--d:atlasUnitTests +--d:atlasNoUnitTestFiles diff --git a/tests/unit_tests/testCollectVersions.nim b/tests/unit_tests/testCollectVersions.nim new file mode 100644 index 0000000..5de33ee --- /dev/null +++ b/tests/unit_tests/testCollectVersions.nim @@ -0,0 +1,48 @@ + +import std/unittest +import std/strutils +import std/os +import std/tempfiles +import std/options + +import ../setups + +import context, reporters, nimbleparser, pkgurls +import compiledpatterns +import compiledpatterns +import pkgurls +import depgraphs + +proc toDirSep(s: string): string = + result = s.replace("/", $DirSep) + +template setupDepsAndGraph(dir: string) = + var + p {.inject.} = initPatterns() + u {.inject.} = createUrl("file://" & dir, p) + c {.inject.} = AtlasContext() + g {.inject.} = createGraph(c, u, readConfig = false) + + c.depsDir = "source" + c.workspace = dir.toDirSep + c.projectDir = dir.toDirSep + c.verbosity = 3 + +suite "test pkgurls": + + test "basic url": + withTempTestDir "basic_url": + buildGraphNoGitTags() + echo "\n" + setupDepsAndGraph(dir) + var d = Dependency() + let depDir = "source" / "proj_a/" + ## TODOX: how to handle this relative or not thing? + let nimble = "proj_a.nimble" + setCurrentDir(depDir) + d.pkg = createUrl("file://" & $depDir, p) + d.nimbleFile = some nimble + echo "D: ", d + let versions = collectNimbleVersions(c, d) + echo "VERSIONS: ", versions + diff --git a/tests/unit_tests/testCollectVersions.nims b/tests/unit_tests/testCollectVersions.nims new file mode 100644 index 0000000..cfda6f2 --- /dev/null +++ b/tests/unit_tests/testCollectVersions.nims @@ -0,0 +1,9 @@ +--noNimblePath +--path:"../../src" +--path:"$nim" +--d:atlasStandAlone +--d:atlasUnitTests +--d:atlasNoUnitTestFiles + +## todo remove? +--d:"removeTempDirs:false" \ No newline at end of file diff --git a/tests/unit_tests/testpkgurls.nim b/tests/unit_tests/testpkgurls.nim new file mode 100644 index 0000000..4d89a0f --- /dev/null +++ b/tests/unit_tests/testpkgurls.nim @@ -0,0 +1,108 @@ + +import std/unittest +import std/strutils +import std/paths +import std/options + +import ../setups + +import context, reporters, nimbleparser +import compiledpatterns +import pkgurls +import depgraphs + +proc toDirSep(s: string): string = + result = s.replace("/", $DirSep) + +template setupDepsAndGraph(url: string) = + var + p {.inject.} = initPatterns() + u {.inject.} = createUrl(url, p) + c {.inject.} = AtlasContext() + g {.inject.} = createGraph(c, u, readConfig = false) + d {.inject.} = Dependency() + + c.depsDir = "fakeDeps" + c.workspace = "/workspace/".toDirSep + c.projectDir = "/workspace".toDirSep + +template withProjectDir*(names: varargs[string], blk: untyped) = + createDir "workspace" + setCurrentDir("workspace") + c.workspace = os.getCurrentDir() + for name in names: + createDir name + setCurrentDir(name) + c.projectDir = os.getCurrentDir() + `blk` + discard + when false: + echo "\n<<<<<< CWD: ", os.getCurrentDir() + echo "workspace: ", c.workspace + echo "projectDir: ", c.projectDir, "\n" + discard execShellCmd("find " & dir) + echo ">>>>>>\n\n" + +suite "test pkgurls": + + test "basic url": + setupDepsAndGraph("https://github.com/example/proj.git") + check $u == "https://github.com/example/proj.git" + check u.projectName == "proj" + + test "basic url no git": + setupDepsAndGraph("https://github.com/example/proj") + check $u == "https://github.com/example/proj" + check u.projectName == "proj" + + test "basic url prefix": + setupDepsAndGraph("https://github.com/example/nim-proj") + check $u == "https://github.com/example/nim-proj" + check u.projectName == "nim-proj" + +# var testTemplateDir: string +# withTempTestDirFull("test_template", remove=false): +# buildGraphNoGitTags() +# testTemplateDir = dir + +suite "nimble stuff": + setup: + setupDepsAndGraph("https://github.com/example/nim-testProj1") + + test "find nimble in project dir from project dir": + withTempTestDir "test": + withProjectDir "fakeDeps", "testProj1": + writeFile("testProj1.nimble", "") + let projDir = "workspace" / "fakeDeps" / "testProj1" + let res1 = findNimbleFile(c, u, dir / projDir) + check res1.get().relativePath(dir) == projDir / "testProj1.nimble" + + setCurrentDir(dir / "workspace") + let res2 = findNimbleFile(c, u, dir / projDir) + check res2.get().relativePath(dir) == projDir / "testProj1.nimble" + + test "find nimble in project dir with other name": + withTempTestDir "test": + withProjectDir "fakeDeps", "nim-testProj1": + writeFile("testProj1.nimble", "") + let projDir = "workspace" / "fakeDeps" / "nim-testProj1" + let res = findNimbleFile(c, u, dir / projDir) + check res.get().relativePath(dir) == projDir / "testProj1.nimble" + + test "missing": + withTempTestDir "basic_url": + withProjectDir "fakeDeps", "testProj1": + let projDir = "workspace" / "fakeDeps" / "testProj1" + let res1 = findNimbleFile(c, u, dir / projDir) + check res1.isNone() + + test "ambiguous": + withTempTestDir "basic_url": + withProjectDir "fakeDeps", "testProj1": + writeFile("testProj1.nimble", "") + writeFile("testProj2.nimble", "") + let projDir = "workspace" / "fakeDeps" / "testProj1" + let res = findNimbleFile(c, u, dir / projDir) + check res == string.none + check c.errors == 1 + diff --git a/tests/unittests.nim b/tests/unittests.nim index e32a446..8c44fce 100644 --- a/tests/unittests.nim +++ b/tests/unittests.nim @@ -4,8 +4,7 @@ import std/[unittest, os, strutils] import context, osutils, versions -when false: - from nameresolver import resolvePackage +import unit_tests/testpkgurls let basicExamples = { diff --git a/tests/unittests.nims b/tests/unittests.nims index 5c99662..224d42f 100644 --- a/tests/unittests.nims +++ b/tests/unittests.nims @@ -1,4 +1,3 @@ ---noNimblePath --path:"../src" --path:"$nim" --d:atlasStandAlone From 6ccb5672760fc6b43a603f73d156e302c83a223e Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Sat, 30 Mar 2024 14:51:15 +0200 Subject: [PATCH 2/8] remove debug exception handlers --- src/depgraphs.nim | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/depgraphs.nim b/src/depgraphs.nim index 128e4f2..b2546cc 100644 --- a/src/depgraphs.nim +++ b/src/depgraphs.nim @@ -153,17 +153,6 @@ iterator releases(c: var AtlasContext; if produced == 0: yield (FromHead, Commit(h: "", v: Version"#head")) - - except Exception as err: - echo "Atlas traverseDependency error: " - error c, pkg.projectName, "error: " & $err.msg - except Defect as err: - echo "Atlas traverseDependency Defect: " - error c, pkg.projectName, "error: " & $err.msg - except CatchableError as err: - echo "Atlas traverseDependency catchable error: " - error c, pkg.projectName, "error: " & $err.msg - finally: # discard exec(c, GitCheckout, [cc.get()]) trace c, pkg.projectName, "attempt to checkout out " & cc.get() From 53e6a3dfb6a43753ef178aa9a2cc35ffa2891ae1 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 29 Jan 2024 17:35:11 +0200 Subject: [PATCH 3/8] add short options for workspaces or deps dirs --- src/atlas.nim | 96 +++++++++++++++++++++++++------------------ src/confighandler.nim | 3 +- 2 files changed, 59 insertions(+), 40 deletions(-) diff --git a/src/atlas.nim b/src/atlas.nim index b39f85a..09bf482 100644 --- a/src/atlas.nim +++ b/src/atlas.nim @@ -74,12 +74,17 @@ Command: env setup a Nim virtual environment --keep keep the c_code subdirectory +Command type options: + -p interperate command as a project command + -w interperate command as a workspace command + Options: --keepCommits do not perform any `git checkouts` --full perform full checkouts rather than the default shallow --cfgHere also create/maintain a nim.cfg in the current working directory - --workspace=DIR use DIR as workspace + --workspace=DIR, -d=DIR + use DIR as workspace --project=DIR use DIR as the current project --noexec do not perform any action that may run arbitrary code --autoenv detect the minimal Nim $version and setup a @@ -91,12 +96,12 @@ Options: --showGraph show the dependency graph --keepWorkspace do not update/overwrite `atlas.workspace` --list list all available and installed versions - --version show the version + --version, -v show the version --ignoreUrls don't error on mismatching urls --verbosity=normal|trace|debug set verbosity level to normal, trace, debug - --global use global workspace in ~/.atlas - --help show this help + --global, -g use global workspace in ~/.atlas + --help, -h show this help """ proc writeHelp() = @@ -232,7 +237,6 @@ proc updateDir(c: var AtlasContext; dir, filter: string) = trace c, file, "updating directory" gitops.updateDir(c, file, filter) - proc detectWorkspace(currentDir: string): string = ## find workspace by checking `currentDir` and its parents. result = currentDir @@ -240,13 +244,7 @@ proc detectWorkspace(currentDir: string): string = if fileExists(result / AtlasWorkspace): return result result = result.parentDir() - when false: - # That is a bad idea and I know no other tool (git etc.) that - # does such shenanigans. - # alternatively check for "sub-directory" workspace - for kind, file in walkDir(currentDir): - if kind == pcDir and fileExists(file / AtlasWorkspace): - return file + ## TODO: implement possible better default workspace option proc autoWorkspace(currentDir: string): string = result = currentDir @@ -323,6 +321,7 @@ proc newProject(c: var AtlasContext; projectName: string) = proc main(c: var AtlasContext) = var action = "" var args: seq[string] = @[] + template singleArg() = if args.len != 1: fatal action & " command takes a single package name" @@ -338,18 +337,51 @@ proc main(c: var AtlasContext) = fatal action & " command takes no arguments" template projectCmd() = + if explicitWorkspaceCmd: + fatal action & " command cannot be executed as a workspace command" if c.projectDir == c.workspace or c.projectDir == c.depsDir: fatal action & " command must be executed in a project, not in the workspace" + template setWorkspaceDir(val: string) = + if val == ".": + c.workspace = getCurrentDir() + createWorkspaceIn c + elif val.len > 0: + c.workspace = val + if not explicitProjectOverride: + c.currentDir = val + createDir(val) + createWorkspaceIn c + else: + writeHelp() + + template setProjectDir(val: string) = + explicitProjectOverride = true + if isAbsolute(val): + c.currentDir = val + else: + c.currentDir = getCurrentDir() / val + proc findCurrentNimble(): string = for x in walkPattern("*.nimble"): return x - var autoinit = false - var explicitProjectOverride = false - var explicitDepsDirOverride = false + var + autoinit = false + explicitProjectOverride = false + explicitDepsDirOverride = false + explicitProjCmd = false + explicitWorkspaceCmd = false + + # process cli environment variables if existsEnv("NO_COLOR") or not isatty(stdout) or (getEnv("TERM") == "dumb"): c.noColors = true + if existsEnv("ATLAS_WORKSPACE"): + setWorkspaceDir(getEnv("ATLAS_WORKSPACE")) + if existsEnv("ATLAS_PROJECT"): + setWorkspaceDir(getEnv("ATLAS_PROJECT")) + + # process cli option flags for kind, key, val in getopt(): case kind of cmdArgument: @@ -361,25 +393,13 @@ proc main(c: var AtlasContext) = case normalize(key) of "help", "h": writeHelp() of "version", "v": writeVersion() + of "p": explicitProjCmd = true + of "w": explicitWorkspaceCmd = true of "keepcommits": c.flags.incl KeepCommits - of "workspace": - if val == ".": - c.workspace = getCurrentDir() - createWorkspaceIn c - elif val.len > 0: - c.workspace = val - if not explicitProjectOverride: - c.currentDir = val - createDir(val) - createWorkspaceIn c - else: - writeHelp() + of "workspace", "d": + setWorkspaceDir(val) of "project": - explicitProjectOverride = true - if isAbsolute(val): - c.currentDir = val - else: - c.currentDir = getCurrentDir() / val + setProjectDir(val) of "deps": if val.len > 0: c.depsDir = val @@ -440,6 +460,10 @@ proc main(c: var AtlasContext) = if action != "tag": createDir(c.depsDir) + if explicitProjCmd and explicitWorkspaceCmd: + fatal "Cannot specify both -w and -p flags together since they conflict with each other." + + # process cli command case action of "": fatal "No action." @@ -479,7 +503,7 @@ proc main(c: var AtlasContext) = of "pin": optSingleArg(LockFileName) - if c.projectDir == c.workspace or c.projectDir == c.depsDir: + if explicitWorkspaceCmd or c.projectDir == c.workspace or c.projectDir == c.depsDir: pinWorkspace c, args[0] else: let exportNimble = args[0] == NimbleLockFileName @@ -558,12 +582,6 @@ proc main(c: var AtlasContext) = setupNimEnv c, c.workspace, args[0], Keep in c.flags of "outdated": listOutdated(c) - #of "checksum": - # singleArg() - # let pkg = resolvePackage(c, args[0]) - # let cfg = findCfgDir(c, pkg) - # let sha = nimbleChecksum(c, pkg, cfg) - # info c, pkg, "SHA1Digest: " & sha of "new": singleArg() newProject(c, args[0]) diff --git a/src/confighandler.nim b/src/confighandler.nim index ae06b7f..a35d75e 100644 --- a/src/confighandler.nim +++ b/src/confighandler.nim @@ -8,7 +8,7 @@ ## Configuration handling. -import std / [strutils, os, streams, json] +import std / [strutils, os, streams, json, options] import versions, context, reporters, compiledpatterns, parse_requires proc parseOverridesFile(c: var AtlasContext; filename: string) = @@ -49,6 +49,7 @@ type overrides: string plugins: string resolver: string + requires: Option[seq[string]] graph: JsonNode proc writeDefaultConfigFile*(c: var AtlasContext) = From a8edb213cb9c92ca531a686f60e237ab836c3d0d Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 13 May 2024 22:09:48 +0300 Subject: [PATCH 4/8] Apply suggestions from code review Don't error on ambiguous nimble files --- src/nimbleparser.nim | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/nimbleparser.nim b/src/nimbleparser.nim index ebbe5dd..0bb0672 100644 --- a/src/nimbleparser.nim +++ b/src/nimbleparser.nim @@ -124,8 +124,6 @@ proc parseNimbleFile*(c: NimbleContext; nimbleFile: string; p: Patterns): Requir proc findNimbleFile*(c: var Reporter, dir: string): Option[string] = ## Search the given directory for a Nimble file. ## - ## An error is reported if there are multiple Nimble files - ## which results in an ambiguity. var nimbleFile = "" var found = 0 @@ -134,7 +132,7 @@ proc findNimbleFile*(c: var Reporter, dir: string): Option[string] = found.inc if found > 1: - error c, dir, "ambiguous .nimble files; found: " & $found & " options" + warn c, dir, "ambiguous .nimble files; found: " & $found & " options" result = string.none() elif found == 0: warn c, dir, "no nimble file found" From 2b7fa08fa366be0bba494757c9dd196a4412a671 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 13 May 2024 22:10:41 +0300 Subject: [PATCH 5/8] Apply suggestions from code review spelling --- src/atlas.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/atlas.nim b/src/atlas.nim index 7210ef5..55ab9f2 100644 --- a/src/atlas.nim +++ b/src/atlas.nim @@ -75,7 +75,7 @@ Command: --keep keep the c_code subdirectory Command type options: - -p interperate command as a project command + -p interpret command as a project command -w interperate command as a workspace command Options: From 71d6db9064bdbc248f4290ccd49b61ad1d613807 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 13 May 2024 22:14:27 +0300 Subject: [PATCH 6/8] Update src/atlas.nim --- src/atlas.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/atlas.nim b/src/atlas.nim index 55ab9f2..223da0b 100644 --- a/src/atlas.nim +++ b/src/atlas.nim @@ -76,7 +76,7 @@ Command: Command type options: -p interpret command as a project command - -w interperate command as a workspace command + -w interpret command as a workspace command Options: --keepCommits do not perform any `git checkouts` From fc5c5a642f76763bbf5b5091f29ba9306a9b49d4 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 14 May 2024 01:15:31 +0300 Subject: [PATCH 7/8] Update src/atlas.nim Co-authored-by: Andreas Rumpf --- src/atlas.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/atlas.nim b/src/atlas.nim index 223da0b..e2aedb8 100644 --- a/src/atlas.nim +++ b/src/atlas.nim @@ -464,7 +464,7 @@ proc main(c: var AtlasContext) = createDir(c.depsDir) if explicitProjCmd and explicitWorkspaceCmd: - fatal "Cannot specify both -w and -p flags together since they conflict with each other." + fatal "Cannot specify both -w and -p flags together since they conflict with each other." # process cli command case action From 8f1b51723b58c7030a879364934ccb1d9109dba3 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 14 May 2024 01:27:36 +0300 Subject: [PATCH 8/8] Update tests/unit_tests/testpkgurls.nim --- tests/unit_tests/testpkgurls.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit_tests/testpkgurls.nim b/tests/unit_tests/testpkgurls.nim index 4d89a0f..1e1a575 100644 --- a/tests/unit_tests/testpkgurls.nim +++ b/tests/unit_tests/testpkgurls.nim @@ -104,5 +104,5 @@ suite "nimble stuff": let projDir = "workspace" / "fakeDeps" / "testProj1" let res = findNimbleFile(c, u, dir / projDir) check res == string.none - check c.errors == 1 + # check c.errors == 1