Fixes various Nimble and Dependency handling issues

This PR started to work on #117 but uncovered a few other items which needed cleaned up or fixed.

Changes:

Combine multiple ways to search for Nimble files into one place
fixes some edge cases in handling finding Nimble files
fixes issue with writing default config file and handling the deps cli option
osutils.nim provides shims for unit testing proc's using a subset of file operations
unit tests for various nimble file states
unit tests for parsing PkgUrl's
fixes some adhoc git execs to use the gitops module, restoring logging etc

.github/workflows/ci.yml

@@ -15,19 +15,19 @@ jobs:
         target:
           - os: linux
             cpu: amd64
-            nim_branch: devel
+            nim_branch: version-2-0
           # - os: linux
           #   cpu: i386
           #   nim_branch: devel
           - os: macos
             cpu: amd64
-            nim_branch: devel
+            nim_branch: version-2-0
           - os: windows
             cpu: amd64
-            nim_branch: devel
+            nim_branch: version-2-0
           - os: windows
             cpu: i386
-            nim_branch: devel
+            nim_branch: version-2-0
         include:
           - target:
               os: linux
@@ -52,6 +52,7 @@ jobs:
           git config --global init.defaultBranch master
           git config --global user.email "[email protected]"
           git config --global user.name "atlasbot"
+          git config --global gc.auto 0
 
       - name: Checkout atlas
         uses: actions/checkout@v4
@@ -128,6 +129,10 @@ jobs:
           version: ${{ matrix.target.nim_branch }}
           architecture: ${{ matrix.target.cpu }}
 
+      - name: Nim Version
+        run: |
+          nim -v
+
       - name: Run tests
         run: |
           cd atlas

.gitignore

@@ -10,3 +10,4 @@ tests/*
 !tests/*.*
 atlas
 deps.png
+nim.cfg

config.nims

@@ -1,6 +1,6 @@
 
 task build, "Build local atlas":
-  exec "nim c -d:debug -o:./atlas src/atlas.nim"
+  exec "nim c -d:debug -o:./bin/atlas src/atlas.nim"
 
 task unitTests, "Runs unit tests":
   exec "nim c -d:debug -r tests/unittests.nim"

src/atlas.nim

@@ -219,9 +219,12 @@ proc installDependencies(c: var AtlasContext; nc: var NimbleContext; nimbleFile:
   let (dir, pkgname, _) = splitFile(nimbleFile)
   info c, pkgname, "installing dependencies for " & pkgname & ".nimble"
   var g = createGraph(c, createUrlSkipPatterns(dir))
+  trace c, pkgname, "traversing depency loop"
   let paths = traverseLoop(c, nc, g)
+  trace c, pkgname, "done traversing depencies"
   let cfgPath = if CfgHere in c.flags: CfgPath c.currentDir else: findCfgDir(c)
   patchNimCfg(c, paths, cfgPath)
+  trace c, pkgname, "executing post install actions"
   afterGraphActions c, g
 
 proc updateDir(c: var AtlasContext; dir, filter: string) =
@@ -341,10 +344,6 @@ proc main(c: var AtlasContext) =
     if c.projectDir == c.workspace or c.projectDir == c.depsDir:
       fatal action & " command must be executed in a project, not in the workspace"
 
-  proc findCurrentNimble(): string =
-    for x in walkPattern("*.nimble"):
-      return x
-
   var autoinit = false
   var explicitProjectOverride = false
   var explicitDepsDirOverride = false
@@ -459,23 +458,21 @@ proc main(c: var AtlasContext) =
     patchNimCfg c, deps, cfgPath
   of "use":
     singleArg()
-    #fillPackageLookupTable(c.nimbleContext, c, )
-    var amb = false
-    var nimbleFile = findNimbleFile(c, c.workspace, amb)
+    var nimbleFile = findNimbleFile(c, c.workspace)
     var nc = createNimbleContext(c, c.depsDir)
 
-    if nimbleFile.len == 0:
-      nimbleFile = c.workspace / extractProjectName(c.workspace) & ".nimble"
-      writeFile(nimbleFile, "")
-    patchNimbleFile(nc, c, c.overrides, nimbleFile, args[0])
+    if nimbleFile.isNone:
+      trace c, getCurrentDir().relativePath(c.workspace), "no nimble file found for project"
+      nimbleFile = some c.workspace / extractProjectName(c.workspace) & ".nimble"
+      writeFile(nimbleFile.get, "")
+      nimbleFile = findNimbleFile(c, c.workspace)
+      trace c, getCurrentDir().relativePath(c.workspace), "wrote new nimble file"
+
+    patchNimbleFile(nc, c, c.overrides, nimbleFile.get(), args[0])
     if c.errors > 0:
       discard "don't continue for 'cannot resolve'"
-    elif nimbleFile.len > 0 and not amb:
-      installDependencies(c, nc, nimbleFile)
-    elif amb:
-      error c, args[0], "ambiguous .nimble file"
     else:
-      error c, args[0], "cannot find .nimble file"
+      installDependencies(c, nc, nimbleFile.get())
 
   of "pin":
     optSingleArg(LockFileName)
@@ -500,16 +497,16 @@ proc main(c: var AtlasContext) =
     # projectCmd()
     if args.len > 1:
       fatal "install command takes a single argument"
-    var nimbleFile = ""
+    var nimbleFile: Option[string]
     if args.len == 1:
-      nimbleFile = args[0]
+      nimbleFile = some args[0]
     else:
-      nimbleFile = findCurrentNimble()
-    if nimbleFile.len == 0:
+      nimbleFile = findNimbleFile(c, getCurrentDir())
+    if nimbleFile.isNone:
       fatal "could not find a .nimble file"
     else:
       var nc = createNimbleContext(c, c.depsDir)
-      installDependencies(c, nc, nimbleFile)
+      installDependencies(c, nc, nimbleFile.get())
   of "refresh":
     noArgs()
     updatePackages(c, c.depsDir)

src/cloner.nim

@@ -53,9 +53,11 @@ proc cloneUrl*(c: var AtlasContext,
   infoNow c, url.projectName, "Cloning url: " & modurl
 
   # Checking repo with git
+  trace c, "atlas cloner", "checking repo " & $url
   let gitCmdStr = "git ls-remote --quiet --tags " & modurl
   var success = execCmdEx(gitCmdStr)[1] == QuitSuccess
   if not success and isGitHub:
+    trace c, "atlas cloner", "failed check ls-remote..."
     # retry multiple times to avoid annoying GitHub timeouts:
     success = retryUrl(gitCmdStr, modurl, c, url.projectName, false)
 
@@ -74,215 +76,8 @@ proc cloneUrl*(c: var AtlasContext,
           (OtherError, "exernal program failed: " & hgCmdStr)
   else:
     if gitops.clone(c, url.url, dest, fullClones=true): # gitops.clone has buit-in retrying
+      infoNow c, url.projectName, "Cloning success"
       (Ok, "")
     else:
+      infoNow c, url.projectName, "Cloning failed: " & modurl
       (OtherError, "exernal program failed: " & $GitClone)
-
-when false:
-  proc updatePackages*(c: var AtlasContext) =
-    if dirExists(c.depsDir / DefaultPackagesSubDir):
-      withDir(c, c.depsDir / DefaultPackagesSubDir):
-        gitPull(c, DefaultPackagesSubDir)
-    else:
-      withDir c, c.depsDir:
-        let (status, err) = cloneUrl(c, PkgUrl"https://github.com/nim-lang/packages", DefaultPackagesSubDir, false)
-        if status != Ok:
-          error c, DefaultPackagesSubDir, err
-
-  proc fillPackageLookupTable(c: var AtlasContext) =
-    if not c.hasPackageList:
-      c.hasPackageList = true
-      if not fileExists(c.depsDir / DefaultPackagesSubDir / "packages.json"):
-        updatePackages(c)
-      let plist = getPackageInfos(c.depsDir)
-      debug c, "fillPackageLookupTable", "initializing..."
-      for entry in plist:
-        let url = getUrl(entry.url)
-        let pkg = Package(name: PackageName unicode.toLower entry.name,
-                          repo: PackageRepo lastPathComponent($url),
-                          url: url)
-        c.urlMapping["name:" & pkg.name.string] = pkg
-
-when false:
-  proc dependencyDir*(c: var AtlasContext; pkg: Package): PackageDir =
-    template checkDir(dir: string) =
-      if dir.len > 0 and dirExists(dir):
-        debug c, pkg, "dependencyDir: found: " & dir
-        return PackageDir dir
-      else:
-        debug c, pkg, "dependencyDir: not found: " & dir
-
-    debug c, pkg, "dependencyDir: check: pth: " & pkg.path.string & " cd: " & getCurrentDir() & " ws: " & c.workspace
-    if pkg.exists:
-      debug c, pkg, "dependencyDir: exists: " & pkg.path.string
-      return PackageDir pkg.path.string.absolutePath
-    if c.workspace.lastPathComponent == pkg.repo.string:
-      debug c, pkg, "dependencyDir: workspace: " & c.workspace
-      return PackageDir c.workspace
-
-    if pkg.path.string.len > 0:
-      checkDir pkg.path.string
-      checkDir c.workspace / pkg.path.string
-      checkDir c.depsDir / pkg.path.string
-
-    checkDir c.workspace / pkg.repo.string
-    checkDir c.depsDir / pkg.repo.string
-    checkDir c.workspace / pkg.name.string
-    checkDir c.depsDir / pkg.name.string
-    result = PackageDir c.depsDir / pkg.repo.string
-    trace c, pkg, "dependency not found using default"
-
-  proc findNimbleFile*(c: var AtlasContext; pkg: Package; depDir = PackageDir""): string =
-    let dir = if depDir.string.len == 0: dependencyDir(c, pkg).string
-              else: depDir.string
-    result = dir / (pkg.name.string & ".nimble")
-    debug c, pkg, "findNimbleFile: searching: " & pkg.repo.string & " path: " & pkg.path.string & " dir: " & dir & " curr: " & result
-    if not fileExists(result):
-      debug c, pkg, "findNimbleFile: not found: " & result
-      result = ""
-      for file in walkFiles(dir / "*.nimble"):
-        if result.len == 0:
-          result = file
-          trace c, pkg, "nimble file found " & result
-        else:
-          error c, pkg, "ambiguous .nimble file " & result
-          return ""
-    else:
-      trace c, pkg, "nimble file found " & result
-
-when false:
-  proc resolvePackageUrl(c: var AtlasContext; url: string, checkOverrides = true): Package =
-    result = Package(url: getUrl(url),
-                    name: url.toRepo().PackageName,
-                    repo: url.toRepo())
-
-    debug c, result, "resolvePackageUrl: search: " & url
-
-    let isFile = result.url.scheme == "file"
-    var isUrlOverriden = false
-    if not isFile and checkOverrides and UsesOverrides in c.flags:
-      let url = c.overrides.substitute($result.url)
-      if url.len > 0:
-        warn c, result, "resolvePackageUrl: url override found: " & $url
-        result.url = url.getUrl()
-        isUrlOverriden = true
-
-    let namePkg = c.urlMapping.getOrDefault("name:" & result.name.string, nil)
-    let repoPkg = c.urlMapping.getOrDefault("repo:" & result.repo.string, nil)
-
-    if not namePkg.isNil:
-      debug c, result, "resolvePackageUrl: found by name: " & $result.name.string
-      if namePkg.url != result.url and isUrlOverriden:
-        namePkg.url = result.url # update package url to match
-        result = namePkg
-      elif namePkg.url != result.url:
-        # package conflicts
-        # change package repo to `repo.user.host`
-        let purl = result.url
-        let host = purl.hostname
-        let org = purl.path.parentDir.lastPathPart
-        let rname = purl.path.lastPathPart
-        let pname = [rname, org, host].join(".")
-        warn c, result,
-                "conflicting url's for package; renaming package: " &
-                  result.name.string & " to " & pname
-        result.repo = PackageRepo pname
-        c.urlMapping["name:" & result.name.string] = result
-      else:
-        result = namePkg
-    elif not repoPkg.isNil:
-      debug c, result, "resolvePackageUrl: found by repo: " & $result.repo.string
-      result = repoPkg
-    else:
-      # package doesn't exit and doesn't conflict
-      # set the url with package name as url name
-      c.urlMapping["repo:" & result.name.string] = result
-      trace c, result, "resolvePackageUrl: not found; set pkg: " & $result.repo.string
-
-    #if result.url.scheme == "file":
-    #  result.path = PackageDir result.url.hostname & result.url.path
-    #  trace c, result, "resolvePackageUrl: setting manual path: " & $result.path.string
-
-  proc resolvePackageName(c: var AtlasContext; name: string): Package =
-    result = Package(name: PackageName name,
-                    repo: PackageRepo name)
-
-    # the project name can be overwritten too!
-    if UsesOverrides in c.flags:
-      let name = c.overrides.substitute(name)
-      if name.len > 0:
-        if name.isUrl():
-          return c.resolvePackageUrl(name, checkOverrides=false)
-
-    # echo "URL MAP: ", repr c.urlMapping.keys().toSeq()
-    let namePkg = c.urlMapping.getOrDefault("name:" & result.name.string, nil)
-    let repoPkg = c.urlMapping.getOrDefault("repo:" & result.name.string, nil)
-
-    debug c, result, "resolvePackageName: searching for package name: " & result.name.string
-    if not namePkg.isNil:
-      # great, found package!
-      debug c, result, "resolvePackageName: found!"
-      result = namePkg
-      result.inPackages = true
-    elif not repoPkg.isNil:
-      # check if rawHandle is a package repo name
-      debug c, result, "resolvePackageName: found by repo!"
-      result = repoPkg
-      result.inPackages = true
-
-    if UsesOverrides in c.flags:
-      let newUrl = c.overrides.substitute($result.url)
-      if newUrl.len > 0:
-        trace c, result, "resolvePackageName: not url: UsesOverrides: " & $newUrl
-        result.url = getUrl newUrl
-
-  proc resolvePackage*(c: var AtlasContext; rawHandle: string): Package =
-    ## Takes a raw handle which can be a name, a repo name, or a url
-    ## and resolves it into a package. If not found it will create
-    ## a new one.
-    ##
-    ## Note that Package should be unique globally. This happens
-    ## by updating the packages list when new packages are added or
-    ## loaded from a packages.json.
-    ##
-    result = Package()
-
-    fillPackageLookupTable(c)
-
-    trace c, rawHandle, "resolving package"
-
-    if rawHandle.isUrl:
-      result = c.resolvePackageUrl(rawHandle)
-    else:
-      result = c.resolvePackageName(unicode.toLower(rawHandle))
-
-    result.path = dependencyDir(c, result)
-    let res = c.findNimbleFile(result, result.path)
-    if res.len > 0:
-      let nimble = PackageNimble res
-      result.exists = true
-      result.nimble = nimble
-      # the nimble package name is <name>.nimble
-      result.name = PackageName nimble.string.splitFile().name
-      debug c, result, "resolvePackageName: nimble: found: " & $result
-    else:
-      debug c, result, "resolvePackageName: nimble: not found: " & $result
-
-
-  proc resolveNimble*(c: var AtlasContext; pkg: Package) =
-    ## Try to resolve the nimble file for the given package.
-    ##
-    ## This should be done after cloning a new repo.
-    ##
-    if pkg.exists: return
-
-    pkg.path = dependencyDir(c, pkg)
-    let res = c.findNimbleFile(pkg)
-    if res.len > 0:
-      let nimble = PackageNimble res
-      # let path = PackageDir res.parentDir()
-      pkg.exists = true
-      pkg.nimble = nimble
-      info c, pkg, "resolvePackageName: nimble: found: " & $pkg
-    else:
-      info c, pkg, "resolvePackageName: nimble: not found: " & $pkg

src/confighandler.nim

@@ -52,7 +52,11 @@ type
     graph: JsonNode
 
 proc writeDefaultConfigFile*(c: var AtlasContext) =
-  let config = JsonConfig(resolver: $SemVer, graph: newJNull())
+  let config = JsonConfig(
+    resolver: $SemVer,
+    graph: newJNull(),
+    deps: c.depsDir
+  )
   let configFile = c.workspace / AtlasWorkspace
   writeFile(configFile, pretty %*config)
 

src/context.nim

@@ -67,7 +67,7 @@ template projectFromCurrentDir*(): untyped = c.currentDir.absolutePath
 
 template withDir*(c: var AtlasContext; dir: string; body: untyped) =
   let oldDir = getCurrentDir()
-  debug c, dir, "Current directory is now: " & dir
+  debug c, dir.relativePath(c.workspace), "Current directory is now: " & dir
   try:
     setCurrentDir(dir)
     body