From fe1295f91407bcc04c66c0878d32fa6efcb0bde9 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Mon, 25 Nov 2024 11:40:20 +0000 Subject: [PATCH] locally caches tagged packages --- src/nimblepkg/nimblesat.nim | 44 ++++++++++++++++++++++++++++++++++++- tests/tsat.nim | 41 ++++++++++++++++++++++++---------- 2 files changed, 73 insertions(+), 12 deletions(-) diff --git a/src/nimblepkg/nimblesat.nim b/src/nimblepkg/nimblesat.nim index 3aae89be..bbb16e65 100644 --- a/src/nimblepkg/nimblesat.nim +++ b/src/nimblepkg/nimblesat.nim @@ -2,7 +2,7 @@ import sat/[sat, satvars] import version, packageinfotypes, download, packageinfo, packageparser, options, sha1hashes, tools, downloadnim, cli -import std/[tables, sequtils, algorithm, sets, strutils, options, strformat, os] +import std/[tables, sequtils, algorithm, sets, strutils, options, strformat, os, json, jsonutils] type @@ -59,6 +59,23 @@ type GetPackageMinimal* = proc (pv: PkgTuple, options: Options): seq[PackageMinimalInfo] + TaggedPackageVersions = object + maxTaggedVersions: int # Maximum number of tags. When number changes, we invalidate the cache + versions: seq[PackageMinimalInfo] + +const TaggedVersionsFileName* = "tagged_versions.json" + +proc initFromJson*(dst: var PkgTuple, jsonNode: JsonNode, jsonPath: var string) = + dst = parseRequires(jsonNode.str) + +proc toJsonHook*(src: PkgTuple): JsonNode = + let ver = if src.ver.kind == verAny: "" else: $src.ver + case src.ver.kind + of verAny: newJString(src.name) + of verSpecial: newJString(src.name & ver) + else: + newJString(src.name & " " & ver) + #From the STD as it is not available in older Nim versions func addUnique*[T](s: var seq[T], x: sink T) = ## Adds `x` to the container `s` if it is not already present. @@ -444,9 +461,33 @@ proc getAllNimReleases(options: Options): seq[PackageMinimalInfo] = for release in releases: result.add PackageMinimalInfo(name: "nim", version: release) +proc getTaggedVersions*(repoDir: string, options: Options): Option[TaggedPackageVersions] = + let file = repoDir / TaggedVersionsFileName + if file.fileExists: + try: + let taggedVersions = file.readFile.parseJson().to(TaggedPackageVersions) + if taggedVersions.maxTaggedVersions != options.maxTaggedVersions: + return none(TaggedPackageVersions) + return some taggedVersions + except CatchableError as e: + displayWarning(&"Error reading tagged versions: {e.msg}", HighPriority) + return none(TaggedPackageVersions) + else: + none(TaggedPackageVersions) + +proc saveTaggedVersions*(repoDir: string, taggedVersions: TaggedPackageVersions) = + try: + let file = repoDir / TaggedVersionsFileName + file.writeFile((taggedVersions.toJson()).pretty) + except CatchableError as e: + displayWarning(&"Error saving tagged versions: {e.msg}", HighPriority) + proc getPackageMinimalVersionsFromRepo*(repoDir, pkgName: string, downloadMethod: DownloadMethod, options: Options): seq[PackageMinimalInfo] = #This is expensive. We need to cache it. Potentially it could be also run in parallel # echo &"Discovering version for {pkgName}" + let taggedVersions = getTaggedVersions(repoDir, options) + if taggedVersions.isSome: + return taggedVersions.get.versions gitFetchTags(repoDir, downloadMethod) #First package must be the current one result.add getPkgInfo(repoDir, options).getMinimalInfo(options) @@ -467,6 +508,7 @@ proc getPackageMinimalVersionsFromRepo*(repoDir, pkgName: string, downloadMethod except CatchableError as e: displayWarning(&"Error reading tag {tag}: for package {pkgName}. This may not be relevant as it could be an old version of the package. \n {e.msg}", HighPriority) + saveTaggedVersions(repoDir, TaggedPackageVersions(maxTaggedVersions: options.maxTaggedVersions, versions: result)) proc downloadMinimalPackage*(pv: PkgTuple, options: Options): seq[PackageMinimalInfo] = if pv.name == "": return newSeq[PackageMinimalInfo]() if pv.isNim and not options.disableNimBinaries: return getAllNimReleases(options) diff --git a/tests/tsat.nim b/tests/tsat.nim index d3eb368d..d66bf3e3 100644 --- a/tests/tsat.nim +++ b/tests/tsat.nim @@ -6,16 +6,6 @@ import std/[tables, sequtils, json, jsonutils, strutils, times, options, strform import nimblepkg/[version, nimblesat, options, config, download, packageinfotypes, packageinfo] from nimblepkg/common import cd -proc initFromJson*(dst: var PkgTuple, jsonNode: JsonNode, jsonPath: var string) = - dst = parseRequires(jsonNode.str) - -proc toJsonHook*(src: PkgTuple): JsonNode = - let ver = if src.ver.kind == verAny: "" else: $src.ver - case src.ver.kind - of verAny: newJString(src.name) - of verSpecial: newJString(src.name & ver) - else: - newJString(src.name & " " & ver) #Test utils: proc downloadAndStorePackageVersionTableFor(pkgName: string, options: Options) = @@ -305,13 +295,42 @@ suite "SAT solver": let pv = parseRequires("nimfp >= 0.3.4") let repoDir = pv.downloadPkgFromUrl(options)[0].dir #This is just to setup the test. We need a git dir to work on let downloadMethod = DownloadMethod git - let packageVersions = getPackageMinimalVersionsFromRepo(repoDir, pv[0], downloadMethod, options) #we know these versions are available let availableVersions = @["0.3.4", "0.3.5", "0.3.6", "0.4.5", "0.4.4"].mapIt(newVersion(it)) for version in availableVersions: check version in packageVersions.mapIt(it.version) + check fileExists(repoDir / TaggedVersionsFileName) + + test "should not use the cache when switching versions": + var options = initOptions() + options.maxTaggedVersions = 0 #all + options.nimBin = some options.makeNimBin("nim") + options.config.packageLists["official"] = PackageList(name: "Official", urls: @[ + "https://raw.githubusercontent.com/nim-lang/packages/master/packages.json", + "https://nim-lang.org/nimble/packages.json" + ]) + for dir in walkDir(".", true): + if dir.kind == PathComponent.pcDir and dir.path.startsWith("githubcom_vegansknimfp"): + echo "Removing dir", dir.path + removeDir(dir.path) + + let pvPrev = parseRequires("nimfp >= 0.3.4") + let repoDirPrev = pvPrev.downloadPkgFromUrl(options)[0].dir + discard getPackageMinimalVersionsFromRepo(repoDirPrev, pvPrev[0], DownloadMethod.git, options) + check fileExists(repoDirPrev / TaggedVersionsFileName) + + let pv = parseRequires("nimfp >= 0.4.4") + let repoDir = pv.downloadPkgFromUrl(options)[0].dir + check not fileExists(repoDir / TaggedVersionsFileName) + + let packageVersions = getPackageMinimalVersionsFromRepo(repoDir, pv[0], DownloadMethod.git, options) + #we know these versions are available + let availableVersions = @["0.4.5", "0.4.4"].mapIt(newVersion(it)) + for version in availableVersions: + check version in packageVersions.mapIt(it.version) + check fileExists(repoDir / TaggedVersionsFileName) test "if a dependency is unsatisfable, it should fallback to the previous version of the depency when available": let pkgVersionTable = {