Add .rehydrate_comfy_deps method to StandalonePython (#162)

* add progress bar to tarfile creation in `StandalonePython`

* add `extraSpecs` parameter to `DependencyCompiler` constructor

* make `extraSpecs` parameter actually function

* `StandalonePython.to_tarball`: better progress bar for writing tarball

* simplify methods of `StandalonePython` into single `.dehydrate_comfy_deps`

* add methods to `DependencyCompiler` to install from downloaded artifacts

* fix pathing in tarball creation

* in `StandalonePython`, add `.rehydrate_comfy_deps` method

* add `--rehydrate` option to `standalone` command

* fix `--rehydrate` option

* improve wheel install triggered by `--rehydrate` option

.gitignore

@@ -3,7 +3,6 @@ __pycache__/
 
 #COMMON CONFIGs
 .DS_Store
-.ruff_cache
 .src_port
 .webpack_watch.log
 *.swp
@@ -48,7 +47,9 @@ share/python-wheels/
 *.manifest
 *.spec
 
-# temporary files created by tests
+# temporary files created by linting, tests, etc
+.pytest_cache/
+.ruff_cache/
 tests/temp/
 
 venv/

comfy_cli/cmdline.py

@@ -547,6 +547,13 @@ def feedback():
 @app.command(help="Download a standalone Python interpreter and dependencies based on an existing comfyui workspace")
 @tracking.track_command()
 def standalone(
+    cli_spec: Annotated[
+        str,
+        typer.Option(
+            show_default=False,
+            help="setuptools-style requirement specificer pointing to an instance of comfy-cli",
+        ),
+    ] = "comfy-cli",
     platform: Annotated[
         Optional[constants.OS],
         typer.Option(
@@ -603,6 +610,13 @@ def standalone(
             callback=g_gpu_exclusivity.validate,
         ),
     ] = None,
+    rehydrate: Annotated[
+        bool,
+        typer.Option(
+            show_default=False,
+            help="Create standalone Python for CPU",
+        ),
+    ] = False,
 ):
     comfy_path, _ = workspace_manager.get_workspace_path()
 
@@ -645,9 +659,13 @@ def standalone(
                 raise typer.Exit(code=0)
         print("[bold yellow]Installing on Intel ARC is in beta stage.[/bold yellow]")
 
-    sty = StandalonePython.FromDistro(platform=platform, proc=proc)
-    sty.precache_comfy_deps(comfyDir=comfy_path, gpu=gpu)
-    sty.to_tarball()
+    if rehydrate:
+        sty = StandalonePython.FromTarball(fpath="python.tgz")
+        sty.rehydrate_comfy_deps()
+    else:
+        sty = StandalonePython.FromDistro(platform=platform, proc=proc)
+        sty.dehydrate_comfy_deps(comfyDir=comfy_path, gpu=gpu, extraSpecs=cli_spec)
+        sty.to_tarball()
 
 
 app.add_typer(models_command.app, name="model", help="Manage models.")

comfy_cli/command/custom_nodes/cm_cli_util.py

@@ -63,7 +63,8 @@ def execute_cm_cli(args, channel=None, fast_deps=False, mode=None) -> str | None
         if fast_deps and args[0] in _dependency_cmds:
             # we're using the fast_deps behavior and just ran a command that invalidated the dependencies
             depComp = DependencyCompiler(cwd=workspace_path)
-            depComp.install_comfy_deps()
+            depComp.compile_deps()
+            depComp.install_deps()
 
         return result.stdout
     except subprocess.CalledProcessError as e:

comfy_cli/command/install.py

@@ -229,7 +229,8 @@ def execute(
 
     if fast_deps:
         depComp = DependencyCompiler(cwd=repo_dir, gpu=gpu)
-        depComp.install_comfy_deps()
+        depComp.compile_deps()
+        depComp.install_deps()
 
     if not skip_manager:
         update_node_id_cache()

comfy_cli/standalone.py

@@ -6,6 +6,9 @@
 from typing import Optional
 
 import requests
+from rich.live import Live
+from rich.progress import Progress, TextColumn
+from rich.table import Table
 
 from comfy_cli.constants import GPU_OPTION, OS, PROC
 from comfy_cli.typing import PathLike
@@ -146,31 +149,58 @@ def run_comfy_cli(self, *args: list[str]):
     def install_comfy(self, *args: list[str], gpu_arg: str = "--nvidia"):
         self.run_comfy_cli("--here", "--skip-prompt", "install", "--fast-deps", gpu_arg, *args)
 
-    def compile_comfy_deps(self, comfyDir: PathLike, gpu: GPU_OPTION, outDir: Optional[PathLike] = None):
-        outDir = self.rpath if outDir is None else outDir
+    def dehydrate_comfy_deps(
+        self,
+        comfyDir: PathLike,
+        gpu: GPU_OPTION,
+        extraSpecs: Optional[list[str]] = None,
+    ):
+        self.dep_comp = DependencyCompiler(
+            cwd=comfyDir,
+            executable=self.executable,
+            gpu=gpu,
+            outDir=self.rpath,
+            extraSpecs=extraSpecs,
+        )
+        self.dep_comp.compile_deps()
+        self.dep_comp.fetch_dep_wheels()
 
-        self.dep_comp = DependencyCompiler(cwd=comfyDir, executable=self.executable, gpu=gpu, outDir=outDir)
-        self.dep_comp.compile_comfy_deps()
+    def rehydrate_comfy_deps(self):
+        self.dep_comp = DependencyCompiler(
+            executable=self.executable, outDir=self.rpath, reqFilesCore=[], reqFilesExt=[]
+        )
+        self.dep_comp.install_wheels_directly()
 
-    def install_comfy_deps(self, comfyDir: PathLike, gpu: GPU_OPTION, outDir: Optional[PathLike] = None):
-        outDir = self.rpath if outDir is None else outDir
+    def to_tarball(self, outPath: Optional[PathLike] = None, progress: bool = True):
+        outPath = self.rpath.with_suffix(".tgz") if outPath is None else Path(outPath)
 
-        self.dep_comp = DependencyCompiler(cwd=comfyDir, executable=self.executable, gpu=gpu, outDir=outDir)
-        self.dep_comp.install_core_plus_ext()
+        if progress:
+            fileSize = sum(f.stat().st_size for f in self.rpath.glob("**/*"))
 
-    def precache_comfy_deps(self, comfyDir: PathLike, gpu: GPU_OPTION, outDir: Optional[PathLike] = None):
-        outDir = self.rpath if outDir is None else outDir
+            barProg = Progress()
+            addTar = barProg.add_task("[cyan]Creating tarball...", total=fileSize)
+            pathProg = Progress(TextColumn("{task.description}"))
+            pathTar = pathProg.add_task("")
 
-        self.dep_comp = DependencyCompiler(cwd=comfyDir, executable=self.executable, gpu=gpu, outDir=outDir)
-        self.dep_comp.precache_comfy_deps()
+            progress_table = Table.grid()
+            progress_table.add_row(barProg)
+            progress_table.add_row(pathProg)
 
-    def wheel_comfy_deps(self, comfyDir: PathLike, gpu: GPU_OPTION, outDir: Optional[PathLike] = None):
-        outDir = self.rpath if outDir is None else outDir
+            _size = 0
 
-        self.dep_comp = DependencyCompiler(cwd=comfyDir, executable=self.executable, gpu=gpu, outDir=outDir)
-        self.dep_comp.wheel_comfy_deps()
+            def _filter(tinfo: tarfile.TarInfo):
+                nonlocal _size
+                pathProg.update(pathTar, description=tinfo.path)
+                barProg.advance(addTar, _size)
+                _size = Path(tinfo.path).stat().st_size
+                return tinfo
+        else:
+            _filter = None
 
-    def to_tarball(self, outPath: Optional[PathLike] = None):
-        outPath = self.rpath.with_suffix(".tgz") if outPath is None else Path(outPath)
-        with tarfile.open(outPath, "w:gz") as tar:
-            tar.add(self.rpath, arcname=self.rpath.parent)
+        with Live(progress_table, refresh_per_second=10):
+            with tarfile.open(outPath, "w:gz") as tar:
+                tar.add(self.rpath.relative_to(Path(".").expanduser().resolve()), filter=_filter)
+
+            if progress:
+                barProg.advance(addTar, _size)
+                pathProg.update(pathTar, description="")

comfy_cli/utils.py

@@ -90,7 +90,7 @@ def is_running(pid):
         return False
 
 
-def create_choice_completer(opts):
+def create_choice_completer(opts: list[str]):
     def f(incomplete: str) -> list[str]:
         return [opt for opt in opts if opt.startswith(incomplete)]