From 4d616a7788ef025a30b348277ffdae369f2242dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mi=C5=82osz=20Skaza?= Date: Thu, 4 Jul 2024 15:15:48 +0200 Subject: [PATCH] add support for git subrepo --- README.md | 11 +++ ctfcli/cli/challenges.py | 186 ++++++++++++++++++++++----------------- ctfcli/utils/git.py | 7 ++ 3 files changed, 124 insertions(+), 80 deletions(-) diff --git a/README.md b/README.md index da36616..3479ff0 100644 --- a/README.md +++ b/README.md @@ -205,3 +205,14 @@ The specification format has already been tested and used with CTFd in productio `ctfcli` plugins are essentially additions to the command line interface via dynamic class modifications. See the [plugin documentation page](docs/plugins.md) for a simple example. *`ctfcli` is an alpha project! The plugin interface is likely to change!* + +# Sub-Repos as alternative to Sub-Trees + +`ctfcli` manages git-based challenges by using the built-in git `subtree` mechanism. While it works most of the time, it's been proven to have disadvantages and tends to create problems and merge conflicts. + +As an alternative, we're currently experimenting with the git [`git subrepo`](https://github.com/ingydotnet/git-subrepo) extension. +This functionality can be enabled by adding a `use_subrepo = True` property to the `[config]` section inside a ctfcli project config. + +Subrepo has to be installed separately, and is not backwards compatible with the default `subtree`. +Once challenges have been added by using either method, they will not work properly if you change it, and you will have to add the challenges again. + diff --git a/ctfcli/cli/challenges.py b/ctfcli/cli/challenges.py index 51755e6..551969b 100644 --- a/ctfcli/cli/challenges.py +++ b/ctfcli/cli/challenges.py @@ -20,7 +20,7 @@ LintException, RemoteChallengeNotFound, ) -from ctfcli.utils.git import get_git_repo_head_branch +from ctfcli.utils.git import check_if_git_subrepo_is_installed, get_git_repo_head_branch log = logging.getLogger("ctfcli.cli.challenges") @@ -119,17 +119,24 @@ def templates(self) -> int: return TemplatesCommand.list() - def add(self, repo: str, directory: str = None, yaml_path: str = None) -> int: - log.debug(f"add: {repo} (directory={directory}, yaml_path={yaml_path})") + def add( + self, repo: str, directory: str = None, branch: str = None, force: bool = False, yaml_path: str = None + ) -> int: + log.debug(f"add: {repo} (directory={directory}, branch={branch}, force={force}, yaml_path={yaml_path})") config = Config() - # check if we're working with a remote challenge which has to be pulled first + # Check if we're working with a remote challenge which has to be pulled first if repo.endswith(".git"): + use_subrepo = config["config"].getboolean("use_subrepo", fallback=False) + if use_subrepo and not check_if_git_subrepo_is_installed(): + click.secho("This project is configured to use git subrepo, but it's not installed.") + return 1 + # Get a relative path from project root to current directory project_path = config.project_path project_relative_cwd = Path.cwd().relative_to(project_path) - # Get a new directory that will add the git subtree + # Get a new directory that will add the git subtree / git subrepo repository_basename = Path(repo).stem # Use the custom subdirectory for the challenge if one was provided @@ -148,29 +155,25 @@ def add(self, repo: str, directory: str = None, yaml_path: str = None) -> int: # Add a new challenge to the config config["challenges"][str(challenge_key)] = repo - head_branch = get_git_repo_head_branch(repo) - log.debug( - f"call(['git', 'subtree', 'add', '--prefix', '{challenge_path}', " - f"'{repo}', '{head_branch}', '--squash'], cwd='{project_path}')" - ) - git_subtree_add = subprocess.call( - [ - "git", - "subtree", - "add", - "--prefix", - challenge_path, - repo, - head_branch, - "--squash", - ], - cwd=project_path, - ) + if use_subrepo: + # Clone with subrepo if configured + cmd = ["git", "subrepo", "clone", repo, challenge_path] - if git_subtree_add != 0: + if branch is not None: + cmd += ["-b", branch] + + if force: + cmd += ["-f"] + else: + # Otherwise default to the built-in subtree + head_branch = get_git_repo_head_branch(repo) + cmd = ["git", "subtree", "add", "--prefix", challenge_path, repo, head_branch, "--squash"] + + log.debug(f"call({cmd}, cwd='{project_path}')") + if subprocess.call(cmd, cwd=project_path) != 0: click.secho( - "Could not add the challenge subtree. " "Please check git error messages above.", + "Could not add the challenge repository. Please check git error messages above.", fg="red", ) return 1 @@ -186,7 +189,7 @@ def add(self, repo: str, directory: str = None, yaml_path: str = None) -> int: if any(r != 0 for r in [git_add, git_commit]): click.secho( - "Could not commit the challenge subtree. " "Please check git error messages above.", + "Could not commit the challenge repository. Please check git error messages above.", fg="red", ) return 1 @@ -205,7 +208,7 @@ def add(self, repo: str, directory: str = None, yaml_path: str = None) -> int: return 1 def push(self, challenge: str = None, no_auto_pull: bool = False, quiet=False) -> int: - log.debug(f"push: (challenge={challenge})") + log.debug(f"push: (challenge={challenge}, no_auto_pull={no_auto_pull}, quiet={quiet})") config = Config() if challenge: @@ -224,6 +227,11 @@ def push(self, challenge: str = None, no_auto_pull: bool = False, quiet=False) - else: context = click.progressbar(challenges, label="Pushing challenges") + use_subrepo = config["config"].getboolean("use_subrepo", fallback=False) + if use_subrepo and not check_if_git_subrepo_is_installed(): + click.secho("This project is configured to use git subrepo, but it's not installed.") + return 1 + with context as context_challenges: for challenge_instance in context_challenges: click.echo() @@ -256,7 +264,6 @@ def push(self, challenge: str = None, no_auto_pull: bool = False, quiet=False) - continue click.secho(f"Pushing '{challenge_path}' to '{challenge_repo}'", fg="blue") - head_branch = get_git_repo_head_branch(challenge_repo) log.debug( f"call(['git', 'status', '--porcelain'], cwd='{config.project_path / challenge_path}'," @@ -287,32 +294,22 @@ def push(self, challenge: str = None, no_auto_pull: bool = False, quiet=False) - if any(r != 0 for r in [git_add, git_commit]): click.secho( - "Could not commit the challenge changes. " "Please check git error messages above.", + "Could not commit the challenge changes. Please check git error messages above.", fg="red", ) failed_pushes.append(challenge_instance) continue - log.debug( - f"call(['git', 'subtree', 'push', '--prefix', '{challenge_path}', '{challenge_repo}', " - f"'{head_branch}'], cwd='{config.project_path / challenge_path}')" - ) - git_subtree_push = subprocess.call( - [ - "git", - "subtree", - "push", - "--prefix", - challenge_path, - challenge_repo, - head_branch, - ], - cwd=config.project_path, - ) + if use_subrepo: + cmd = ["git", "subrepo", "push", challenge_path] + else: + head_branch = get_git_repo_head_branch(challenge_repo) + cmd = ["git", "subtree", "push", "--prefix", challenge_path, challenge_repo, head_branch] - if git_subtree_push != 0: + log.debug(f"call({cmd}, cwd='{config.project_path / challenge_path}')") + if subprocess.call(cmd, cwd=config.project_path) != 0: click.secho( - "Could not push the challenge subtree. " "Please check git error messages above.", + "Could not push the challenge repository. Please check git error messages above.", fg="red", ) failed_pushes.append(challenge_instance) @@ -335,8 +332,8 @@ def push(self, challenge: str = None, no_auto_pull: bool = False, quiet=False) - return 1 - def pull(self, challenge: str = None, quiet=False) -> int: - log.debug(f"pull: (challenge={challenge})") + def pull(self, challenge: str = None, strategy: str = "fast-forward", quiet: bool = False) -> int: + log.debug(f"pull: (challenge={challenge}, quiet={quiet})") config = Config() if challenge: @@ -353,6 +350,11 @@ def pull(self, challenge: str = None, quiet=False) -> int: else: context = click.progressbar(challenges, label="Pulling challenges") + use_subrepo = config["config"].getboolean("use_subrepo", fallback=False) + if use_subrepo and not check_if_git_subrepo_is_installed(): + click.secho("This project is configured to use git subrepo, but it's not installed.") + return 1 + failed_pulls = [] with context as context_challenges: for challenge_instance in context_challenges: @@ -386,18 +388,25 @@ def pull(self, challenge: str = None, quiet=False) -> int: continue click.secho(f"Pulling latest '{challenge_repo}' to '{challenge_path}'", fg="blue") - head_branch = get_git_repo_head_branch(challenge_repo) - - log.debug( - f"call(['git', 'subtree', 'pull', '--prefix', '{challenge_path}', " - f"'{challenge_repo}', '{head_branch}', '--squash'], cwd='{config.project_path}')" - ) pull_env = os.environ.copy() - pull_env["GIT_MERGE_AUTOEDIT"] = "no" - - git_subtree_pull = subprocess.call( - [ + if use_subrepo: + cmd = ["git", "subrepo", "pull", challenge_path] + + if strategy == "rebase": + cmd += ["--rebase"] + elif strategy == "merge": + cmd += ["--merge"] + elif strategy == "force": + cmd += ["--force"] + elif strategy == "fast-forward": + pass # fast-forward is the default strategy + else: + click.secho(f"Cannot pull challenge - '{strategy}' is not a valid pull strategy", fg="red") + else: + head_branch = get_git_repo_head_branch(challenge_repo) + pull_env["GIT_MERGE_AUTOEDIT"] = "no" + cmd = [ "git", "subtree", "pull", @@ -406,12 +415,10 @@ def pull(self, challenge: str = None, quiet=False) -> int: challenge_repo, head_branch, "--squash", - ], - cwd=config.project_path, - env=pull_env, - ) + ] - if git_subtree_pull != 0: + log.debug(f"call({cmd}, cwd='{config.project_path})") + if subprocess.call(cmd, cwd=config.project_path, env=pull_env) != 0: click.secho( f"Could not pull the subtree for challenge '{challenge_path}'. " "Please check git error messages above.", @@ -420,25 +427,26 @@ def pull(self, challenge: str = None, quiet=False) -> int: failed_pulls.append(challenge_instance) continue - log.debug(f"call(['git', 'mergetool'], cwd='{config.project_path / challenge_path}')") - git_mergetool = subprocess.call(["git", "mergetool"], cwd=config.project_path / challenge_path) + if not use_subrepo: + log.debug(f"call(['git', 'mergetool'], cwd='{config.project_path / challenge_path}')") + git_mergetool = subprocess.call(["git", "mergetool"], cwd=config.project_path / challenge_path) - log.debug(f"call(['git', 'commit', '--no-edit'], cwd='{config.project_path / challenge_path}')") - subprocess.call(["git", "commit", "--no-edit"], cwd=config.project_path / challenge_path) + log.debug(f"call(['git', 'commit', '--no-edit'], cwd='{config.project_path / challenge_path}')") + subprocess.call(["git", "commit", "--no-edit"], cwd=config.project_path / challenge_path) - log.debug(f"call(['git', 'clean', '-f'], cwd='{config.project_path / challenge_path}')") - git_clean = subprocess.call(["git", "clean", "-f"], cwd=config.project_path / challenge_path) + log.debug(f"call(['git', 'clean', '-f'], cwd='{config.project_path / challenge_path}')") + git_clean = subprocess.call(["git", "clean", "-f"], cwd=config.project_path / challenge_path) - # git commit is allowed to return a non-zero code - # because it would also mean that there's nothing to commit - if any(r != 0 for r in [git_mergetool, git_clean]): - click.secho( - f"Could not commit the subtree for challenge '{challenge_path}'. " - "Please check git error messages above.", - fg="red", - ) - failed_pulls.append(challenge_instance) - continue + # git commit is allowed to return a non-zero code + # because it would also mean that there's nothing to commit + if any(r != 0 for r in [git_mergetool, git_clean]): + click.secho( + f"Could not commit the changes for challenge '{challenge_path}'. " + "Please check git error messages above.", + fg="red", + ) + failed_pulls.append(challenge_instance) + continue if len(failed_pulls) == 0: if not quiet: @@ -460,6 +468,11 @@ def restore(self, challenge: str = None) -> int: click.secho("Could not find any added challenges to restore", fg="yellow") return 1 + use_subrepo = config["config"].getboolean("use_subrepo", fallback=False) + if use_subrepo and not check_if_git_subrepo_is_installed(): + click.secho("This project is configured to use git subrepo, but it's not installed.") + return 1 + failed_restores = [] for challenge_key, challenge_source in config.challenges.items(): if challenge is not None and challenge_key != challenge: @@ -483,6 +496,19 @@ def restore(self, challenge: str = None) -> int: failed_restores.append(challenge_key) continue + # If we're using subrepo - the restore can be achieved by performing a force pull + if use_subrepo: + if self.pull(challenge, strategy="force") != 0: + click.secho( + f"Failed to restore challenge '{challenge_key}' via subrepo force pull. " + "Please check git error messages above.", + fg="red", + ) + failed_restores.append(challenge_key) + + continue + + # Otherwise - default to restoring the repository via re-adding the subtree # Check if target directory exits if (config.project_path / challenge_key).exists(): click.secho( diff --git a/ctfcli/utils/git.py b/ctfcli/utils/git.py index a2ab596..6e8b659 100644 --- a/ctfcli/utils/git.py +++ b/ctfcli/utils/git.py @@ -3,6 +3,13 @@ from typing import Optional, Union +def check_if_git_subrepo_is_installed() -> bool: + output = subprocess.run(["git", "subrepo"], capture_output=True, text=True) + if "git: 'subrepo' is not a git command" in output.stderr: + return False + return True + + def get_git_repo_head_branch(repo: str) -> Optional[str]: """ A helper method to get the reference of the HEAD branch of a git remote repo.