Skip to content

Commit

Permalink
add support for git subrepo
Browse files Browse the repository at this point in the history
  • Loading branch information
Miłosz Skaza committed Jul 4, 2024
1 parent 31ec803 commit 4d616a7
Show file tree
Hide file tree
Showing 3 changed files with 124 additions and 80 deletions.
11 changes: 11 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,3 +205,14 @@ The specification format has already been tested and used with CTFd in productio
`ctfcli` plugins are essentially additions to the command line interface via dynamic class modifications. See the [plugin documentation page](docs/plugins.md) for a simple example.

*`ctfcli` is an alpha project! The plugin interface is likely to change!*

# Sub-Repos as alternative to Sub-Trees

`ctfcli` manages git-based challenges by using the built-in git `subtree` mechanism. While it works most of the time, it's been proven to have disadvantages and tends to create problems and merge conflicts.

As an alternative, we're currently experimenting with the git [`git subrepo`](https://github.com/ingydotnet/git-subrepo) extension.
This functionality can be enabled by adding a `use_subrepo = True` property to the `[config]` section inside a ctfcli project config.

Subrepo has to be installed separately, and is not backwards compatible with the default `subtree`.
Once challenges have been added by using either method, they will not work properly if you change it, and you will have to add the challenges again.

186 changes: 106 additions & 80 deletions ctfcli/cli/challenges.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
LintException,
RemoteChallengeNotFound,
)
from ctfcli.utils.git import get_git_repo_head_branch
from ctfcli.utils.git import check_if_git_subrepo_is_installed, get_git_repo_head_branch

log = logging.getLogger("ctfcli.cli.challenges")

Expand Down Expand Up @@ -119,17 +119,24 @@ def templates(self) -> int:

return TemplatesCommand.list()

def add(self, repo: str, directory: str = None, yaml_path: str = None) -> int:
log.debug(f"add: {repo} (directory={directory}, yaml_path={yaml_path})")
def add(
self, repo: str, directory: str = None, branch: str = None, force: bool = False, yaml_path: str = None
) -> int:
log.debug(f"add: {repo} (directory={directory}, branch={branch}, force={force}, yaml_path={yaml_path})")
config = Config()

# check if we're working with a remote challenge which has to be pulled first
# Check if we're working with a remote challenge which has to be pulled first
if repo.endswith(".git"):
use_subrepo = config["config"].getboolean("use_subrepo", fallback=False)
if use_subrepo and not check_if_git_subrepo_is_installed():
click.secho("This project is configured to use git subrepo, but it's not installed.")
return 1

# Get a relative path from project root to current directory
project_path = config.project_path
project_relative_cwd = Path.cwd().relative_to(project_path)

# Get a new directory that will add the git subtree
# Get a new directory that will add the git subtree / git subrepo
repository_basename = Path(repo).stem

# Use the custom subdirectory for the challenge if one was provided
Expand All @@ -148,29 +155,25 @@ def add(self, repo: str, directory: str = None, yaml_path: str = None) -> int:

# Add a new challenge to the config
config["challenges"][str(challenge_key)] = repo
head_branch = get_git_repo_head_branch(repo)

log.debug(
f"call(['git', 'subtree', 'add', '--prefix', '{challenge_path}', "
f"'{repo}', '{head_branch}', '--squash'], cwd='{project_path}')"
)
git_subtree_add = subprocess.call(
[
"git",
"subtree",
"add",
"--prefix",
challenge_path,
repo,
head_branch,
"--squash",
],
cwd=project_path,
)
if use_subrepo:
# Clone with subrepo if configured
cmd = ["git", "subrepo", "clone", repo, challenge_path]

if git_subtree_add != 0:
if branch is not None:
cmd += ["-b", branch]

if force:
cmd += ["-f"]
else:
# Otherwise default to the built-in subtree
head_branch = get_git_repo_head_branch(repo)
cmd = ["git", "subtree", "add", "--prefix", challenge_path, repo, head_branch, "--squash"]

log.debug(f"call({cmd}, cwd='{project_path}')")
if subprocess.call(cmd, cwd=project_path) != 0:
click.secho(
"Could not add the challenge subtree. " "Please check git error messages above.",
"Could not add the challenge repository. Please check git error messages above.",
fg="red",
)
return 1
Expand All @@ -186,7 +189,7 @@ def add(self, repo: str, directory: str = None, yaml_path: str = None) -> int:

if any(r != 0 for r in [git_add, git_commit]):
click.secho(
"Could not commit the challenge subtree. " "Please check git error messages above.",
"Could not commit the challenge repository. Please check git error messages above.",
fg="red",
)
return 1
Expand All @@ -205,7 +208,7 @@ def add(self, repo: str, directory: str = None, yaml_path: str = None) -> int:
return 1

def push(self, challenge: str = None, no_auto_pull: bool = False, quiet=False) -> int:
log.debug(f"push: (challenge={challenge})")
log.debug(f"push: (challenge={challenge}, no_auto_pull={no_auto_pull}, quiet={quiet})")
config = Config()

if challenge:
Expand All @@ -224,6 +227,11 @@ def push(self, challenge: str = None, no_auto_pull: bool = False, quiet=False) -
else:
context = click.progressbar(challenges, label="Pushing challenges")

use_subrepo = config["config"].getboolean("use_subrepo", fallback=False)
if use_subrepo and not check_if_git_subrepo_is_installed():
click.secho("This project is configured to use git subrepo, but it's not installed.")
return 1

with context as context_challenges:
for challenge_instance in context_challenges:
click.echo()
Expand Down Expand Up @@ -256,7 +264,6 @@ def push(self, challenge: str = None, no_auto_pull: bool = False, quiet=False) -
continue

click.secho(f"Pushing '{challenge_path}' to '{challenge_repo}'", fg="blue")
head_branch = get_git_repo_head_branch(challenge_repo)

log.debug(
f"call(['git', 'status', '--porcelain'], cwd='{config.project_path / challenge_path}',"
Expand Down Expand Up @@ -287,32 +294,22 @@ def push(self, challenge: str = None, no_auto_pull: bool = False, quiet=False) -

if any(r != 0 for r in [git_add, git_commit]):
click.secho(
"Could not commit the challenge changes. " "Please check git error messages above.",
"Could not commit the challenge changes. Please check git error messages above.",
fg="red",
)
failed_pushes.append(challenge_instance)
continue

log.debug(
f"call(['git', 'subtree', 'push', '--prefix', '{challenge_path}', '{challenge_repo}', "
f"'{head_branch}'], cwd='{config.project_path / challenge_path}')"
)
git_subtree_push = subprocess.call(
[
"git",
"subtree",
"push",
"--prefix",
challenge_path,
challenge_repo,
head_branch,
],
cwd=config.project_path,
)
if use_subrepo:
cmd = ["git", "subrepo", "push", challenge_path]
else:
head_branch = get_git_repo_head_branch(challenge_repo)
cmd = ["git", "subtree", "push", "--prefix", challenge_path, challenge_repo, head_branch]

if git_subtree_push != 0:
log.debug(f"call({cmd}, cwd='{config.project_path / challenge_path}')")
if subprocess.call(cmd, cwd=config.project_path) != 0:
click.secho(
"Could not push the challenge subtree. " "Please check git error messages above.",
"Could not push the challenge repository. Please check git error messages above.",
fg="red",
)
failed_pushes.append(challenge_instance)
Expand All @@ -335,8 +332,8 @@ def push(self, challenge: str = None, no_auto_pull: bool = False, quiet=False) -

return 1

def pull(self, challenge: str = None, quiet=False) -> int:
log.debug(f"pull: (challenge={challenge})")
def pull(self, challenge: str = None, strategy: str = "fast-forward", quiet: bool = False) -> int:
log.debug(f"pull: (challenge={challenge}, quiet={quiet})")
config = Config()

if challenge:
Expand All @@ -353,6 +350,11 @@ def pull(self, challenge: str = None, quiet=False) -> int:
else:
context = click.progressbar(challenges, label="Pulling challenges")

use_subrepo = config["config"].getboolean("use_subrepo", fallback=False)
if use_subrepo and not check_if_git_subrepo_is_installed():
click.secho("This project is configured to use git subrepo, but it's not installed.")
return 1

failed_pulls = []
with context as context_challenges:
for challenge_instance in context_challenges:
Expand Down Expand Up @@ -386,18 +388,25 @@ def pull(self, challenge: str = None, quiet=False) -> int:
continue

click.secho(f"Pulling latest '{challenge_repo}' to '{challenge_path}'", fg="blue")
head_branch = get_git_repo_head_branch(challenge_repo)

log.debug(
f"call(['git', 'subtree', 'pull', '--prefix', '{challenge_path}', "
f"'{challenge_repo}', '{head_branch}', '--squash'], cwd='{config.project_path}')"
)

pull_env = os.environ.copy()
pull_env["GIT_MERGE_AUTOEDIT"] = "no"

git_subtree_pull = subprocess.call(
[
if use_subrepo:
cmd = ["git", "subrepo", "pull", challenge_path]

if strategy == "rebase":
cmd += ["--rebase"]
elif strategy == "merge":
cmd += ["--merge"]
elif strategy == "force":
cmd += ["--force"]
elif strategy == "fast-forward":
pass # fast-forward is the default strategy
else:
click.secho(f"Cannot pull challenge - '{strategy}' is not a valid pull strategy", fg="red")
else:
head_branch = get_git_repo_head_branch(challenge_repo)
pull_env["GIT_MERGE_AUTOEDIT"] = "no"
cmd = [
"git",
"subtree",
"pull",
Expand All @@ -406,12 +415,10 @@ def pull(self, challenge: str = None, quiet=False) -> int:
challenge_repo,
head_branch,
"--squash",
],
cwd=config.project_path,
env=pull_env,
)
]

if git_subtree_pull != 0:
log.debug(f"call({cmd}, cwd='{config.project_path})")
if subprocess.call(cmd, cwd=config.project_path, env=pull_env) != 0:
click.secho(
f"Could not pull the subtree for challenge '{challenge_path}'. "
"Please check git error messages above.",
Expand All @@ -420,25 +427,26 @@ def pull(self, challenge: str = None, quiet=False) -> int:
failed_pulls.append(challenge_instance)
continue

log.debug(f"call(['git', 'mergetool'], cwd='{config.project_path / challenge_path}')")
git_mergetool = subprocess.call(["git", "mergetool"], cwd=config.project_path / challenge_path)
if not use_subrepo:
log.debug(f"call(['git', 'mergetool'], cwd='{config.project_path / challenge_path}')")
git_mergetool = subprocess.call(["git", "mergetool"], cwd=config.project_path / challenge_path)

log.debug(f"call(['git', 'commit', '--no-edit'], cwd='{config.project_path / challenge_path}')")
subprocess.call(["git", "commit", "--no-edit"], cwd=config.project_path / challenge_path)
log.debug(f"call(['git', 'commit', '--no-edit'], cwd='{config.project_path / challenge_path}')")
subprocess.call(["git", "commit", "--no-edit"], cwd=config.project_path / challenge_path)

log.debug(f"call(['git', 'clean', '-f'], cwd='{config.project_path / challenge_path}')")
git_clean = subprocess.call(["git", "clean", "-f"], cwd=config.project_path / challenge_path)
log.debug(f"call(['git', 'clean', '-f'], cwd='{config.project_path / challenge_path}')")
git_clean = subprocess.call(["git", "clean", "-f"], cwd=config.project_path / challenge_path)

# git commit is allowed to return a non-zero code
# because it would also mean that there's nothing to commit
if any(r != 0 for r in [git_mergetool, git_clean]):
click.secho(
f"Could not commit the subtree for challenge '{challenge_path}'. "
"Please check git error messages above.",
fg="red",
)
failed_pulls.append(challenge_instance)
continue
# git commit is allowed to return a non-zero code
# because it would also mean that there's nothing to commit
if any(r != 0 for r in [git_mergetool, git_clean]):
click.secho(
f"Could not commit the changes for challenge '{challenge_path}'. "
"Please check git error messages above.",
fg="red",
)
failed_pulls.append(challenge_instance)
continue

if len(failed_pulls) == 0:
if not quiet:
Expand All @@ -460,6 +468,11 @@ def restore(self, challenge: str = None) -> int:
click.secho("Could not find any added challenges to restore", fg="yellow")
return 1

use_subrepo = config["config"].getboolean("use_subrepo", fallback=False)
if use_subrepo and not check_if_git_subrepo_is_installed():
click.secho("This project is configured to use git subrepo, but it's not installed.")
return 1

failed_restores = []
for challenge_key, challenge_source in config.challenges.items():
if challenge is not None and challenge_key != challenge:
Expand All @@ -483,6 +496,19 @@ def restore(self, challenge: str = None) -> int:
failed_restores.append(challenge_key)
continue

# If we're using subrepo - the restore can be achieved by performing a force pull
if use_subrepo:
if self.pull(challenge, strategy="force") != 0:
click.secho(
f"Failed to restore challenge '{challenge_key}' via subrepo force pull. "
"Please check git error messages above.",
fg="red",
)
failed_restores.append(challenge_key)

continue

# Otherwise - default to restoring the repository via re-adding the subtree
# Check if target directory exits
if (config.project_path / challenge_key).exists():
click.secho(
Expand Down
7 changes: 7 additions & 0 deletions ctfcli/utils/git.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,13 @@
from typing import Optional, Union


def check_if_git_subrepo_is_installed() -> bool:
output = subprocess.run(["git", "subrepo"], capture_output=True, text=True)
if "git: 'subrepo' is not a git command" in output.stderr:
return False
return True


def get_git_repo_head_branch(repo: str) -> Optional[str]:
"""
A helper method to get the reference of the HEAD branch of a git remote repo.
Expand Down

0 comments on commit 4d616a7

Please sign in to comment.