Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add support for git subrepo #150

Merged
merged 1 commit into from
Jul 14, 2024
Merged

add support for git subrepo #150

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,3 +205,14 @@ The specification format has already been tested and used with CTFd in productio
`ctfcli` plugins are essentially additions to the command line interface via dynamic class modifications. See the [plugin documentation page](docs/plugins.md) for a simple example.

*`ctfcli` is an alpha project! The plugin interface is likely to change!*

# Sub-Repos as alternative to Sub-Trees

`ctfcli` manages git-based challenges by using the built-in git `subtree` mechanism. While it works most of the time, it's been proven to have disadvantages and tends to create problems and merge conflicts.

As an alternative, we're currently experimenting with the git [`git subrepo`](https://github.com/ingydotnet/git-subrepo) extension.
This functionality can be enabled by adding a `use_subrepo = True` property to the `[config]` section inside a ctfcli project config.

Subrepo has to be installed separately, and is not backwards compatible with the default `subtree`.
Once challenges have been added by using either method, they will not work properly if you change it, and you will have to add the challenges again.

186 changes: 106 additions & 80 deletions ctfcli/cli/challenges.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
LintException,
RemoteChallengeNotFound,
)
from ctfcli.utils.git import get_git_repo_head_branch
from ctfcli.utils.git import check_if_git_subrepo_is_installed, get_git_repo_head_branch

log = logging.getLogger("ctfcli.cli.challenges")

Expand Down Expand Up @@ -119,17 +119,24 @@ def templates(self) -> int:

return TemplatesCommand.list()

def add(self, repo: str, directory: str = None, yaml_path: str = None) -> int:
log.debug(f"add: {repo} (directory={directory}, yaml_path={yaml_path})")
def add(
self, repo: str, directory: str = None, branch: str = None, force: bool = False, yaml_path: str = None
) -> int:
log.debug(f"add: {repo} (directory={directory}, branch={branch}, force={force}, yaml_path={yaml_path})")
config = Config()

# check if we're working with a remote challenge which has to be pulled first
# Check if we're working with a remote challenge which has to be pulled first
if repo.endswith(".git"):
use_subrepo = config["config"].getboolean("use_subrepo", fallback=False)
if use_subrepo and not check_if_git_subrepo_is_installed():
click.secho("This project is configured to use git subrepo, but it's not installed.")
return 1

# Get a relative path from project root to current directory
project_path = config.project_path
project_relative_cwd = Path.cwd().relative_to(project_path)

# Get a new directory that will add the git subtree
# Get a new directory that will add the git subtree / git subrepo
repository_basename = Path(repo).stem

# Use the custom subdirectory for the challenge if one was provided
Expand All @@ -148,29 +155,25 @@ def add(self, repo: str, directory: str = None, yaml_path: str = None) -> int:

# Add a new challenge to the config
config["challenges"][str(challenge_key)] = repo
head_branch = get_git_repo_head_branch(repo)

log.debug(
f"call(['git', 'subtree', 'add', '--prefix', '{challenge_path}', "
f"'{repo}', '{head_branch}', '--squash'], cwd='{project_path}')"
)
git_subtree_add = subprocess.call(
[
"git",
"subtree",
"add",
"--prefix",
challenge_path,
repo,
head_branch,
"--squash",
],
cwd=project_path,
)
if use_subrepo:
# Clone with subrepo if configured
cmd = ["git", "subrepo", "clone", repo, challenge_path]

if git_subtree_add != 0:
if branch is not None:
cmd += ["-b", branch]

if force:
cmd += ["-f"]
else:
# Otherwise default to the built-in subtree
head_branch = get_git_repo_head_branch(repo)
cmd = ["git", "subtree", "add", "--prefix", challenge_path, repo, head_branch, "--squash"]

log.debug(f"call({cmd}, cwd='{project_path}')")
if subprocess.call(cmd, cwd=project_path) != 0:
click.secho(
"Could not add the challenge subtree. " "Please check git error messages above.",
"Could not add the challenge repository. Please check git error messages above.",
fg="red",
)
return 1
Expand All @@ -186,7 +189,7 @@ def add(self, repo: str, directory: str = None, yaml_path: str = None) -> int:

if any(r != 0 for r in [git_add, git_commit]):
click.secho(
"Could not commit the challenge subtree. " "Please check git error messages above.",
"Could not commit the challenge repository. Please check git error messages above.",
fg="red",
)
return 1
Expand All @@ -205,7 +208,7 @@ def add(self, repo: str, directory: str = None, yaml_path: str = None) -> int:
return 1

def push(self, challenge: str = None, no_auto_pull: bool = False, quiet=False) -> int:
log.debug(f"push: (challenge={challenge})")
log.debug(f"push: (challenge={challenge}, no_auto_pull={no_auto_pull}, quiet={quiet})")
config = Config()

if challenge:
Expand All @@ -224,6 +227,11 @@ def push(self, challenge: str = None, no_auto_pull: bool = False, quiet=False) -
else:
context = click.progressbar(challenges, label="Pushing challenges")

use_subrepo = config["config"].getboolean("use_subrepo", fallback=False)
if use_subrepo and not check_if_git_subrepo_is_installed():
click.secho("This project is configured to use git subrepo, but it's not installed.")
return 1

with context as context_challenges:
for challenge_instance in context_challenges:
click.echo()
Expand Down Expand Up @@ -256,7 +264,6 @@ def push(self, challenge: str = None, no_auto_pull: bool = False, quiet=False) -
continue

click.secho(f"Pushing '{challenge_path}' to '{challenge_repo}'", fg="blue")
head_branch = get_git_repo_head_branch(challenge_repo)

log.debug(
f"call(['git', 'status', '--porcelain'], cwd='{config.project_path / challenge_path}',"
Expand Down Expand Up @@ -287,32 +294,22 @@ def push(self, challenge: str = None, no_auto_pull: bool = False, quiet=False) -

if any(r != 0 for r in [git_add, git_commit]):
click.secho(
"Could not commit the challenge changes. " "Please check git error messages above.",
"Could not commit the challenge changes. Please check git error messages above.",
fg="red",
)
failed_pushes.append(challenge_instance)
continue

log.debug(
f"call(['git', 'subtree', 'push', '--prefix', '{challenge_path}', '{challenge_repo}', "
f"'{head_branch}'], cwd='{config.project_path / challenge_path}')"
)
git_subtree_push = subprocess.call(
[
"git",
"subtree",
"push",
"--prefix",
challenge_path,
challenge_repo,
head_branch,
],
cwd=config.project_path,
)
if use_subrepo:
cmd = ["git", "subrepo", "push", challenge_path]
else:
head_branch = get_git_repo_head_branch(challenge_repo)
cmd = ["git", "subtree", "push", "--prefix", challenge_path, challenge_repo, head_branch]

if git_subtree_push != 0:
log.debug(f"call({cmd}, cwd='{config.project_path / challenge_path}')")
if subprocess.call(cmd, cwd=config.project_path) != 0:
click.secho(
"Could not push the challenge subtree. " "Please check git error messages above.",
"Could not push the challenge repository. Please check git error messages above.",
fg="red",
)
failed_pushes.append(challenge_instance)
Expand All @@ -335,8 +332,8 @@ def push(self, challenge: str = None, no_auto_pull: bool = False, quiet=False) -

return 1

def pull(self, challenge: str = None, quiet=False) -> int:
log.debug(f"pull: (challenge={challenge})")
def pull(self, challenge: str = None, strategy: str = "fast-forward", quiet: bool = False) -> int:
log.debug(f"pull: (challenge={challenge}, quiet={quiet})")
config = Config()

if challenge:
Expand All @@ -353,6 +350,11 @@ def pull(self, challenge: str = None, quiet=False) -> int:
else:
context = click.progressbar(challenges, label="Pulling challenges")

use_subrepo = config["config"].getboolean("use_subrepo", fallback=False)
if use_subrepo and not check_if_git_subrepo_is_installed():
click.secho("This project is configured to use git subrepo, but it's not installed.")
return 1

failed_pulls = []
with context as context_challenges:
for challenge_instance in context_challenges:
Expand Down Expand Up @@ -386,18 +388,25 @@ def pull(self, challenge: str = None, quiet=False) -> int:
continue

click.secho(f"Pulling latest '{challenge_repo}' to '{challenge_path}'", fg="blue")
head_branch = get_git_repo_head_branch(challenge_repo)

log.debug(
f"call(['git', 'subtree', 'pull', '--prefix', '{challenge_path}', "
f"'{challenge_repo}', '{head_branch}', '--squash'], cwd='{config.project_path}')"
)

pull_env = os.environ.copy()
pull_env["GIT_MERGE_AUTOEDIT"] = "no"

git_subtree_pull = subprocess.call(
[
if use_subrepo:
cmd = ["git", "subrepo", "pull", challenge_path]

if strategy == "rebase":
cmd += ["--rebase"]
elif strategy == "merge":
cmd += ["--merge"]
elif strategy == "force":
cmd += ["--force"]
elif strategy == "fast-forward":
pass # fast-forward is the default strategy
else:
click.secho(f"Cannot pull challenge - '{strategy}' is not a valid pull strategy", fg="red")
else:
head_branch = get_git_repo_head_branch(challenge_repo)
pull_env["GIT_MERGE_AUTOEDIT"] = "no"
cmd = [
"git",
"subtree",
"pull",
Expand All @@ -406,12 +415,10 @@ def pull(self, challenge: str = None, quiet=False) -> int:
challenge_repo,
head_branch,
"--squash",
],
cwd=config.project_path,
env=pull_env,
)
]

if git_subtree_pull != 0:
log.debug(f"call({cmd}, cwd='{config.project_path})")
if subprocess.call(cmd, cwd=config.project_path, env=pull_env) != 0:
click.secho(
f"Could not pull the subtree for challenge '{challenge_path}'. "
"Please check git error messages above.",
Expand All @@ -420,25 +427,26 @@ def pull(self, challenge: str = None, quiet=False) -> int:
failed_pulls.append(challenge_instance)
continue

log.debug(f"call(['git', 'mergetool'], cwd='{config.project_path / challenge_path}')")
git_mergetool = subprocess.call(["git", "mergetool"], cwd=config.project_path / challenge_path)
if not use_subrepo:
log.debug(f"call(['git', 'mergetool'], cwd='{config.project_path / challenge_path}')")
git_mergetool = subprocess.call(["git", "mergetool"], cwd=config.project_path / challenge_path)

log.debug(f"call(['git', 'commit', '--no-edit'], cwd='{config.project_path / challenge_path}')")
subprocess.call(["git", "commit", "--no-edit"], cwd=config.project_path / challenge_path)
log.debug(f"call(['git', 'commit', '--no-edit'], cwd='{config.project_path / challenge_path}')")
subprocess.call(["git", "commit", "--no-edit"], cwd=config.project_path / challenge_path)

log.debug(f"call(['git', 'clean', '-f'], cwd='{config.project_path / challenge_path}')")
git_clean = subprocess.call(["git", "clean", "-f"], cwd=config.project_path / challenge_path)
log.debug(f"call(['git', 'clean', '-f'], cwd='{config.project_path / challenge_path}')")
git_clean = subprocess.call(["git", "clean", "-f"], cwd=config.project_path / challenge_path)

# git commit is allowed to return a non-zero code
# because it would also mean that there's nothing to commit
if any(r != 0 for r in [git_mergetool, git_clean]):
click.secho(
f"Could not commit the subtree for challenge '{challenge_path}'. "
"Please check git error messages above.",
fg="red",
)
failed_pulls.append(challenge_instance)
continue
# git commit is allowed to return a non-zero code
# because it would also mean that there's nothing to commit
if any(r != 0 for r in [git_mergetool, git_clean]):
click.secho(
f"Could not commit the changes for challenge '{challenge_path}'. "
"Please check git error messages above.",
fg="red",
)
failed_pulls.append(challenge_instance)
continue

if len(failed_pulls) == 0:
if not quiet:
Expand All @@ -460,6 +468,11 @@ def restore(self, challenge: str = None) -> int:
click.secho("Could not find any added challenges to restore", fg="yellow")
return 1

use_subrepo = config["config"].getboolean("use_subrepo", fallback=False)
if use_subrepo and not check_if_git_subrepo_is_installed():
click.secho("This project is configured to use git subrepo, but it's not installed.")
return 1

failed_restores = []
for challenge_key, challenge_source in config.challenges.items():
if challenge is not None and challenge_key != challenge:
Expand All @@ -483,6 +496,19 @@ def restore(self, challenge: str = None) -> int:
failed_restores.append(challenge_key)
continue

# If we're using subrepo - the restore can be achieved by performing a force pull
if use_subrepo:
if self.pull(challenge, strategy="force") != 0:
click.secho(
f"Failed to restore challenge '{challenge_key}' via subrepo force pull. "
"Please check git error messages above.",
fg="red",
)
failed_restores.append(challenge_key)

continue

# Otherwise - default to restoring the repository via re-adding the subtree
# Check if target directory exits
if (config.project_path / challenge_key).exists():
click.secho(
Expand Down
7 changes: 7 additions & 0 deletions ctfcli/utils/git.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,13 @@
from typing import Optional, Union


def check_if_git_subrepo_is_installed() -> bool:
output = subprocess.run(["git", "subrepo"], capture_output=True, text=True)
if "git: 'subrepo' is not a git command" in output.stderr:
return False
return True


def get_git_repo_head_branch(repo: str) -> Optional[str]:
"""
A helper method to get the reference of the HEAD branch of a git remote repo.
Expand Down
Loading