[Backport 2023.1] feature(auto-backport.py): replace Mergify

.github/scripts/auto-backport.py

@@ -0,0 +1,189 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import re
+import sys
+import tempfile
+import logging
+
+from github import Github, GithubException
+from git import Repo, GitCommandError
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+try:
+    github_token = os.environ["GITHUB_TOKEN"]
+except KeyError:
+    print("Please set the 'GITHUB_TOKEN' environment variable")
+    sys.exit(1)
+
+g = Github(github_token)
+promoted_label = 'promoted-to-master'
+backport_label_pattern = re.compile(r'backport/(perf-*.\d+$|\d+\.\d+$)')
+
+
+def is_pull_request():
+    return '--pull-request' in sys.argv[1:]
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--repo', type=str, required=True, help='Github repository name')
+    parser.add_argument('--base-branch', type=str, default='refs/heads/next', help='Base branch')
+    parser.add_argument('--commits', default=None, type=str, help='Range of promoted commits.')
+    parser.add_argument('--pull-request', type=int, help='Pull request number to be backported')
+    parser.add_argument('--head-commit', type=str, required=is_pull_request(),
+                        help='The HEAD of target branch after the pull request specified by --pull-request is merged')
+    parser.add_argument('--label', type=str, required=is_pull_request(),
+                        help='Backport label name when --pull-request is defined')
+    return parser.parse_args()
+
+
+def create_pull_request(repo, new_branch_name, base_branch_name, pr, backport_pr_title, commits, is_draft=False):
+    pr_body = f'{pr.body}\n\n'
+    for commit in commits:
+        pr_body += f'- (cherry picked from commit {commit})\n\n'
+    pr_body += f'Parent PR: #{pr.number}'
+    try:
+        backport_pr = repo.create_pull(
+            title=backport_pr_title,
+            body=pr_body,
+            head=f'scylladbbot:{new_branch_name}',
+            base=base_branch_name,
+            draft=is_draft
+        )
+        logging.info(f"Pull request created: {backport_pr.html_url}")
+        backport_pr.add_to_assignees(pr.user)
+        if is_draft:
+            backport_pr.add_to_labels("conflicts")
+            pr_comment = f"@{pr.user.login} - This PR has conflicts, therefore it was moved to `draft` \n"
+            pr_comment += "Please resolve them and mark this PR as ready for review"
+            backport_pr.create_issue_comment(pr_comment)
+        logging.info(f"Assigned PR to original author: {pr.user}")
+        return backport_pr
+    except GithubException as e:
+        if 'A pull request already exists' in str(e):
+            logging.warning(f'A pull request already exists for {pr.user}:{new_branch_name}')
+        else:
+            logging.error(f'Failed to create PR: {e}')
+
+
+def get_pr_commits(repo, pr, stable_branch, start_commit=None):
+    commits = []
+    if pr.merged:
+        merge_commit = repo.get_commit(pr.merge_commit_sha)
+        if len(merge_commit.parents) > 1:  # Check if this merge commit includes multiple commits
+            commits.append(pr.merge_commit_sha)
+        else:
+            if start_commit:
+                promoted_commits = repo.compare(start_commit, stable_branch).commits
+            else:
+                promoted_commits = repo.get_commits(sha=stable_branch)
+            for commit in pr.get_commits():
+                for promoted_commit in promoted_commits:
+                    commit_title = commit.commit.message.splitlines()[0]
+                    # In Scylla-pkg and scylla-dtest, for example,
+                    # we don't create a merge commit for a PR with multiple commits,
+                    # according to the GitHub API, the last commit will be the merge commit,
+                    # which is not what we need when backporting (we need all the commits).
+                    # So here, we are validating the correct SHA for each commit so we can cherry-pick
+                    if promoted_commit.commit.message.startswith(commit_title):
+                        commits.append(promoted_commit.sha)
+
+    elif pr.state == 'closed':
+        events = pr.get_issue_events()
+        for event in events:
+            if event.event == 'closed':
+                commits.append(event.commit_id)
+    return commits
+
+
+def create_pr_comment_and_remove_label(pr, comment_body):
+    labels = pr.get_labels()
+    pattern = re.compile(r"backport/(perf-*.\d+$|\d+\.\d+$)")
+    for label in labels:
+        if pattern.match(label.name):
+            print(f"Removing label: {label.name}")
+            comment_body += f'- {label.name}\n'
+            pr.remove_from_labels(label)
+    pr.create_issue_comment(comment_body)
+
+
+def backport(repo, pr, version, commits, backport_base_branch):
+    new_branch_name = f'backport/{pr.number}/to-{version}'
+    backport_pr_title = f'[Backport {version}] {pr.title}'
+    repo_url = f'https://scylladbbot:{github_token}@github.com/{repo.full_name}.git'
+    fork_repo = f'https://scylladbbot:{github_token}@github.com/scylladbbot/{repo.name}.git'
+    with (tempfile.TemporaryDirectory() as local_repo_path):
+        try:
+            repo_local = Repo.clone_from(repo_url, local_repo_path, branch=backport_base_branch)
+            repo_local.git.checkout(b=new_branch_name)
+            is_draft = False
+            for commit in commits:
+                try:
+                    repo_local.git.cherry_pick(commit, '-m1', '-x')
+                except GitCommandError as e:
+                    logging.warning(f'Cherry-pick conflict on commit {commit}: {e}')
+                    is_draft = True
+                    repo_local.git.add(A=True)
+                    repo_local.git.cherry_pick('--continue')
+            repo_local.git.push(fork_repo, new_branch_name, force=True)
+            create_pull_request(repo, new_branch_name, backport_base_branch, pr, backport_pr_title, commits,
+                                is_draft=is_draft)
+        except GitCommandError as e:
+            logging.warning(f"GitCommandError: {e}")
+
+
+def main():
+    args = parse_args()
+    base_branch = args.base_branch.split('/')[2]
+    repo_name = args.repo
+
+    stable_branch = 'master' if base_branch == 'master' else base_branch.replace('master', 'branch')
+
+    repo = g.get_repo(repo_name)
+    scylladbbot_repo = g.get_repo('scylladbbot/scylla-cluster-tests')
+    closed_prs = []
+    start_commit = None
+
+    if args.commits:
+        start_commit, end_commit = args.commits.split('..')
+        commits = repo.compare(start_commit, end_commit).commits
+        for commit in commits:
+            for pr in commit.get_pulls():
+                closed_prs.append(pr)
+    if args.pull_request:
+        start_commit = args.head_commit
+        pr = repo.get_pull(args.pull_request)
+        closed_prs = [pr]
+
+    for pr in closed_prs:
+        if args.pull_request:
+            backport_labels = [args.label]
+        else:
+            labels = [label.name for label in pr.labels]
+            backport_labels = [label for label in labels if backport_label_pattern.match(label)]
+            if promoted_label not in labels:
+                print(f'no {promoted_label} label: {pr.number}')
+                continue
+            if not backport_labels:
+                print(f'no backport label: {pr.number}')
+                continue
+            if not not scylladbbot_repo.has_in_collaborators(pr.user.login):
+                logging.info(
+                    f"Sending an invite to {pr.user.login} to become a collaborator to {scylladbbot_repo.full_name} ")
+                scylladbbot_repo.add_to_collaborators(pr.user.login)
+                comment = f':warning:  @{pr.user.login} you have been added as collaborator to scylladbbot fork '
+                comment += 'Please check your inbox and approve the invitation, once it is done, please add the backport labels again\n'
+                create_pr_comment_and_remove_label(pr, comment)
+                continue
+        commits = get_pr_commits(repo, pr, stable_branch, start_commit)
+        logging.info(f"Found PR #{pr.number} with commit {commits} and the following labels: {backport_labels}")
+        for backport_label in backport_labels:
+            version = backport_label.replace('backport/', '')
+            backport_base_branch = backport_label.replace('backport/', 'master')
+            backport(repo, pr, version, commits, backport_base_branch)
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/search_commits.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+import argparse
+import re
+import requests
+
+from github import Github
+
+try:
+    github_token = os.environ["GITHUB_TOKEN"]
+except KeyError:
+    print("Please set the 'GITHUB_TOKEN' environment variable")
+    sys.exit(1)
+
+
+def get_parser():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--repository', type=str, default='scylladb/scylla-pkg', help='Github repository name')
+    parser.add_argument('--commits', type=str, required=True, help='Range of promoted commits.')
+    parser.add_argument('--label', type=str, default='promoted-to-master', help='Label to use')
+    parser.add_argument('--ref', type=str, required=True, help='PR target branch')
+    return parser.parse_args()
+
+
+def main():  # pylint: disable=too-many-locals  # noqa: PLR0914
+    args = get_parser()
+    github = Github(github_token)
+    repo = github.get_repo(args.repository, lazy=False)
+    start_commit, end_commit = args.commits.split('..')
+    commits = repo.compare(start_commit, end_commit).commits
+
+    processed_prs = set()
+    for commit in commits:
+        search_url = 'https://api.github.com/search/issues'
+        query = f"repo:{args.repository} is:pr is:merged sha:{commit.sha}"
+        params = {
+            "q": query,
+        }
+        headers = {
+            "Authorization": f"token {github_token}",
+            "Accept": "application/vnd.github.v3+json"
+        }
+        response = requests.get(search_url, headers=headers, params=params)
+        prs = response.json().get("items", [])
+        for pr in prs:  # pylint: disable=invalid-name
+            match = re.findall(r'Parent PR: #(\d+)', pr["body"])
+            if match:
+                pr_number = int(match[0])
+                if pr_number in processed_prs:
+                    continue
+                ref = re.search(r'-(\d+\.\d+|perf-v(\d+))', args.ref)
+                label_to_add = f'backport/{ref.group(1)}-done'
+                label_to_remove = f'backport/{ref.group(1)}'
+                remove_label_url = f'https://api.github.com/repos/{args.repository}/issues/{pr_number}/labels/{label_to_remove}'
+                del_data = {
+                    "labels": [f'{label_to_remove}']
+                }
+                response = requests.delete(remove_label_url, headers=headers, json=del_data)
+                if response.ok:
+                    print(f'Label {label_to_remove} removed successfully')
+                else:
+                    print(f'Label {label_to_remove} cant be removed')
+            else:
+                pr_number = pr["number"]
+                label_to_add = args.label
+            data = {
+                "labels": [f'{label_to_add}']
+            }
+            add_label_url = f'https://api.github.com/repos/{args.repository}/issues/{pr_number}/labels'
+            response = requests.post(add_label_url, headers=headers, json=data)
+            if response.ok:
+                print(f"Label added successfully to {add_label_url}")
+            else:
+                print(f"No label was added to {add_label_url}")
+            processed_prs.add(pr_number)
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/add-label-when-promoted.yaml

@@ -0,0 +1,67 @@
+name: Check if commits are promoted
+
+on:
+  push:
+    branches:
+      - master
+      - branch-*.*
+      - branch-perf-v*
+      - manager-*.*
+    pull_request_target:
+      types: [labeled]
+      branches: [master]
+
+env:
+  DEFAULT_BRANCH: 'master'
+
+jobs:
+  check-commit:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+      issues: write
+    steps:
+      - name: Dump GitHub context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ github.repository }}
+          ref: ${{ env.DEFAULT_BRANCH }}
+          token: ${{ secrets.AUTO_BACKPORT_TOKEN }}
+          fetch-depth: 0  # Fetch all history for all tags and branches
+      - name: Set up Git identity
+        run: |
+          git config --global user.name "GitHub Action"
+          git config --global user.email "[email protected]"
+          git config --global merge.conflictstyle diff3
+      - name: Install dependencies
+        run: sudo apt-get install -y python3-github python3-git
+      - name: Run python script
+        if: github.event_name == 'push'
+        env:
+          GITHUB_TOKEN: ${{ secrets.AUTO_BACKPORT_TOKEN }}
+        run: python .github/scripts/search_commits.py  --commits ${{ github.event.before }}..${{ github.sha }} --repository ${{ github.repository }} --ref ${{ github.ref }}
+      - name: Run auto-backport.py when promotion completed
+        if: github.event_name == 'push' && github.ref == format('refs/heads/{0}', env.DEFAULT_BRANCH)
+        env:
+          GITHUB_TOKEN: ${{ secrets.AUTO_BACKPORT_TOKEN }}
+        run: python .github/scripts/auto-backport.py --repo ${{ github.repository }} --base-branch ${{ github.ref }} --commits ${{ github.event.before }}..${{ github.sha }}
+      - name: Check if label starts with 'backport/' and contains digits
+        id: check_label
+        run: |
+          label_name="${{ github.event.label.name }}"
+          if [[ "$label_name" =~ ^backport/[0-9]+\.[0-9]+$ ]]; then
+            echo "Label matches backport/X.X pattern."
+            echo "backport_label=true" >> $GITHUB_OUTPUT
+          else
+            echo "Label does not match the required pattern."
+            echo "backport_label=false" >> $GITHUB_OUTPUT
+          fi
+      - name: Run auto-backport.py when label was added
+        if: github.event_name == 'pull_request_target' && steps.check_label.outputs.backport_label == 'true' && github.event.pull_request.state == 'closed'
+        env:
+          GITHUB_TOKEN: ${{ secrets.AUTO_BACKPORT_TOKEN }}
+        run: python .github/scripts/auto-backport.py --repo ${{ github.repository }} --base-branch ${{ github.ref }} --pull-request ${{ github.event.pull_request.number }} --head-commit ${{ github.event.pull_request.base.sha }} --label ${{ github.event.label.name }}

.pre-commit-config.yaml

@@ -48,7 +48,7 @@ repos:
         name: pylint
         entry: pylint -j 1 -d consider-using-f-string
         language: system
-        exclude: '(^docker/alternator-dns/.*$|^argus/.*$)'
+        exclude: '(^docker/alternator-dns/.*$|^argus/.*$|^\.github/.*$)'
         types: [python]
 
 - repo: https://github.com/alessandrojcm/commitlint-pre-commit-hook