Merge branch 'pulp:main' into main

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.4.0.dev
+current_version = 3.5.0.dev
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\.(?P<release>[a-z]+))?
@@ -17,5 +17,3 @@ values =
 [bumpversion:file:./pulp_deb/app/__init__.py]
 
 [bumpversion:file:./setup.py]
-
-[bumpversion:file:./docs/conf.py]

.ci/ansible/Containerfile.j2

@@ -9,7 +9,10 @@ ADD ./{{ item.name }} ./{{ item.name }}
 # S3 botocore needs to be patched to handle responses from minio during 0-byte uploads
 # Hacking botocore (https://github.com/boto/botocore/pull/1990)
 
-RUN pip3 install
+# This MUST be the ONLY call to pip install in inside the container.
+RUN pip3 install --upgrade pip setuptools wheel && \
+  rm -rf /root/.cache/pip && \
+  pip3 install
 {%- if s3_test | default(false) -%}
 {{ " " }}git+https://github.com/gerrod3/botocore.git@fix-100-continue
 {%- endif -%}
@@ -25,7 +28,9 @@ RUN pip3 install
 {{ " " }}-r ./{{ item.name }}/ci_requirements.txt
 {%- endif -%}
 {%- endfor %}
-{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt
+{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt \
+  pipdeptree && \
+  rm -rf /root/.cache/pip
 
 {% if pulp_env is defined and pulp_env %}
 {% for key, value in pulp_env.items() %}

.ci/assets/ci_constraints.txt

@@ -5,3 +5,8 @@ pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*
 
 tablib!=3.6.0
 # 3.6.0: This release introduced a regression removing the "html" optional dependency.
+
+
+
+# Newer version seem to have a conflict around packaging, that pip fails to resolve in time. Remove this when this starts to impose an issue.
+pipdeptree<=3.23.1

.ci/scripts/calc_constraints.py

@@ -62,6 +62,8 @@ def to_upper_bound(req):
                     max_version = f"{version.major}.{version.minor}.{version.micro-1}"
                 elif version.minor != 0:
                     max_version = f"{version.major}.{version.minor-1}"
+                elif version.major != 0:
+                    max_version = f"{version.major-1}.0"
                 else:
                     return f"# NO BETTER CONSTRAINT: {req}"
                 return f"{requirement.name}{operator}{max_version}"

.ci/scripts/check_release.py

@@ -64,6 +64,7 @@ def main():
         for branch in branches:
             if branch != DEFAULT_BRANCH:
                 # Check if a Z release is needed
+                reasons = []
                 changes = repo.git.ls_tree("-r", "--name-only", f"origin/{branch}", "CHANGES/")
                 z_changelog = False
                 for change in changes.split("\n"):
@@ -76,23 +77,27 @@ def main():
                         )
                     elif ext in Z_CHANGELOG_EXTS:
                         z_changelog = True
+                if z_changelog:
+                    reasons.append("Backports")
 
                 last_tag = repo.git.describe("--tags", "--abbrev=0", f"origin/{branch}")
                 req_txt_diff = repo.git.diff(
                     f"{last_tag}", f"origin/{branch}", "--name-only", "--", "requirements.txt"
                 )
-                if z_changelog or req_txt_diff:
+                if req_txt_diff:
+                    reasons.append("requirements.txt")
+
+                if reasons:
                     curr_version = Version(last_tag)
                     assert curr_version.base_version.startswith(
                         branch
                     ), "Current-version has to belong to the current branch!"
                     next_version = Version(f"{branch}.{curr_version.micro + 1}")
-                    reason = "CHANGES" if z_changelog else "requirements.txt"
                     print(
                         f"A Z-release is needed for {branch}, "
                         f"Prev: {last_tag}, "
                         f"Next: {next_version.base_version}, "
-                        f"Reason: {reason}"
+                        f"Reason: {','.join(reasons)}"
                     )
                     releases.append(next_version)
             else:

.ci/scripts/collect_changes.py

@@ -103,7 +103,7 @@ def main():
             for change in main_changes:
                 fp.write(change[1])
 
-        repo.git.commit("-m", "Update Changelog", "-m" "[noissue]", CHANGELOG_FILE)
+        repo.git.commit("-m", "Update Changelog", CHANGELOG_FILE)
 
 
 if __name__ == "__main__":

.ci/scripts/pr_labels.py

@@ -0,0 +1,60 @@
+#!/bin/env python3
+
+# This script is running with elevated privileges from the main branch against pull requests.
+
+import re
+import sys
+import tomllib
+from pathlib import Path
+
+from git import Repo
+
+
+def main():
+    assert len(sys.argv) == 3
+
+    with open("pyproject.toml", "rb") as fp:
+        PYPROJECT_TOML = tomllib.load(fp)
+    BLOCKING_REGEX = re.compile(r"DRAFT|WIP|NO\s*MERGE|DO\s*NOT\s*MERGE|EXPERIMENT")
+    ISSUE_REGEX = re.compile(r"(?:fixes|closes)[\s:]+#(\d+)")
+    CHERRY_PICK_REGEX = re.compile(r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$")
+    try:
+        CHANGELOG_EXTS = {
+            f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"]
+        }
+    except KeyError:
+        CHANGELOG_EXTS = {".feature", ".bugfix", ".doc", ".removal", ".misc"}
+
+    repo = Repo(".")
+
+    base_commit = repo.commit(sys.argv[1])
+    head_commit = repo.commit(sys.argv[2])
+
+    pr_commits = list(repo.iter_commits(f"{base_commit}..{head_commit}"))
+
+    labels = {
+        "multi-commit": len(pr_commits) > 1,
+        "cherry-pick": False,
+        "no-issue": False,
+        "no-changelog": False,
+        "wip": False,
+    }
+    for commit in pr_commits:
+        labels["wip"] |= BLOCKING_REGEX.search(commit.summary) is not None
+        no_issue = ISSUE_REGEX.search(commit.message, re.IGNORECASE) is None
+        labels["no-issue"] |= no_issue
+        cherry_pick = CHERRY_PICK_REGEX.search(commit.message) is not None
+        labels["cherry-pick"] |= cherry_pick
+        changelog_snippets = [
+            k
+            for k in commit.stats.files
+            if k.startswith("CHANGES/") and Path(k).suffix in CHANGELOG_EXTS
+        ]
+        labels["no-changelog"] |= not changelog_snippets
+
+    print("ADD_LABELS=" + ",".join((k for k, v in labels.items() if v)))
+    print("REMOVE_LABELS=" + ",".join((k for k, v in labels.items() if not v)))
+
+
+if __name__ == "__main__":
+    main()

.ci/scripts/validate_commit_message.py

@@ -9,21 +9,16 @@
 import sys
 from pathlib import Path
 import subprocess
-
-
 import os
 import warnings
 from github import Github
 
-
-NO_ISSUE = "[noissue]"
 CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc", ".deprecation"]
+KEYWORDS = ["fixes", "closes"]
+
 sha = sys.argv[1]
 message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8")
 
-
-KEYWORDS = ["fixes", "closes"]
-
 g = Github(os.environ.get("GITHUB_TOKEN"))
 repo = g.get_repo("pulp/pulp_deb")
 
@@ -64,15 +59,5 @@ def __check_changelog(issue):
     for issue in pattern.findall(message):
         __check_status(issue)
         __check_changelog(issue)
-else:
-    if NO_ISSUE in message:
-        print("Commit {sha} has no issues but is tagged {tag}.".format(sha=sha[0:7], tag=NO_ISSUE))
-    elif "Merge" in message and "cherry picked from commit" in message:
-        pass
-    else:
-        sys.exit(
-            "Error: no attached issues found for {sha}. If this was intentional, add "
-            " '{tag}' to the commit message.".format(sha=sha[0:7], tag=NO_ISSUE)
-        )
 
 print("Commit message for {sha} passed.".format(sha=sha[0:7]))

.github/template_gitref

@@ -1 +1 @@
-2021.08.26-354-g82d22de
+2021.08.26-387-ge627e91

.github/workflows/ci.yml

@@ -46,6 +46,9 @@ jobs:
         run: |
           python .ci/scripts/check_requirements.py
 
+  docs:
+    uses: "./.github/workflows/docs.yml"
+
   lint:
     uses: "./.github/workflows/lint.yml"
 
@@ -84,6 +87,7 @@ jobs:
       - "check-commits"
       - "lint"
       - "test"
+      - "docs"
     if: "always()"
     steps:
       - name: "Collect needed jobs results"

.github/workflows/create-branch.yml

@@ -26,14 +26,20 @@ jobs:
           fetch-depth: 0
           path: "pulp_deb"
 
+      - uses: "actions/checkout@v4"
+        with:
+          fetch-depth: 1
+          repository: "pulp/plugin_template"
+          path: "plugin_template"
+
       - uses: "actions/setup-python@v5"
         with:
           python-version: "3.11"
 
       - name: "Install python dependencies"
         run: |
           echo ::group::PYDEPS
-          pip install bump2version jinja2 pyyaml packaging
+          pip install bump2version packaging -r plugin_template/requirements.txt
           echo ::endgroup::
 
       - name: "Setting secrets"
@@ -71,13 +77,6 @@ jobs:
         run: |
           find CHANGES -type f -regex ".*\.\(bugfix\|doc\|feature\|misc\|deprecation\|removal\)" -exec git rm {} +
 
-      - name: Checkout plugin template
-        uses: actions/checkout@v4
-        with:
-          repository: pulp/plugin_template
-          path: plugin_template
-          fetch-depth: 0
-
       - name: Update CI branches in template_config
         working-directory: plugin_template
         run: |
@@ -94,10 +93,8 @@ jobs:
           branch: minor-version-bump
           base: main
           title: Bump minor version
-          body: '[noissue]'
           commit-message: |
             Bump minor version
-            [noissue]
           delete-branch: true
 
       - name: Push release branch

.github/workflows/docs.yml

@@ -0,0 +1,54 @@
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --github pulp_deb' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
+
+---
+name: "Docs"
+on:
+  workflow_call:
+
+jobs:
+  test:
+    if: "endsWith(github.base_ref, 'main')"
+    runs-on: "ubuntu-20.04"
+    defaults:
+      run:
+        working-directory: "pulp_deb"
+    steps:
+      - uses: "actions/checkout@v4"
+        with:
+          fetch-depth: 1
+          path: "pulp_deb"
+      - uses: "actions/setup-python@v5"
+        with:
+          python-version: "3.11"
+      - name: "Setup cache key"
+        run: |
+          git ls-remote https://github.com/pulp/pulp-docs main | tee pulp-docs-main-sha
+      - uses: "actions/cache@v4"
+        with:
+          path: "~/.cache/pip"
+          key: ${{ runner.os }}-pip-${{ hashFiles('pulp-docs-main-sha') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: "Install python dependencies"
+        run: |
+          echo ::group::PYDEPS
+          pip install -r doc_requirements.txt
+          echo ::endgroup::
+      - name: "Build changelog"
+        run: |
+          towncrier build --yes --version 4.0.0.ci
+      - name: "Build docs"
+        run: |
+          pulp-docs build
+
+  no-test:
+    if: "!endsWith(github.base_ref, 'main')"
+    runs-on: "ubuntu-20.04"
+    steps:
+      - run: |
+          echo "Skip docs testing on non-main branches."

.github/workflows/nightly.yml

@@ -98,6 +98,26 @@ jobs:
           name: "plugin_package"
           path: "pulp_deb/dist/"
 
+      - name: "Download API specs"
+        uses: "actions/download-artifact@v4"
+        with:
+          name: "api_spec"
+          path: "pulp_deb/"
+
+      - name: "Download client packages"
+        uses: "actions/download-artifact@v4"
+        with:
+          name: "python-client.tar"
+          path: "pulp_deb"
+
+      - name: "Unpack client packages"
+        working-directory: "pulp-openapi-generator"
+        run: |
+          mkdir -p "pulp_deb-client"
+          pushd "pulp_deb-client"
+          tar xvf "../../pulp_deb/deb-python-client.tar"
+          popd
+
       - uses: "actions/setup-python@v5"
         with:
           python-version: "3.11"