Update CI Files

[noissue]

.bumpversion.cfg

@@ -14,4 +14,8 @@ values =
 	dev
 	prod
 
+[bumpversion:file:./pulpcore/cli/ostree/__init__.py]
+
+[bumpversion:file:./pulp-glue-ostree/setup.py]
+
 [bumpversion:file:./setup.py]

.ci/run_container.sh

@@ -4,6 +4,8 @@
 # Known places:
 #   - https://github.com/pulp/pulp-cli/blob/main/.ci/run_container.sh
 #   - https://github.com/pulp/pulp-cli-deb/blob/main/.ci/run_container.sh
+#   - https://github.com/pulp/pulp-cli-gem/blob/main/.ci/run_container.sh
+#   - https://github.com/pulp/pulp-cli-maven/blob/main/.ci/run_container.sh
 #   - https://github.com/pulp/pulp-cli-ostree/blob/main/.ci/run_container.sh
 #   - https://github.com/pulp/squeezer/blob/develop/tests/run_container.sh
 

.ci/scripts/collect_changes.py

@@ -0,0 +1,94 @@
+import itertools
+import os
+import re
+
+import toml
+from git import GitCommandError, Repo
+from pkg_resources import parse_version
+
+# Read Towncrier settings
+tc_settings = toml.load("pyproject.toml")["tool"]["towncrier"]
+
+CHANGELOG_FILE = tc_settings.get("filename", "NEWS.rst")
+START_STRING = tc_settings.get(
+    "start_string",
+    "<!-- towncrier release notes start -->\n"
+    if CHANGELOG_FILE.endswith(".md")
+    else ".. towncrier release notes start\n",
+)
+TITLE_FORMAT = tc_settings.get("title_format", "{name} {version} ({project_date})")
+
+
+NAME_REGEX = r".*"
+VERSION_REGEX = r"([0-9]+\.[0-9]+\.[0-9][0-9ab]*)"
+DATE_REGEX = r"[0-9]{4}-[0-9]{2}-[0-9]{2}"
+TITLE_REGEX = (
+    "("
+    + re.escape(
+        TITLE_FORMAT.format(name="NAME_REGEX", version="VERSION_REGEX", project_date="DATE_REGEX")
+    )
+    .replace("NAME_REGEX", NAME_REGEX)
+    .replace("VERSION_REGEX", VERSION_REGEX)
+    .replace("DATE_REGEX", DATE_REGEX)
+    + ")"
+)
+
+
+def get_changelog(repo, branch):
+    return repo.git.show(f"{branch}:{CHANGELOG_FILE}") + "\n"
+
+
+def _tokenize_changes(splits):
+    assert len(splits) % 3 == 0
+    for i in range(len(splits) // 3):
+        title = splits[3 * i]
+        version = parse_version(splits[3 * i + 1])
+        yield [version, title + splits[3 * i + 2]]
+
+
+def split_changelog(changelog):
+    preamble, rest = changelog.split(START_STRING, maxsplit=1)
+    split_rest = re.split(TITLE_REGEX, rest)
+    return preamble + START_STRING + split_rest[0], list(_tokenize_changes(split_rest[1:]))
+
+
+def main():
+    repo = Repo(os.getcwd())
+    remote = repo.remotes[0]
+    branches = [ref for ref in remote.refs if re.match(r"^([0-9]+)\.([0-9]+)$", ref.remote_head)]
+    branches.sort(key=lambda ref: parse_version(ref.remote_head), reverse=True)
+    branches = [ref.name for ref in branches]
+
+    with open(CHANGELOG_FILE, "r") as f:
+        main_changelog = f.read()
+    preamble, main_changes = split_changelog(main_changelog)
+    old_length = len(main_changes)
+
+    for branch in branches:
+        print(f"Looking at branch {branch}")
+        try:
+            changelog = get_changelog(repo, branch)
+        except GitCommandError:
+            print("No changelog found on this branch.")
+            continue
+        dummy, changes = split_changelog(changelog)
+        new_changes = sorted(main_changes + changes, key=lambda x: x[0], reverse=True)
+        # Now remove duplicates (retain the first one)
+        main_changes = [new_changes[0]]
+        for left, right in itertools.pairwise(new_changes):
+            if left[0] != right[0]:
+                main_changes.append(right)
+
+    new_length = len(main_changes)
+    if old_length < new_length:
+        print(f"{new_length - old_length} new versions have been added.")
+        with open(CHANGELOG_FILE, "w") as fp:
+            fp.write(preamble)
+            for change in main_changes:
+                fp.write(change[1])
+
+        repo.git.commit("-m", "Update Changelog", "-m" "[noissue]", CHANGELOG_FILE)
+
+
+if __name__ == "__main__":
+    main()

.ci/scripts/create_release_branch.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+BRANCH="$(git branch --show-current)"
+
+if ! [[ "${BRANCH}" = "main" ]]
+then
+  echo ERROR: This is not the main branch!
+  exit 1
+fi
+
+NEW_BRANCH="$(bump2version --dry-run --list release | sed -Ene 's/^new_version=([[:digit:]]+\.[[:digit:]]+)\..*$/\1/p')"
+
+if [[ -z "${NEW_BRANCH}" ]]
+then
+  echo ERROR: Could not parse new version.
+  exit 1
+fi
+
+git branch "${NEW_BRANCH}"
+
+# Clean changelog snippets.
+find CHANGES/ \( -name "*.feature" -o -name "*.bugfix" -o -name "*.doc" -o -name "*.translation" -o -name "*.devel" -o -name "*.misc" \) -exec git rm -f \{\} +
+
+bumpversion minor --commit --message $'Bump version to {new_version}\n\n[noissue]' --allow-dirty
+
+git push origin "${NEW_BRANCH}"

.ci/scripts/release.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+BRANCH=$(git branch --show-current)
+
+if ! [[ "${BRANCH}" =~ ^[0-9]+\.[0-9]+$ ]]
+then
+  echo ERROR: This is not a release branch!
+  exit 1
+fi
+
+NEW_VERSION="$(bump2version --dry-run --list release | sed -ne 's/^new_version=//p')"
+echo "Release ${NEW_VERSION}"
+
+if ! [[ "${NEW_VERSION}" == "${BRANCH}"* ]]
+then
+  echo ERROR: Version does not match release branch
+  exit 1
+fi
+
+towncrier build --yes --version "${NEW_VERSION}"
+bumpversion release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty
+bumpversion patch --commit
+
+git push origin "${BRANCH}" "${NEW_VERSION}"

.ci/scripts/validate_commit_message.py

@@ -21,7 +21,6 @@
 ]
 
 sha = sys.argv[1]
-project = "pulp-cli-ostree"
 message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8")
 
 if any((re.match(pattern, message) for pattern in BLOCKING_REGEX)):
@@ -52,13 +51,16 @@ def check_changelog(issue):
 print("Checking commit message for {sha}.".format(sha=sha[0:7]))
 
 # validate the issue attached to the commit
-regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords=("|").join(KEYWORDS))
-issues = re.findall(regex, message, re.IGNORECASE)
+issue_regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords=("|").join(KEYWORDS))
+issues = re.findall(issue_regex, message, re.IGNORECASE)
+cherry_pick_regex = r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$"
+cherry_pick = re.search(cherry_pick_regex, message, re.MULTILINE)
 
 if issues:
     for issue in issues:
-        check_status(issue)
-        check_changelog(issue)
+        if not cherry_pick:
+            check_status(issue)
+            check_changelog(issue)
 else:
     if NO_ISSUE in message:
         print("Commit {sha} has no issues but is tagged {tag}.".format(sha=sha[0:7], tag=NO_ISSUE))

.ci/settings/settings.py

@@ -1,4 +1,3 @@
 CONTENT_ORIGIN = "http://localhost:8080/"
 ALLOWED_EXPORT_PATHS = ["/tmp"]
-ORPHAN_PROTECTION_TIME = 0
 TELEMETRY = False

.github/workflows/codeql.yml

@@ -0,0 +1,44 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches:
+      - main
+  workflow_call:
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/*constraints.lock', '**/setup.py', '**/pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+      - name: Manually install from sources
+        run: |
+          python -m pip install -e . -e ./pulp-glue-ostree
+          echo "CODEQL_PYTHON=$(which python)" >> $GITHUB_ENV
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: python
+          setup-python-dependencies: false
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2
+        with:
+          category: "/language:python"

.github/workflows/collect_changes.yml

@@ -0,0 +1,32 @@
+name: Collect changes
+on:
+  workflow_call:
+  workflow_dispatch:
+
+jobs:
+  collect-changes:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: "main"
+          fetch-depth: 0
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+      - name: Setup git
+        run: |
+          git config user.name pulpbot
+          git config user.email [email protected]
+      - name: Collect changes
+        run: |
+          pip install GitPython toml
+          python3 .ci/scripts/collect_changes.py
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v5
+        with:
+          token: ${{ secrets.RELEASE_TOKEN }}
+          title: "Update Changelog"
+          body: ""
+          branch: "update_changes"
+          delete-branch: true

.github/workflows/lint.yml

@@ -0,0 +1,30 @@
+name: Lint
+
+on:
+  workflow_call:
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python:
+          - "3.8"
+          - "3.11"
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/*constraints.lock', '**/setup.py', '**/pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install python dependencies
+        run: pip install -r lint_requirements.txt
+      - name: Lint code
+        run: make lint