From 0dfee2d2f043ab2436e75dcc479a7788585e9da1 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Mon, 14 Oct 2024 16:38:58 +0200 Subject: [PATCH 1/4] Add auto CHANGELOG --- .github/workflows/changelog.py | 239 ++++++++++++++++++++++++++++++++ .github/workflows/changelog.yml | 90 ++++++++++++ 2 files changed, 329 insertions(+) create mode 100755 .github/workflows/changelog.py create mode 100644 .github/workflows/changelog.yml diff --git a/.github/workflows/changelog.py b/.github/workflows/changelog.py new file mode 100755 index 0000000000..e6d8cb3fb4 --- /dev/null +++ b/.github/workflows/changelog.py @@ -0,0 +1,239 @@ +#! /usr/bin/env python3 +""" +Taken from https://github.com/MultiQC/MultiQC/blob/main/.github/workflows/changelog.py and updated for nf-core + +To be called by a CI action. Assumes the following environment variables are set: +PR_TITLE, PR_NUMBER, GITHUB_WORKSPACE. + +Adds a line into the CHANGELOG.md: +* Looks for the section to add the line to, based on the PR title, e.g. `Added:`, `Changed:`. +* All other change will go under the "## dev" section. +* If an entry for the PR is already added, it will not run. + +Other assumptions: +- CHANGELOG.md has a running section for an ongoing "dev" version +(i.e. titled "## dev"). +""" + +import os +import re +import sys +from pathlib import Path +from typing import List, Tuple + +REPO_URL = "https://github.com/nf-core/tools" + +# Assumes the environment is set by the GitHub action. +pr_title = os.environ["PR_TITLE"] +pr_number = os.environ["PR_NUMBER"] +comment = os.environ.get("COMMENT", "") +workspace_path = Path(os.environ.get("GITHUB_WORKSPACE", "")) + +assert pr_title, pr_title +assert pr_number, pr_number + +# Trim the PR number added when GitHub squashes commits, e.g. "Added: Updated (#2026)" +pr_title = pr_title.removesuffix(f" (#{pr_number})") # type: ignore + +changelog_path = workspace_path / "CHANGELOG.md" + +if any( + line in pr_title.lower() + for line in [ + "skip changelog", + "skip change log", + "no changelog", + "no change log", + "bump version", + ] +): + print("Skipping changelog update") + sys.exit(0) + + +def _determine_change_type(pr_title) -> Tuple[str, str]: + """ + Determine the type of the PR: Added, Changed, Fixed, or Removed + Returns a tuple of the section name and the module info. + """ + sections = { + "Added": "### Added", + "Changed": "### Changed", + "Fixed": "### Fixed", + "Removed": "### Removed", + } + current_section_header = "## dev" + current_section = "dev" + + # Check if the PR in any of the sections. + for section, section_header in sections.items(): + # check if the PR title contains any of the section headers, with some loose matching, e.g. removing plural and suffixes + if re.sub(r"s$", "", section.lower().replace("ing", "")) in pr_title.lower(): + current_section_header = section_header + current_section = section + print(f"Detected section: {current_section}") + return current_section, current_section_header + + +# Determine the type of the PR +section, section_header = _determine_change_type(pr_title) + +# Remove section indicator from the PR title. +pr_title = re.sub(rf"{section}:[\s]*", "", pr_title, flags=re.IGNORECASE) + +# Prepare the change log entry. +pr_link = f"([#{pr_number}]({REPO_URL}/pull/{pr_number}))" + +# Handle manual changelog entries through comments. +if comment := comment.removeprefix("@nf-core-bot changelog").strip(): # type: ignore + print(f"Adding manual changelog entry: {comment}") + pr_title = comment +new_lines = [ + f"- {pr_link} - {pr_title}\n", +] +print(f"Adding new lines into section '{section}':\n" + "".join(new_lines)) + +# Finally, updating the changelog. +# Read the current changelog lines. We will print them back as is, except for one new +# entry, corresponding to this new PR. +with changelog_path.open("r") as f: + orig_lines = f.readlines() +updated_lines: List[str] = [] + + +def _skip_existing_entry_for_this_pr(line: str, same_section: bool = True) -> str: + if line.strip().endswith(pr_link): + print(f"Found existing entry for this pull request #{pr_number}:") + existing_lines = [line] + if new_lines and new_lines == existing_lines and same_section: + print( + f"Found existing identical entry for this pull request #{pr_number} in the same section:" + ) + print("".join(existing_lines)) + sys.exit(0) # Just leaving the CHANGELOG intact + else: + print( + f"Found existing entry for this pull request #{pr_number}. It will be replaced and/or moved to proper section" + ) + print("".join(existing_lines)) + for _ in range(len(existing_lines)): + try: + line = orig_lines.pop(0) + except IndexError: + break + return line + + +# Find the next line in the change log that matches the pattern "## dev" +# If it doesn't exist, exist with code 1 (let's assume that a new section is added +# manually or by CI when a release is pushed). +# Else, find the next line that matches the `section` variable, and insert a new line +# under it (we also assume that section headers are added already). +inside_version_dev = False +already_added_entry = False +while orig_lines: + line = orig_lines.pop(0) + + # If the line already contains a link to the PR, don't add it again. + line = _skip_existing_entry_for_this_pr(line, same_section=False) + + if ( + line.startswith("## ") and not line.strip() == "# nf-core/sarek: Changelog" + ): # Version header, e.g. "## 2.12dev" + print(f"Found version header: {line.strip()}") + updated_lines.append(line) + + # Parse version from the line `## 3.4.4` or + # `## [3.4.4](https://github.com/nf-core/sarek/releases/tag/3.4.4) - Ruopsokjåkhå` ... + if not (m := re.match(r".*(\d+\.\d+.\d*(dev)?).*", line)): + print(f"Cannot parse version from line {line.strip()}.", file=sys.stderr) + sys.exit(1) + version = m.group(1) + print(f"Found version: {version}") + + if not inside_version_dev: + if not version.endswith("dev"): + print( + "Can't find a 'dev' version section in the changelog. Make sure " + "it's created, and all the required sections, e.g. `### Template` are created under it .", + file=sys.stderr, + ) + sys.exit(1) + inside_version_dev = True + else: + if version.endswith("dev"): + print( + f"Found another 'dev' version section in the changelog, make" + f"sure to change it to a 'release' stable version tag. " + f"Line: {line.strip()}", + file=sys.stderr, + ) + sys.exit(1) + # We are past the dev version, so just add back the rest of the lines and break. + while orig_lines: + line = orig_lines.pop(0) + line = _skip_existing_entry_for_this_pr(line, same_section=False) + if line: + updated_lines.append(line) + break + continue + print(f"Found line: {line.strip()}") + print(f"inside_version_dev: {inside_version_dev}") + print(f"section_header: {section_header}") + if inside_version_dev and line.lower().startswith( + section_header.lower() + ): # Section of interest header + print(f"Found section header: {line.strip()}") + if already_added_entry: + print( + f"Already added new lines into section {section}, is the section duplicated?", + file=sys.stderr, + ) + sys.exit(1) + updated_lines.append(line) + # Collecting lines until the next section. + section_lines: List[str] = [] + while True: + line = orig_lines.pop(0) + if line.startswith("#"): + print(f"Found the next section header: {line.strip()}") + # Found the next section header, so need to put all the lines we collected. + updated_lines.append("\n") + _updated_lines = [_l for _l in section_lines + new_lines if _l.strip()] + updated_lines.extend(_updated_lines) + updated_lines.append("\n") + if new_lines: + print( + f"Updated {changelog_path} section '{section}' with lines:\n" + + "".join(new_lines) + ) + else: + print( + f"Removed existing entry from {changelog_path} section '{section}'" + ) + already_added_entry = True + # Pushing back the next section header line + orig_lines.insert(0, line) + break + # If the line already contains a link to the PR, don't add it again. + line = _skip_existing_entry_for_this_pr(line, same_section=True) + section_lines.append(line) + + else: + updated_lines.append(line) + + +def collapse_newlines(lines: List[str]) -> List[str]: + updated = [] + for idx in range(len(lines)): + if idx != 0 and not lines[idx].strip() and not lines[idx - 1].strip(): + continue + updated.append(lines[idx]) + return updated + + +updated_lines = collapse_newlines(updated_lines) + +# Finally, writing the updated lines back. +with changelog_path.open("w") as f: + f.writelines(updated_lines) diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml new file mode 100644 index 0000000000..cebcc854bc --- /dev/null +++ b/.github/workflows/changelog.yml @@ -0,0 +1,90 @@ +name: Update CHANGELOG.md +on: + issue_comment: + types: [created] + pull_request_target: + types: [opened] + branches: + - dev + +jobs: + update_changelog: + runs-on: ubuntu-latest + # Run if comment is on a PR with the main repo, and if it contains the magic keywords. + # Or run on PR creation, unless asked otherwise in the title. + if: | + github.repository_owner == 'nf-core' && ( + github.event_name == 'pull_request_target' || + github.event.issue.pull_request && startsWith(github.event.comment.body, '@nf-core-bot changelog') + ) + + steps: + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 + with: + token: ${{ secrets.NF_CORE_BOT_AUTH_TOKEN }} + + # Action runs on the issue comment, so we don't get the PR by default. + # Use the GitHub CLI to check out the PR: + - name: Checkout Pull Request + env: + GH_TOKEN: ${{ secrets.NF_CORE_BOT_AUTH_TOKEN }} + run: | + if [[ "${{ github.event_name }}" == "issue_comment" ]]; then + PR_NUMBER="${{ github.event.issue.number }}" + elif [[ "${{ github.event_name }}" == "pull_request_target" ]]; then + PR_NUMBER="${{ github.event.pull_request.number }}" + fi + gh pr checkout $PR_NUMBER + + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 + with: + python-version: "3.12" + + - name: Install packages + run: | + python -m pip install --upgrade pip + pip install pyyaml + + - name: Update CHANGELOG.md from the PR title + env: + COMMENT: ${{ github.event.comment.body }} + GH_TOKEN: ${{ secrets.NF_CORE_BOT_AUTH_TOKEN }} + run: | + if [[ "${{ github.event_name }}" == "issue_comment" ]]; then + export PR_NUMBER='${{ github.event.issue.number }}' + export PR_TITLE='${{ github.event.issue.title }}' + elif [[ "${{ github.event_name }}" == "pull_request_target" ]]; then + export PR_NUMBER='${{ github.event.pull_request.number }}' + export PR_TITLE='${{ github.event.pull_request.title }}' + fi + python ${GITHUB_WORKSPACE}/.github/workflows/changelog.py + + - name: Check if CHANGELOG.md actually changed + run: | + git diff --exit-code ${GITHUB_WORKSPACE}/CHANGELOG.md || echo "changed=YES" >> $GITHUB_ENV + echo "File changed: ${{ env.changed }}" + + - name: Set up Python 3.12 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 + with: + python-version: "3.12" + cache: "pip" + + - name: Install pre-commit + run: pip install pre-commit + + - name: Run pre-commit checks + if: env.changed == 'YES' + run: | + pre-commit run --all-files + + - name: Commit and push changes + if: env.changed == 'YES' + run: | + git config user.email "core@nf-co.re" + git config user.name "nf-core-bot" + git config push.default upstream + git add ${GITHUB_WORKSPACE}/CHANGELOG.md + git status + git commit -m "[automated] Update CHANGELOG.md" + git push From be8a5272adf39c2c3fa6a25471312aa33aefc816 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Mon, 14 Oct 2024 16:42:00 +0200 Subject: [PATCH 2/4] fix script --- .github/workflows/changelog.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/changelog.py b/.github/workflows/changelog.py index e6d8cb3fb4..95df0ab45f 100755 --- a/.github/workflows/changelog.py +++ b/.github/workflows/changelog.py @@ -6,7 +6,7 @@ PR_TITLE, PR_NUMBER, GITHUB_WORKSPACE. Adds a line into the CHANGELOG.md: -* Looks for the section to add the line to, based on the PR title, e.g. `Added:`, `Changed:`. +* Looks for the section to add the line to, based on the PR title, e.g. `Feat:`, `Fix:`. * All other change will go under the "## dev" section. * If an entry for the PR is already added, it will not run. @@ -57,10 +57,10 @@ def _determine_change_type(pr_title) -> Tuple[str, str]: Returns a tuple of the section name and the module info. """ sections = { - "Added": "### Added", - "Changed": "### Changed", - "Fixed": "### Fixed", - "Removed": "### Removed", + "ADD": "### Added", + "FEAT": "### Changed", + "FIX": "### Fixed", + "REMOVE": "### Removed", } current_section_header = "## dev" current_section = "dev" From 3c887c960ac80d4d6b1c8e9cc28306e4ee5e34fd Mon Sep 17 00:00:00 2001 From: maxulysse Date: Mon, 14 Oct 2024 16:47:27 +0200 Subject: [PATCH 3/4] env or ENV? --- .github/workflows/changelog.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index cebcc854bc..a7c72a4c98 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -21,13 +21,13 @@ jobs: steps: - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 with: - token: ${{ secrets.NF_CORE_BOT_AUTH_TOKEN }} + token: ${{ secrets.nf_core_bot_auth_token }} # Action runs on the issue comment, so we don't get the PR by default. # Use the GitHub CLI to check out the PR: - name: Checkout Pull Request env: - GH_TOKEN: ${{ secrets.NF_CORE_BOT_AUTH_TOKEN }} + GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }} run: | if [[ "${{ github.event_name }}" == "issue_comment" ]]; then PR_NUMBER="${{ github.event.issue.number }}" @@ -48,7 +48,7 @@ jobs: - name: Update CHANGELOG.md from the PR title env: COMMENT: ${{ github.event.comment.body }} - GH_TOKEN: ${{ secrets.NF_CORE_BOT_AUTH_TOKEN }} + GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }} run: | if [[ "${{ github.event_name }}" == "issue_comment" ]]; then export PR_NUMBER='${{ github.event.issue.number }}' From 286ff1faa6f56b8eabc98eb8f5c1b74a6313ec6a Mon Sep 17 00:00:00 2001 From: maxulysse Date: Mon, 14 Oct 2024 16:49:14 +0200 Subject: [PATCH 4/4] change trigger --- .github/workflows/changelog.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index a7c72a4c98..42dd6a525b 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -12,11 +12,10 @@ jobs: runs-on: ubuntu-latest # Run if comment is on a PR with the main repo, and if it contains the magic keywords. # Or run on PR creation, unless asked otherwise in the title. - if: | - github.repository_owner == 'nf-core' && ( - github.event_name == 'pull_request_target' || - github.event.issue.pull_request && startsWith(github.event.comment.body, '@nf-core-bot changelog') - ) + if: > + contains(github.event.comment.html_url, '/pull/') && + contains(github.event.comment.body, '@nf-core-bot changelog') && + github.repository == 'nf-core/sarek' steps: - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4