Skip to content
CI/CD Pipeline

[NEW] Py Pkg Gen v2.3.0 Release #1942

Sign in to view logs

[NEW] Py Pkg Gen v2.3.0 Release

[NEW] Py Pkg Gen v2.3.0 Release #1942

Workflow file for this run

.github/workflows/test.yaml at 1ebe0b2
name: CI/CD Pipeline
# Continuous Integration / Continuous Delivery
# Triggers on all Branches and v* Tags
### Stress-Testing, with Multi-Factor Job Matrix, on: ###
# - tags v*
# - the 'stress-test' branch (GITHUB_REF_NAME == 'stress-test')
### Production PyPI Publish, pypi.org, on: ###
# - v* tags on 'master' branch only
### Staging/Test PyPI Publish, test.pypi.org, on: ###
## Test PyPI publish on: ##
# - v*-rc 'pre-release' tags on 'release' branch
### Dockerhub publish on ###
# - all branches and tags
on:
push:
branches:
- "*"
tags:
- v*
env:
### STRESS TEST Job MATRIX ###
FULL_MATRIX_STRATEGY: "{\"platform\": [\"ubuntu-latest\", \"macos-latest\", \"windows-latest\"], \"python-version\": [\"3.7\", \"3.8\", \"3.9\", \"3.10\", \"3.12\"]}"
# Python 3.7 has reached End of Life (EOL) on June 27th, 2023
# Python 3.12 is in bugfix mode, same as 3.11 -> can start supporting 3.12 it
UBUNTU_PY310_STRATEGY: "{\"platform\": [\"ubuntu-latest\"], \"python-version\": [\"3.10\"]}"
TEST_STRATEGY: "{\"platform\": [\"ubuntu-latest\", \"macos-latest\", \"windows-latest\"], \"python-version\":[\"3.10\"]}"
##### JOB ON/OFF SWITCHES - Top/1st level overrides #####
RUN_UNIT_TESTS: "true"
RUN_LINT_CHECKS: "true"
DOCKER_JOB_ON: "true"
PUBLISH_ON_PYPI: "true"
DOCS_JOB_ON: "true"
DRAW_DEPENDENCIES: "true"
##########################
### DOCKER Job Policy ####
# Override Docker Policy-dependent decision-making and
# Accept any ALL (branch/build) to Publish to Dockerhub
# if true, it will push image and ignore DOCKER_JOB_POLICY
ALWAYS_BUILD_N_PUBLISH_DOCKER: "false"
DOCKER_JOB_POLICY: "CDeployment"
# - CDeployment : '2' Builds and Publishes only if Tests ran and passed
# - CDelivery : '3' Builds and Publishes if Tests Passed or if Tests were Skipped
############################
#### DOCS Job ####
# 2nd level override
ALWAYS_DOCS: "false"
DOCS_JOB_POLICY: '2' # {2, 3}
PY_VERSION: "3.11"
#### STATIC CODE ANALYSIS Job ####
ALWAYS_LINT: "false"
LINT_JOB_POLICY: '2' # {2, 3}
## Python Runtime version to set the Job runner with ##
STATIC_ANALYSIS_PY: "3.10"
## Pylint Minimum Acceptance Rating/Score ##
PYLINT_SCORE_THRESHOLD: "8.2"
#### CODE VISUALIZATION Job ####
ALWAYS_CODE_VIZ: "false"
CODE_VIZ_POLICY: '2' # {2, 3}
##########################
jobs:
# we use the below to read the workflow env vars and be able to use in "- if:" Job conditionals
# now we can do -> if: ${{ needs.set_github_outputs.outputs.TESTS_ENABLED == 'true' }}
# github does not have a way to simply do "- if: ${{ env.RUN_UNIT_TESTS == 'true' }} " !!
set_github_outputs:
name: Read Workflow Env Section Vars and set Github Outputs
runs-on: ubuntu-latest
steps:
- name: Pass 'env' section variables to GITHUB_OUTPUT
id: pass-env-to-output
env:
BOARDING_EVENT: 'do Boarding CI Tests'
BOARDING_MSG: "Auto Merging '[^']+' carrying '([^']+)' Changes"
run: |
BRANCH_NAME=${GITHUB_REF_NAME}
PIPE_DOCS_POLICY="${{ (env.DOCS_JOB_ON != 'true' && '0') || (env.ALWAYS_DOCS == 'true' && '1') || env.DOCS_JOB_POLICY }}"
# set the matrix strategy to Full Matrix Stress Test if on master/main or stress-test branch or any tag
if [[ $BRANCH_NAME == "stress-test" || $GITHUB_REF == refs/tags/* ]]; then
echo "matrix=$FULL_MATRIX_STRATEGY" >> $GITHUB_OUTPUT
# github.event.head_commit.message has 'do Boarding CI Tests' string
elif [[ "${{ contains(github.event.head_commit.message, env.BOARDING_EVENT) }}" == 'true' ]]; then
echo "matrix=$TEST_STRATEGY" >> $GITHUB_OUTPUT
# interpret Boarding Message: if head_commit.message matches BOARDING_MSG regex
elif [[ "${{ github.event.head_commit.message }}" =~ ${BOARDING_MSG} ]]; then
affected_components="${BASH_REMATCH[1]}"
echo "--> Detected Affected Components: $affected_components <--"
if [[ "$affected_components" =~ "Distro" ]]; then
echo "matrix=$TEST_STRATEGY" >> $GITHUB_OUTPUT
else
echo "matrix=$UBUNTU_PY310_STRATEGY" >> $GITHUB_OUTPUT
fi
if [[ "$affected_components" =~ "Docs" ]]; then
# set policy to 1 to trigger Docs Build (higher level override might be in place)
PIPE_DOCS_POLICY="${{ (env.DOCS_JOB_ON != 'true' && '0') || (env.ALWAYS_DOCS == 'true' && '1') || '1' }}"
fi
else
# Fall back to the default strategy
echo "matrix=$UBUNTU_PY310_STRATEGY" >> $GITHUB_OUTPUT
fi
## we skip ci, on pushes to 'release' branch ##
echo "TESTS_ENABLED=${{ env.RUN_UNIT_TESTS == 'true' && github.ref_name != 'release' }}" >> $GITHUB_OUTPUT
echo "PUBLISH_ON_PYPI=$PUBLISH_ON_PYPI" >> $GITHUB_OUTPUT
echo "PIPE_DOCS_POLICY=$PIPE_DOCS_POLICY" >> $GITHUB_OUTPUT
echo "PIPE_DOCS_PY=$PY_VERSION" >> $GITHUB_OUTPUT
## Docker - Pipeline Settings ##
- id: derive_docker_policy
run: echo "POL=${{ (env.DOCKER_JOB_ON != 'true' && '0') || (env.ALWAYS_BUILD_N_PUBLISH_DOCKER == 'true' && '1') || (env.DOCKER_JOB_POLICY == 'CDeployment' && '2') || (env.DOCKER_JOB_POLICY == 'CDelivery' && '3') }}" >> $GITHUB_OUTPUT
## Static Code Analysis - Pipeline Settings ##
- id: derive_sqa_policy
run: echo "POL=${{ (env.RUN_LINT_CHECKS != 'true' && '0') || (env.ALWAYS_LINT == 'true' && '1') || env.LINT_JOB_POLICY }}" >> $GITHUB_OUTPUT
- id: read_sqa_py
run: echo SQA_PY=${{ env.STATIC_ANALYSIS_PY }} >> $GITHUB_OUTPUT
- id: read_pylint_score_threshold
run: echo PYLINT_SCORE_THRESHOLD=${{ env.PYLINT_SCORE_THRESHOLD }} >> $GITHUB_OUTPUT
## Code Visualization - Pipeline Settings ##
- id: derive_code_viz_policy
run: echo "POL=${{ (env.DRAW_DEPENDENCIES != 'true' && '0') || (env.ALWAYS_CODE_VIZ == 'true' && '1') || env.CODE_VIZ_POLICY }}" >> $GITHUB_OUTPUT
outputs:
matrix: ${{ steps.pass-env-to-output.outputs.matrix }}
TESTS_ENABLED: ${{ steps.pass-env-to-output.outputs.TESTS_ENABLED }}
PUBLISH_ON_PYPI: ${{ steps.pass-env-to-output.outputs.PUBLISH_ON_PYPI }}
## DOCS - Pipeline Settings ##
PIPE_DOCS_POLICY: ${{ steps.pass-env-to-output.outputs.PIPE_DOCS_POLICY }}
PIPE_DOCS_PY: ${{ steps.pass-env-to-output.outputs.PIPE_DOCS_PY }}
## Docker - Pipeline Settings ##
PIPE_DOCKER_POLICY: ${{ steps.derive_docker_policy.outputs.POL }}
## Static Code Analysis - Pipeline Settings ##
PIPE_SQA_POLICY: ${{ steps.derive_sqa_policy.outputs.POL }}
PIPE_SQA_PY: ${{ steps.read_sqa_py.outputs.SQA_PY }}
PIPE_SQA_PYLINT_PASS_SCORE: ${{ steps.read_pylint_score_threshold.outputs.PYLINT_SCORE_THRESHOLD }}
## Code Visualization - Pipeline Settings ##
PIPE_CODE_VIZ_POLICY: ${{ steps.derive_code_viz_policy.outputs.POL }}
# RUN TEST SUITE ON ALL PLATFORMS
test_suite:
runs-on: ${{ matrix.platform }}
needs: set_github_outputs
if: ${{ needs.set_github_outputs.outputs.TESTS_ENABLED == 'true' }}
strategy:
matrix: ${{fromJSON(needs.set_github_outputs.outputs.matrix)}}
env:
WHEELS_PIP_DIR: "wheels-pip"
outputs:
PEP_VERSION: ${{ steps.extract_wheel_info.outputs.PEP_VERSION }}
steps:
- run: echo "Platform -> ${{ matrix.platform }} , Python -> ${{ matrix.python-version }}"
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- run: python -m pip install --upgrade pip && python -m pip install tox==3.28 tox-gh-actions
- name: Pin 'Static Type Checking' Dependencies
run: tox -vv -s false -e pin-deps -- -E typing
- name: Do Type Checking
run: tox -e type -vv -s false
### Check if Requested RC Pipeline ###
- name: "Update Source Sem Ver with Release Candidate *-rc' suffix"
if: ${{ startsWith(github.event.ref, 'refs/tags/v') && contains(github.event.ref, '-rc') }}
shell: bash
run: |
# Extract PROD Sem Ver
SEMVER="$(grep -E -o '^version\s*=\s*\".*\"' pyproject.toml | cut -d'"' -f2)"
# Derive RC Sem Ver
RC_SEMVER="${SEMVER}-rc"
## Set Source Sem Ver to Release Candidate Sem Ver ##
# if needed: chmod +x ./scripts/distro-sem-ver-bump.sh
sh ./scripts/distro-sem-ver-bump.sh "${RC_SEMVER}"
###### TEST SUITE RUN ######
- name: Run Unit Tests on Edit, and Sdist; and build Wheels
shell: bash
run: |
set -o pipefail
tox -vv -s false | tee test_output.log
env:
PLATFORM: ${{ matrix.platform }}
BUILD_DEST: ${{ env.WHEELS_PIP_DIR }}
- name: Show produced Wheel(s) for Distro and its Requirements
run: ls -l ${{ env.WHEELS_PIP_DIR }}
# parse test_output.log and match string, like:
# Created wheel for cookiecutter_python: filename=cookiecutter_python-1.12.5.dev0-py3-none-any.whl size=199750 sha256=446c75803a6eea1d7ac6af60e0fbb0000483f97ef44eb39469ab3bd997b2a7d8
# starting line with 'Created wheel for' and extracting filename value and size value
- name: Extract Wheel Name and Size
id: extract_wheel_info
shell: bash
run: |
WHEEL_INFO=$(grep -E "Created wheel for" test_output.log | sed -E "s/.*filename=([^ ]+) size=([^ ]+) .*/\1 \2/")
# extract file name
WHEEL_NAME=$(echo $WHEEL_INFO | cut -d ' ' -f 1)
echo "WHEEL_NAME=$WHEEL_NAME" >> $GITHUB_ENV
# extract file size
WHEEL_SIZE=$(echo $WHEEL_INFO | cut -d ' ' -f 2)
echo "WHEEL_SIZE=$WHEEL_SIZE" >> $GITHUB_ENV
# extract, ie '1.12.5dev0' from 'cookiecutter_python-1.15.0rc0-py3-none-any.whl'
PEP_VERSION=$(echo $WHEEL_NAME | sed -E "s/cookiecutter_python-([^ ]+)-py3-none-any.whl/\1/")
echo "PEP_VERSION=${PEP_VERSION}" >> $GITHUB_OUTPUT
- run: 'echo "WHEEL_NAME: $WHEEL_NAME SIZE: $WHEEL_SIZE"'
- run: 'echo "PEP_VERSION: $PEP_VERSION"'
- name: Run Test Suite Against Wheel
run: tox -e ${{ env.TOX_ENV_WHEEL_TEST }} -s false
env:
PLATFORM: ${{ matrix.platform }}
TOX_ENV_WHEEL_TEST: ${{ (matrix.platform == 'windows-latest' && matrix.python-version != '3.7') && 'wheel-test-windows' || 'wheel-test'}}
BUILD_DEST: ${{ env.WHEELS_PIP_DIR }}
WHEEL: ${{ env.WHEEL_NAME }}
## Code Coverage ##
- name: "Combine Coverage (dev, sdist, wheel) & make Reports"
run: tox -e coverage --sitepackages -vv -s false
- name: Rename Coverage Files
shell: bash
run: |
mv ./.tox/coverage.xml ./coverage-${{ matrix.platform }}-${{ matrix.python-version }}.xml
- name: "Upload Test Coverage as Artifacts"
uses: actions/upload-artifact@v3
with:
name: all_coverage_raw
path: coverage-${{ matrix.platform }}-${{ matrix.python-version }}.xml
if-no-files-found: error
- name: Check for compliance with Python Best Practices
shell: bash
env:
PKG_VERSION: ${{ steps.extract_wheel_info.outputs.PEP_VERSION }}
WHEEL_PATH: ${{ env.WHEELS_PIP_DIR }}/${{ env.WHEEL_NAME }}
run: |
DIST_DIR=dist
echo "DIST_DIR=${DIST_DIR}" >> $GITHUB_ENV
mkdir ${DIST_DIR}
mv ".tox/${DIST_DIR}/cookiecutter_python-${PKG_VERSION}.tar.gz" "${DIST_DIR}"
mv "${{ env.WHEEL_PATH }}" "${DIST_DIR}"
tox -e check -vv -s false
- name: Upload Source & Wheel distributions as Artefacts
uses: actions/upload-artifact@v3
with:
name: dist-${{ matrix.platform }}-${{ matrix.python-version }}
path: ${{ env.DIST_DIR }}
if-no-files-found: error
codecov_coverage_host:
runs-on: ubuntu-latest
needs: test_suite
steps:
- uses: actions/checkout@v4
- name: Get Codecov binary
run: |
curl -Os https://uploader.codecov.io/latest/linux/codecov
chmod +x codecov
- name: Download Raw Coverage Data Artefacts
uses: actions/download-artifact@v3
with:
name: all_coverage_raw
- name: Upload Coverage Reports to Codecov
run: |
for file in coverage*.xml; do
OS_NAME=$(echo $file | sed -E "s/coverage-(\w\+)-/\1/")
PY_VERSION=$(echo $file | sed -E "s/coverage-\w\+-(\d\.)\+/\1/")
./codecov -f $file -e "OS=$OS_NAME,PYTHON=$PY_VERSION" --flags unittests --verbose
echo "Sent to Codecov: $file !"
done
## DOCKER BUILD and PUBLISH ON DOCKERHUB ##
# Ref Page: https://automated-workflows.readthedocs.io/en/main/ref_docker/
docker_build:
needs: [set_github_outputs, test_suite]
uses: boromir674/automated-workflows/.github/workflows/[email protected]
if: always()
with:
acceptance_policy: ${{ needs.set_github_outputs.outputs.PIPE_DOCKER_POLICY }}
image_slug: "generate-python"
# target_stage: "some_stage_alias" # no stage, means no `--target` flag, on build
tests_pass: ${{ needs.test_suite.result == 'success' }}
tests_run: ${{ !contains(fromJSON('["skipped", "cancelled"]'), needs.test_suite.result) }}
DOCKER_USER: ${{ vars.DOCKER_USER }}
secrets:
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
## JOB: Signal for Automated PyPI Upload ##
check_which_git_branch_we_are_on:
runs-on: ubuntu-latest
if: ${{ startsWith(github.event.ref, 'refs/tags/v') }}
env:
RELEASE_BR: 'release'
MAIN_BR: 'master'
steps:
# Fetch 'master' and 'release' branches
- uses: actions/checkout@v4
with:
fetch-depth: 0
- run: git branch --track "${{ env.RELEASE_BR }}" "origin/${{ env.RELEASE_BR }}"
- name: "Check if '${{ github.ref }}' tag is on '${{ env.MAIN_BR }}' branch"
uses: rickstaa/action-contains-tag@v1
id: main_contains_tag
with:
reference: ${{ env.MAIN_BR }}
tag: "${{ github.ref }}"
- name: "Check if '${{ github.ref }}' tag is on '${{ env.RELEASE_BR }}' branch"
uses: rickstaa/action-contains-tag@v1
id: release_contains_tag
with:
reference: ${{ env.RELEASE_BR }}
tag: "${{ github.ref }}"
- name: Pick Production or Test Environment, if tag on master or release branch respectively
id: set_environment_name
run: |
DEPLOY=true
if [[ "${{ steps.main_contains_tag.outputs.retval }}" == "true" ]]; then
echo "ENVIRONMENT_NAME=PROD_DEPLOYMENT" >> $GITHUB_OUTPUT
elif [[ "${{ steps.release_contains_tag.outputs.retval }}" == "true" ]]; then
echo "ENVIRONMENT_NAME=TEST_DEPLOYMENT" >> $GITHUB_OUTPUT
else
echo "A tag was pushed but not on master or release branch. No deployment will be done."
DEPLOY=false
fi
echo "AUTOMATED_DEPLOY=$DEPLOY" >> $GITHUB_OUTPUT
outputs:
ENVIRONMENT_NAME: ${{ steps.set_environment_name.outputs.ENVIRONMENT_NAME }}
AUTOMATED_DEPLOY: ${{ steps.set_environment_name.outputs.AUTOMATED_DEPLOY }}
## JOB: PYPI UPLOAD ##
pypi_publish:
needs: [set_github_outputs, test_suite, check_which_git_branch_we_are_on]
uses: boromir674/automated-workflows/.github/workflows/pypi_env.yml@8e97f596067fcbbaa0a6927ec1ee47dce4ab5f1a
with:
should_trigger: ${{ needs.set_github_outputs.outputs.PUBLISH_ON_PYPI == 'true' && needs.check_which_git_branch_we_are_on.outputs.AUTOMATED_DEPLOY == 'true' }}
distro_name: cookiecutter_python
distro_version: ${{ needs.test_suite.outputs.PEP_VERSION }}
pypi_env: '${{ needs.check_which_git_branch_we_are_on.outputs.ENVIRONMENT_NAME }}'
artifacts_path: downloaded-artifacts
require_wheel: true
allow_existing: true
secrets:
TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
### STATIC CODE ANALYSIS & LINTING ###
lint:
name: Static Code Analysis
needs: set_github_outputs
uses: ./.github/workflows/policy_lint.yml
with:
run_policy: ${{ needs.set_github_outputs.outputs.PIPE_SQA_POLICY }}
dedicated_branches: 'master, main, dev'
source_code_targets: 'src,tests,scripts'
python_version: ${{ needs.set_github_outputs.outputs.PIPE_SQA_PY }}
pylint_threshold: ${{ needs.set_github_outputs.outputs.PIPE_SQA_PYLINT_PASS_SCORE }}
### DOCS BUILD/TEST - DOCUMENTATION SITE ###
docs:
name: Build Documentation
needs: set_github_outputs
# bafaa2c2a014758a4421fe9b5c02ba66dbfdbef6
uses: boromir674/automated-workflows/.github/workflows/policy_docs.yml@test
with:
run_policy: '${{ needs.set_github_outputs.outputs.PIPE_DOCS_POLICY }}'
python_version: ${{ needs.set_github_outputs.outputs.PIPE_DOCS_PY }}
command: "tox -e docs --sitepackages -vv -s false"
# command: "tox -e pin-deps -- -E docs && tox -e docs --sitepackages -vv -s false"
### DRAW PYTHON DEPENDENCY GRAPHS ###
code_visualization:
needs: set_github_outputs
name: Code Visualization of Python Imports as Graphs, in .svg
uses: boromir674/automated-workflows/.github/workflows/[email protected]
with:
run_policy: '${{ needs.set_github_outputs.outputs.PIPE_CODE_VIZ_POLICY }}'
branches: 'main, master, dev'
source_code_targets: 'src'
python_version: '3.10'
artifacts_dir: 'dependency-graphs'
### Make a Github Release ###
gh_release:
needs: [test_suite, check_which_git_branch_we_are_on]
if: ${{ needs.check_which_git_branch_we_are_on.outputs.AUTOMATED_DEPLOY == 'true' }}
uses: boromir674/automated-workflows/.github/workflows/gh-release.yml@test
name: 'GH Release'
with:
tag: ${{ github.ref_name }}
draft: ${{ needs.check_which_git_branch_we_are_on.outputs.ENVIRONMENT_NAME == 'TEST_DEPLOYMENT' }}
secrets:
# passing the GH_TOKEN PAT, to render in GH as ie: 'boromir674 released this yesterday', instead of 'github-actions released this yesterday'
gh_token: ${{ secrets.GH_TOKEN }}