set generate workflow to production state #1940
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI/CD Pipeline | |
| # Continuous Integration / Continuous Delivery | |
| # Triggers on all Branches and v* Tags | |
| ### Stress-Testing, with Multi-Factor Job Matrix, on: ### | |
| # - tags v* | |
| # - the 'stress-test' branch (GITHUB_REF_NAME == 'stress-test') | |
| ### Production PyPI Publish, pypi.org, on: ### | |
| # - v* tags on 'master' branch only | |
| ### Staging/Test PyPI Publish, test.pypi.org, on: ### | |
| ## Test PyPI publish on: ## | |
| # - v*-rc 'pre-release' tags on 'release' branch | |
| ### Dockerhub publish on ### | |
| # - all branches and tags | |
| on: | |
| push: | |
| branches: | |
| - "*" | |
| tags: | |
| - v* | |
| env: | |
| ### STRESS TEST Job MATRIX ### | |
| FULL_MATRIX_STRATEGY: "{\"platform\": [\"ubuntu-latest\", \"macos-latest\", \"windows-latest\"], \"python-version\": [\"3.7\", \"3.8\", \"3.9\", \"3.10\", \"3.12\"]}" | |
| # Python 3.7 has reached End of Life (EOL) on June 27th, 2023 | |
| # Python 3.12 is in bugfix mode, same as 3.11 -> can start supporting 3.12 it | |
| UBUNTU_PY310_STRATEGY: "{\"platform\": [\"ubuntu-latest\"], \"python-version\": [\"3.10\"]}" | |
| TEST_STRATEGY: "{\"platform\": [\"ubuntu-latest\", \"macos-latest\", \"windows-latest\"], \"python-version\":[\"3.10\"]}" | |
| ##### JOB ON/OFF SWITCHES - Top/1st level overrides ##### | |
| RUN_UNIT_TESTS: "true" | |
| RUN_LINT_CHECKS: "true" | |
| DOCKER_JOB_ON: "true" | |
| PUBLISH_ON_PYPI: "true" | |
| DOCS_JOB_ON: "true" | |
| DRAW_DEPENDENCIES: "true" | |
| ########################## | |
| ### DOCKER Job Policy #### | |
| # Override Docker Policy-dependent decision-making and | |
| # Accept any ALL (branch/build) to Publish to Dockerhub | |
| # if true, it will push image and ignore DOCKER_JOB_POLICY | |
| ALWAYS_BUILD_N_PUBLISH_DOCKER: "false" | |
| DOCKER_JOB_POLICY: "CDeployment" | |
| # - CDeployment : '2' Builds and Publishes only if Tests ran and passed | |
| # - CDelivery : '3' Builds and Publishes if Tests Passed or if Tests were Skipped | |
| ############################ | |
| #### DOCS Job #### | |
| # 2nd level override | |
| ALWAYS_DOCS: "false" | |
| DOCS_JOB_POLICY: '2' # {2, 3} | |
| PY_VERSION: "3.11" | |
| #### STATIC CODE ANALYSIS Job #### | |
| ALWAYS_LINT: "false" | |
| LINT_JOB_POLICY: '2' # {2, 3} | |
| ## Python Runtime version to set the Job runner with ## | |
| STATIC_ANALYSIS_PY: "3.10" | |
| ## Pylint Minimum Acceptance Rating/Score ## | |
| PYLINT_SCORE_THRESHOLD: "8.2" | |
| #### CODE VISUALIZATION Job #### | |
| ALWAYS_CODE_VIZ: "false" | |
| CODE_VIZ_POLICY: '2' # {2, 3} | |
| ########################## | |
| jobs: | |
| # we use the below to read the workflow env vars and be able to use in "- if:" Job conditionals | |
| # now we can do -> if: ${{ needs.set_github_outputs.outputs.TESTS_ENABLED == 'true' }} | |
| # github does not have a way to simply do "- if: ${{ env.RUN_UNIT_TESTS == 'true' }} " !! | |
| set_github_outputs: | |
| name: Read Workflow Env Section Vars and set Github Outputs | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Pass 'env' section variables to GITHUB_OUTPUT | |
| id: pass-env-to-output | |
| env: | |
| BOARDING_EVENT: 'do Boarding CI Tests' | |
| BOARDING_MSG: "Auto Merging '[^']+' carrying '([^']+)' Changes" | |
| run: | | |
| BRANCH_NAME=${GITHUB_REF_NAME} | |
| PIPE_DOCS_POLICY="${{ (env.DOCS_JOB_ON != 'true' && '0') || (env.ALWAYS_DOCS == 'true' && '1') || env.DOCS_JOB_POLICY }}" | |
| # set the matrix strategy to Full Matrix Stress Test if on master/main or stress-test branch or any tag | |
| if [[ $BRANCH_NAME == "stress-test" || $GITHUB_REF == refs/tags/* ]]; then | |
| echo "matrix=$FULL_MATRIX_STRATEGY" >> $GITHUB_OUTPUT | |
| # github.event.head_commit.message has 'do Boarding CI Tests' string | |
| elif [[ "${{ contains(github.event.head_commit.message, env.BOARDING_EVENT) }}" == 'true' ]]; then | |
| echo "matrix=$TEST_STRATEGY" >> $GITHUB_OUTPUT | |
| # interpret Boarding Message: if head_commit.message matches BOARDING_MSG regex | |
| elif [[ "${{ github.event.head_commit.message }}" =~ ${BOARDING_MSG} ]]; then | |
| affected_components="${BASH_REMATCH[1]}" | |
| echo "--> Detected Affected Components: $affected_components <--" | |
| if [[ "$affected_components" =~ "Distro" ]]; then | |
| echo "matrix=$TEST_STRATEGY" >> $GITHUB_OUTPUT | |
| else | |
| echo "matrix=$UBUNTU_PY310_STRATEGY" >> $GITHUB_OUTPUT | |
| fi | |
| if [[ "$affected_components" =~ "Docs" ]]; then | |
| # set policy to 1 to trigger Docs Build (higher level override might be in place) | |
| PIPE_DOCS_POLICY="${{ (env.DOCS_JOB_ON != 'true' && '0') || (env.ALWAYS_DOCS == 'true' && '1') || '1' }}" | |
| fi | |
| else | |
| # Fall back to the default strategy | |
| echo "matrix=$UBUNTU_PY310_STRATEGY" >> $GITHUB_OUTPUT | |
| fi | |
| ## we skip ci, on pushes to 'release' branch ## | |
| echo "TESTS_ENABLED=${{ env.RUN_UNIT_TESTS == 'true' && github.ref_name != 'release' }}" >> $GITHUB_OUTPUT | |
| echo "PUBLISH_ON_PYPI=$PUBLISH_ON_PYPI" >> $GITHUB_OUTPUT | |
| echo "PIPE_DOCS_POLICY=$PIPE_DOCS_POLICY" >> $GITHUB_OUTPUT | |
| echo "PIPE_DOCS_PY=$PY_VERSION" >> $GITHUB_OUTPUT | |
| ## Docker - Pipeline Settings ## | |
| - id: derive_docker_policy | |
| run: echo "POL=${{ (env.DOCKER_JOB_ON != 'true' && '0') || (env.ALWAYS_BUILD_N_PUBLISH_DOCKER == 'true' && '1') || (env.DOCKER_JOB_POLICY == 'CDeployment' && '2') || (env.DOCKER_JOB_POLICY == 'CDelivery' && '3') }}" >> $GITHUB_OUTPUT | |
| ## Static Code Analysis - Pipeline Settings ## | |
| - id: derive_sqa_policy | |
| run: echo "POL=${{ (env.RUN_LINT_CHECKS != 'true' && '0') || (env.ALWAYS_LINT == 'true' && '1') || env.LINT_JOB_POLICY }}" >> $GITHUB_OUTPUT | |
| - id: read_sqa_py | |
| run: echo SQA_PY=${{ env.STATIC_ANALYSIS_PY }} >> $GITHUB_OUTPUT | |
| - id: read_pylint_score_threshold | |
| run: echo PYLINT_SCORE_THRESHOLD=${{ env.PYLINT_SCORE_THRESHOLD }} >> $GITHUB_OUTPUT | |
| ## Code Visualization - Pipeline Settings ## | |
| - id: derive_code_viz_policy | |
| run: echo "POL=${{ (env.DRAW_DEPENDENCIES != 'true' && '0') || (env.ALWAYS_CODE_VIZ == 'true' && '1') || env.CODE_VIZ_POLICY }}" >> $GITHUB_OUTPUT | |
| outputs: | |
| matrix: ${{ steps.pass-env-to-output.outputs.matrix }} | |
| TESTS_ENABLED: ${{ steps.pass-env-to-output.outputs.TESTS_ENABLED }} | |
| PUBLISH_ON_PYPI: ${{ steps.pass-env-to-output.outputs.PUBLISH_ON_PYPI }} | |
| ## DOCS - Pipeline Settings ## | |
| PIPE_DOCS_POLICY: ${{ steps.pass-env-to-output.outputs.PIPE_DOCS_POLICY }} | |
| PIPE_DOCS_PY: ${{ steps.pass-env-to-output.outputs.PIPE_DOCS_PY }} | |
| ## Docker - Pipeline Settings ## | |
| PIPE_DOCKER_POLICY: ${{ steps.derive_docker_policy.outputs.POL }} | |
| ## Static Code Analysis - Pipeline Settings ## | |
| PIPE_SQA_POLICY: ${{ steps.derive_sqa_policy.outputs.POL }} | |
| PIPE_SQA_PY: ${{ steps.read_sqa_py.outputs.SQA_PY }} | |
| PIPE_SQA_PYLINT_PASS_SCORE: ${{ steps.read_pylint_score_threshold.outputs.PYLINT_SCORE_THRESHOLD }} | |
| ## Code Visualization - Pipeline Settings ## | |
| PIPE_CODE_VIZ_POLICY: ${{ steps.derive_code_viz_policy.outputs.POL }} | |
| # RUN TEST SUITE ON ALL PLATFORMS | |
| test_suite: | |
| runs-on: ${{ matrix.platform }} | |
| needs: set_github_outputs | |
| if: ${{ needs.set_github_outputs.outputs.TESTS_ENABLED == 'true' }} | |
| strategy: | |
| matrix: ${{fromJSON(needs.set_github_outputs.outputs.matrix)}} | |
| env: | |
| WHEELS_PIP_DIR: "wheels-pip" | |
| outputs: | |
| PEP_VERSION: ${{ steps.extract_wheel_info.outputs.PEP_VERSION }} | |
| steps: | |
| - run: echo "Platform -> ${{ matrix.platform }} , Python -> ${{ matrix.python-version }}" | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - run: python -m pip install --upgrade pip && python -m pip install tox==3.28 tox-gh-actions | |
| - name: Pin 'Static Type Checking' Dependencies | |
| run: tox -vv -s false -e pin-deps -- -E typing | |
| - name: Do Type Checking | |
| run: tox -e type -vv -s false | |
| ### Check if Requested RC Pipeline ### | |
| - name: "Update Source Sem Ver with Release Candidate *-rc' suffix" | |
| if: ${{ startsWith(github.event.ref, 'refs/tags/v') && contains(github.event.ref, '-rc') }} | |
| shell: bash | |
| run: | | |
| # Extract PROD Sem Ver | |
| SEMVER="$(grep -E -o '^version\s*=\s*\".*\"' pyproject.toml | cut -d'"' -f2)" | |
| # Derive RC Sem Ver | |
| RC_SEMVER="${SEMVER}-rc" | |
| ## Set Source Sem Ver to Release Candidate Sem Ver ## | |
| # if needed: chmod +x ./scripts/distro-sem-ver-bump.sh | |
| sh ./scripts/distro-sem-ver-bump.sh "${RC_SEMVER}" | |
| ###### TEST SUITE RUN ###### | |
| - name: Run Unit Tests on Edit, and Sdist; and build Wheels | |
| shell: bash | |
| run: | | |
| set -o pipefail | |
| tox -vv -s false | tee test_output.log | |
| env: | |
| PLATFORM: ${{ matrix.platform }} | |
| BUILD_DEST: ${{ env.WHEELS_PIP_DIR }} | |
| - name: Show produced Wheel(s) for Distro and its Requirements | |
| run: ls -l ${{ env.WHEELS_PIP_DIR }} | |
| # parse test_output.log and match string, like: | |
| # Created wheel for cookiecutter_python: filename=cookiecutter_python-1.12.5.dev0-py3-none-any.whl size=199750 sha256=446c75803a6eea1d7ac6af60e0fbb0000483f97ef44eb39469ab3bd997b2a7d8 | |
| # starting line with 'Created wheel for' and extracting filename value and size value | |
| - name: Extract Wheel Name and Size | |
| id: extract_wheel_info | |
| shell: bash | |
| run: | | |
| WHEEL_INFO=$(grep -E "Created wheel for" test_output.log | sed -E "s/.*filename=([^ ]+) size=([^ ]+) .*/\1 \2/") | |
| # extract file name | |
| WHEEL_NAME=$(echo $WHEEL_INFO | cut -d ' ' -f 1) | |
| echo "WHEEL_NAME=$WHEEL_NAME" >> $GITHUB_ENV | |
| # extract file size | |
| WHEEL_SIZE=$(echo $WHEEL_INFO | cut -d ' ' -f 2) | |
| echo "WHEEL_SIZE=$WHEEL_SIZE" >> $GITHUB_ENV | |
| # extract, ie '1.12.5dev0' from 'cookiecutter_python-1.15.0rc0-py3-none-any.whl' | |
| PEP_VERSION=$(echo $WHEEL_NAME | sed -E "s/cookiecutter_python-([^ ]+)-py3-none-any.whl/\1/") | |
| echo "PEP_VERSION=${PEP_VERSION}" >> $GITHUB_OUTPUT | |
| - run: 'echo "WHEEL_NAME: $WHEEL_NAME SIZE: $WHEEL_SIZE"' | |
| - run: 'echo "PEP_VERSION: $PEP_VERSION"' | |
| - name: Run Test Suite Against Wheel | |
| run: tox -e ${{ env.TOX_ENV_WHEEL_TEST }} -s false | |
| env: | |
| PLATFORM: ${{ matrix.platform }} | |
| TOX_ENV_WHEEL_TEST: ${{ (matrix.platform == 'windows-latest' && matrix.python-version != '3.7') && 'wheel-test-windows' || 'wheel-test'}} | |
| BUILD_DEST: ${{ env.WHEELS_PIP_DIR }} | |
| WHEEL: ${{ env.WHEEL_NAME }} | |
| ## Code Coverage ## | |
| - name: "Combine Coverage (dev, sdist, wheel) & make Reports" | |
| run: tox -e coverage --sitepackages -vv -s false | |
| - name: Rename Coverage Files | |
| shell: bash | |
| run: | | |
| mv ./.tox/coverage.xml ./coverage-${{ matrix.platform }}-${{ matrix.python-version }}.xml | |
| - name: "Upload Test Coverage as Artifacts" | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: all_coverage_raw | |
| path: coverage-${{ matrix.platform }}-${{ matrix.python-version }}.xml | |
| if-no-files-found: error | |
| - name: Check for compliance with Python Best Practices | |
| shell: bash | |
| env: | |
| PKG_VERSION: ${{ steps.extract_wheel_info.outputs.PEP_VERSION }} | |
| WHEEL_PATH: ${{ env.WHEELS_PIP_DIR }}/${{ env.WHEEL_NAME }} | |
| run: | | |
| DIST_DIR=dist | |
| echo "DIST_DIR=${DIST_DIR}" >> $GITHUB_ENV | |
| mkdir ${DIST_DIR} | |
| mv ".tox/${DIST_DIR}/cookiecutter_python-${PKG_VERSION}.tar.gz" "${DIST_DIR}" | |
| mv "${{ env.WHEEL_PATH }}" "${DIST_DIR}" | |
| tox -e check -vv -s false | |
| - name: Upload Source & Wheel distributions as Artefacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: dist-${{ matrix.platform }}-${{ matrix.python-version }} | |
| path: ${{ env.DIST_DIR }} | |
| if-no-files-found: error | |
| codecov_coverage_host: | |
| runs-on: ubuntu-latest | |
| needs: test_suite | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Get Codecov binary | |
| run: | | |
| curl -Os https://uploader.codecov.io/latest/linux/codecov | |
| chmod +x codecov | |
| - name: Download Raw Coverage Data Artefacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: all_coverage_raw | |
| - name: Upload Coverage Reports to Codecov | |
| run: | | |
| for file in coverage*.xml; do | |
| OS_NAME=$(echo $file | sed -E "s/coverage-(\w\+)-/\1/") | |
| PY_VERSION=$(echo $file | sed -E "s/coverage-\w\+-(\d\.)\+/\1/") | |
| ./codecov -f $file -e "OS=$OS_NAME,PYTHON=$PY_VERSION" --flags unittests --verbose | |
| echo "Sent to Codecov: $file !" | |
| done | |
| ## DOCKER BUILD and PUBLISH ON DOCKERHUB ## | |
| # Ref Page: https://automated-workflows.readthedocs.io/en/main/ref_docker/ | |
| docker_build: | |
| needs: [set_github_outputs, test_suite] | |
| uses: boromir674/automated-workflows/.github/workflows/[email protected] | |
| if: always() | |
| with: | |
| acceptance_policy: ${{ needs.set_github_outputs.outputs.PIPE_DOCKER_POLICY }} | |
| image_slug: "generate-python" | |
| # target_stage: "some_stage_alias" # no stage, means no `--target` flag, on build | |
| tests_pass: ${{ needs.test_suite.result == 'success' }} | |
| tests_run: ${{ !contains(fromJSON('["skipped", "cancelled"]'), needs.test_suite.result) }} | |
| DOCKER_USER: ${{ vars.DOCKER_USER }} | |
| secrets: | |
| DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} | |
| ## JOB: Signal for Automated PyPI Upload ## | |
| check_which_git_branch_we_are_on: | |
| runs-on: ubuntu-latest | |
| if: ${{ startsWith(github.event.ref, 'refs/tags/v') }} | |
| env: | |
| RELEASE_BR: 'release' | |
| MAIN_BR: 'master' | |
| steps: | |
| # Fetch 'master' and 'release' branches | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - run: git branch --track "${{ env.RELEASE_BR }}" "origin/${{ env.RELEASE_BR }}" | |
| - name: "Check if '${{ github.ref }}' tag is on '${{ env.MAIN_BR }}' branch" | |
| uses: rickstaa/action-contains-tag@v1 | |
| id: main_contains_tag | |
| with: | |
| reference: ${{ env.MAIN_BR }} | |
| tag: "${{ github.ref }}" | |
| - name: "Check if '${{ github.ref }}' tag is on '${{ env.RELEASE_BR }}' branch" | |
| uses: rickstaa/action-contains-tag@v1 | |
| id: release_contains_tag | |
| with: | |
| reference: ${{ env.RELEASE_BR }} | |
| tag: "${{ github.ref }}" | |
| - name: Pick Production or Test Environment, if tag on master or release branch respectively | |
| id: set_environment_name | |
| run: | | |
| DEPLOY=true | |
| if [[ "${{ steps.main_contains_tag.outputs.retval }}" == "true" ]]; then | |
| echo "ENVIRONMENT_NAME=PROD_DEPLOYMENT" >> $GITHUB_OUTPUT | |
| elif [[ "${{ steps.release_contains_tag.outputs.retval }}" == "true" ]]; then | |
| echo "ENVIRONMENT_NAME=TEST_DEPLOYMENT" >> $GITHUB_OUTPUT | |
| else | |
| echo "A tag was pushed but not on master or release branch. No deployment will be done." | |
| DEPLOY=false | |
| fi | |
| echo "AUTOMATED_DEPLOY=$DEPLOY" >> $GITHUB_OUTPUT | |
| outputs: | |
| ENVIRONMENT_NAME: ${{ steps.set_environment_name.outputs.ENVIRONMENT_NAME }} | |
| AUTOMATED_DEPLOY: ${{ steps.set_environment_name.outputs.AUTOMATED_DEPLOY }} | |
| ## JOB: PYPI UPLOAD ## | |
| pypi_publish: | |
| needs: [set_github_outputs, test_suite, check_which_git_branch_we_are_on] | |
| uses: boromir674/automated-workflows/.github/workflows/pypi_env.yml@8e97f596067fcbbaa0a6927ec1ee47dce4ab5f1a | |
| with: | |
| should_trigger: ${{ needs.set_github_outputs.outputs.PUBLISH_ON_PYPI == 'true' && needs.check_which_git_branch_we_are_on.outputs.AUTOMATED_DEPLOY == 'true' }} | |
| distro_name: cookiecutter_python | |
| distro_version: ${{ needs.test_suite.outputs.PEP_VERSION }} | |
| pypi_env: '${{ needs.check_which_git_branch_we_are_on.outputs.ENVIRONMENT_NAME }}' | |
| artifacts_path: downloaded-artifacts | |
| require_wheel: true | |
| allow_existing: true | |
| secrets: | |
| TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} | |
| ### STATIC CODE ANALYSIS & LINTING ### | |
| lint: | |
| name: Static Code Analysis | |
| needs: set_github_outputs | |
| uses: ./.github/workflows/policy_lint.yml | |
| with: | |
| run_policy: ${{ needs.set_github_outputs.outputs.PIPE_SQA_POLICY }} | |
| dedicated_branches: 'master, main, dev' | |
| source_code_targets: 'src,tests,scripts' | |
| python_version: ${{ needs.set_github_outputs.outputs.PIPE_SQA_PY }} | |
| pylint_threshold: ${{ needs.set_github_outputs.outputs.PIPE_SQA_PYLINT_PASS_SCORE }} | |
| ### DOCS BUILD/TEST - DOCUMENTATION SITE ### | |
| docs: | |
| name: Build Documentation | |
| needs: set_github_outputs | |
| # bafaa2c2a014758a4421fe9b5c02ba66dbfdbef6 | |
| uses: boromir674/automated-workflows/.github/workflows/policy_docs.yml@test | |
| with: | |
| run_policy: '${{ needs.set_github_outputs.outputs.PIPE_DOCS_POLICY }}' | |
| python_version: ${{ needs.set_github_outputs.outputs.PIPE_DOCS_PY }} | |
| command: "tox -e docs --sitepackages -vv -s false" | |
| # command: "tox -e pin-deps -- -E docs && tox -e docs --sitepackages -vv -s false" | |
| ### DRAW PYTHON DEPENDENCY GRAPHS ### | |
| code_visualization: | |
| needs: set_github_outputs | |
| name: Code Visualization of Python Imports as Graphs, in .svg | |
| uses: boromir674/automated-workflows/.github/workflows/[email protected] | |
| with: | |
| run_policy: '${{ needs.set_github_outputs.outputs.PIPE_CODE_VIZ_POLICY }}' | |
| branches: 'main, master, dev' | |
| source_code_targets: 'src' | |
| python_version: '3.10' | |
| artifacts_dir: 'dependency-graphs' | |
| ### Make a Github Release ### | |
| gh_release: | |
| needs: [test_suite, check_which_git_branch_we_are_on] | |
| if: ${{ needs.check_which_git_branch_we_are_on.outputs.AUTOMATED_DEPLOY == 'true' }} | |
| uses: boromir674/automated-workflows/.github/workflows/gh-release.yml@test | |
| name: 'GH Release' | |
| with: | |
| tag: ${{ github.ref_name }} | |
| draft: ${{ needs.check_which_git_branch_we_are_on.outputs.ENVIRONMENT_NAME == 'TEST_DEPLOYMENT' }} | |
| secrets: | |
| # passing the GH_TOKEN PAT, to render in GH as ie: 'boromir674 released this yesterday', instead of 'github-actions released this yesterday' | |
| gh_token: ${{ secrets.GH_TOKEN }} |