diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 29c13f25..1e8d3dad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,6 +56,9 @@ jobs: test: needs: "build" uses: "./.github/workflows/test.yml" + with: + matrix_env: | + [{"TEST": "pulp"}, {"TEST": "azure"}, {"TEST": "gcp"}, {"TEST": "s3"}, {"TEST": "lowerbounds"}] deprecations: runs-on: "ubuntu-latest" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 65ab3035..ac345aa3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,6 +9,10 @@ name: "Test" on: workflow_call: + inputs: + matrix_env: + required: true + type: string defaults: run: @@ -20,12 +24,7 @@ jobs: strategy: fail-fast: false matrix: - env: - - TEST: pulp - - TEST: azure - - TEST: gcp - - TEST: s3 - - TEST: lowerbounds + env: ${{ fromJSON(inputs.matrix_env) }} steps: - uses: "actions/checkout@v4" @@ -87,6 +86,10 @@ jobs: - name: "Set environment variables" run: | echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV + if [ "${{ matrix.env.TEST }}" = "performance" ] + then + echo "PERFORMANCE_TEST=${{ matrix.env.PERFORMANCE_TEST }}" >> $GITHUB_ENV + fi - name: "Before Install" run: | @@ -97,10 +100,6 @@ jobs: ANSIBLE_FORCE_COLOR: "1" GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - - uses: ruby/setup-ruby@v1 - if: ${{ env.TEST == 'pulp' }} - with: - ruby-version: "2.6" - name: "Install" run: | diff --git a/plugin-template b/plugin-template index 875f1993..76eb8ea8 100755 --- a/plugin-template +++ b/plugin-template @@ -328,6 +328,7 @@ def write_template_section(config, name, plugin_root_dir, verbose=False): env.filters["dash"] = utils.to_dash env.filters["snake"] = utils.to_snake env.filters["to_yaml"] = utils.to_nice_yaml + env.filters["from_yaml"] = utils.from_yaml env.filters["shquote"] = shlex.quote files_templated = 0 diff --git a/templates/github/.github/workflows/ci.yml.j2 b/templates/github/.github/workflows/ci.yml.j2 index f2a54f49..2d0d2f47 100644 --- a/templates/github/.github/workflows/ci.yml.j2 +++ b/templates/github/.github/workflows/ci.yml.j2 @@ -1,9 +1,10 @@ {% include 'header.j2' %} {% from 'macros.j2' import - set_env_vars, checkout, - setup_python, install_python_deps, + matrix_env, + set_env_vars, + setup_python, with context %} --- name: "{{ plugin_app_label | camel }} CI" @@ -56,6 +57,9 @@ jobs: test: needs: "build" uses: "./.github/workflows/test.yml" + with: + matrix_env: | + {{ matrix_env() | from_yaml | tojson }} {%- if test_deprecations %} deprecations: diff --git a/templates/github/.github/workflows/nightly.yml.j2 b/templates/github/.github/workflows/nightly.yml.j2 index 64402928..9ab78f16 100644 --- a/templates/github/.github/workflows/nightly.yml.j2 +++ b/templates/github/.github/workflows/nightly.yml.j2 @@ -1,13 +1,13 @@ {% include 'header.j2' %} {% from 'macros.j2' import checkout, - setup_python, - setup_ruby, - install_python_deps, - setup_env, + configure_git, display_logs, + install_python_deps, + matrix_env, run_script, - configure_git, + setup_env, + setup_python, with context %} --- name: "{{ plugin_app_label | camel }} Nightly CI" @@ -33,6 +33,9 @@ jobs: test: needs: "build" uses: "./.github/workflows/test.yml" + with: + matrix_env: | + {{ matrix_env(performance=true) | from_yaml | tojson }} changelog: runs-on: ubuntu-latest @@ -57,75 +60,4 @@ jobs: branch: "changelog/update" delete-branch: true path: "{{ plugin_name }}" - -{%- if test_performance %} - - performance: - runs-on: "ubuntu-latest" - needs: "test" - - strategy: - fail-fast: false - matrix: - env: - {%- if test_performance is iterable and test_performance|length > 1 %} - {%- for test in test_performance %} - - TEST: performance - PERFORMANCE_TEST: {{test}} - {%- endfor %} - {%- else %} - - TEST: performance - {%- endif %} - - steps: - {{ checkout(path=plugin_name) | indent(6) }} - - {{ checkout(repository="pulp/pulp-openapi-generator", path="pulp-openapi-generator") | indent(6) }} - - {{ setup_python() | indent(6) }} - - - name: "Download plugin package" - uses: "actions/download-artifact@v4" - with: - name: "plugin_package" - path: "{{ plugin_name }}/dist/" - - - name: "Download API specs" - uses: "actions/download-artifact@v4" - with: - name: "api_spec" - path: "{{ plugin_name }}/" - {%- if plugins %} - - - name: "Download client packages" - uses: "actions/download-artifact@v4" - with: - name: "python-client.tar" - path: "{{ plugin_name }}" - - - name: "Unpack client packages" - working-directory: "pulp-openapi-generator" - run: | - {%- for plugin in plugins %} - mkdir -p "{{ plugin.name | snake }}-client" - pushd "{{ plugin.name | snake }}-client" - tar xvf "../../{{ plugin_name }}/{{ plugin.app_label }}-python-client.tar" - popd - {%- endfor %} - {%- endif %} - - {{ install_python_deps(["towncrier", "twine", "wheel", "httpie", "docker", "netaddr", "boto3", "ansible~=10.3.0", "mkdocs", "jq", "jsonpatch", "bump-my-version"]) | indent(6) }} - - {{ setup_env(test="performance") | indent(6) }} - - {{ run_script(name="Before Install", file="before_install.sh") | indent(6) }} - - {{ run_script(name="Install", file="install.sh") | indent(6) }} - - {{ run_script(name="Before Script", file="before_script.sh", extra_env={"REDIS_DISABLED": "${{ contains('%s', matrix.env.TEST) }}" % (disabled_redis_runners | default(["TILT"]) | join(' '))}) | indent(6) }} - - {{ run_script(name="Script", file="script.sh") | indent(6) }} - - {{ display_logs() | indent(6) }} -{%- endif %} ... diff --git a/templates/github/.github/workflows/test.yml.j2 b/templates/github/.github/workflows/test.yml.j2 index 75f78197..2cc46098 100644 --- a/templates/github/.github/workflows/test.yml.j2 +++ b/templates/github/.github/workflows/test.yml.j2 @@ -1,17 +1,20 @@ {% include 'header.j2' %} {% from 'macros.j2' import checkout, - setup_python, - setup_ruby, - install_python_deps, - setup_env, display_logs, + install_python_deps, run_script, + setup_env, + setup_python, with context %} --- name: "Test" on: workflow_call: + inputs: + matrix_env: + required: true + type: string defaults: run: @@ -23,20 +26,9 @@ jobs: strategy: fail-fast: false matrix: - env: - - TEST: pulp - {%- if test_azure %} - - TEST: azure - {%- endif %} - {%- if test_gcp %} - - TEST: gcp - {%- endif %} - {%- if test_s3 %} - - TEST: s3 - {%- endif %} - {%- if test_lowerbounds %} - - TEST: lowerbounds - {%- endif %} + {%- raw %} + env: ${{ fromJSON(inputs.matrix_env) }} + {%- endraw %} steps: {{ checkout(path=plugin_name) | indent(6) }} diff --git a/templates/macros.j2 b/templates/macros.j2 index 18c7ac48..a67c74d1 100644 --- a/templates/macros.j2 +++ b/templates/macros.j2 @@ -67,10 +67,13 @@ GITHUB_CONTEXT: "{{ '${{ github.event.pull_request.commits_url }}' }}" run: | echo "TEST={{ "${{ matrix.env.TEST }}" }}" >> $GITHUB_ENV - {%- if test_performance and test == "performance" %} {%- if test_performance is iterable and test_performance|length > 1 %} - echo "PERFORMANCE_TEST={{ "${{ matrix.env.PERFORMANCE_TEST }}" }}" >> $GITHUB_ENV - {%- endif %} + {%- raw %} + if [ "${{ matrix.env.TEST }}" = "performance" ] + then + echo "PERFORMANCE_TEST=${{ matrix.env.PERFORMANCE_TEST }}" >> $GITHUB_ENV + fi + {%- endraw %} {%- endif %} {%- endmacro -%} @@ -129,3 +132,32 @@ GITHUB_CONTEXT: "{{ '${{ github.event.pull_request.commits_url }}' }}" git config --global user.name '{{ release_user }}' git config --global user.email '{{ release_email }}' {%- endmacro -%} + + +{%- macro matrix_env(performance=false) -%} +--- +- TEST: "pulp" +{%- if test_azure %} +- TEST: "azure" +{%- endif %} +{%- if test_gcp %} +- TEST: "gcp" +{%- endif %} +{%- if test_s3 %} +- TEST: "s3" +{%- endif %} +{%- if test_lowerbounds %} +- TEST: "lowerbounds" +{%- endif %} +{%- if performance and test_performance %} +{%- if test_performance is iterable and test_performance|length > 1 %} +{%- for test in test_performance %} +- TEST: "performance" + PERFORMANCE_TEST: "{{test}}" +{%- endfor %} +{%- else %} +- TEST: "performance" +{%- endif %} +{%- endif %} +... +{%- endmacro -%} diff --git a/utils.py b/utils.py index 9c387568..6afae8d9 100644 --- a/utils.py +++ b/utils.py @@ -55,6 +55,11 @@ def to_nice_yaml(data): return yaml.dump(data, indent=2, allow_unicode=True, default_flow_style=False) +def from_yaml(data): + """Jinja filter to convert yaml data into a variable.""" + return yaml.safe_load(data) + + # Information gathering