diff --git a/.github/template_gitref b/.github/template_gitref index 256a0012e..b5a481aa8 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-251-g45d399d +2021.08.26-276-g45ce134 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..0bb626793 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,42 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_deb' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: "Build" +on: + workflow_call: + +defaults: + run: + working-directory: "pulp_deb" + +jobs: + build: + runs-on: "ubuntu-latest" + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + path: "pulp_deb" + - uses: actions/setup-python@v4 + with: + python-version: "3.8" + - name: Install python dependencies + run: | + echo ::group::PYDEPS + pip install packaging wheel + echo ::endgroup:: + - name: "Build package" + run: "python3 setup.py sdist bdist_wheel --python-tag py3" + - name: 'Upload Package whl' + uses: "actions/upload-artifact@v3" + with: + name: "plugin_package" + path: "pulp_deb/dist/" + if-no-files-found: "error" + retention-days: 5 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 189496aa7..92260fe94 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,10 @@ concurrency: group: ${{ github.ref_name }}-${{ github.workflow }} cancel-in-progress: true +defaults: + run: + working-directory: "pulp_deb" + jobs: ready-to-ship: @@ -21,6 +25,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 + path: "pulp_deb" - uses: actions/setup-python@v4 with: python-version: "3.8" @@ -31,10 +36,6 @@ jobs: env: PY_COLORS: '1' ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} run: sh .github/workflows/scripts/check_commit.sh @@ -42,156 +43,19 @@ jobs: run: python .ci/scripts/check_requirements.py lint: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - uses: actions/setup-python@v4 - with: - python-version: "3.8" - # lint_requirements contains tools needed for flake8, etc. - - name: Install requirements - run: pip3 install -r lint_requirements.txt - - # run black separately from flake8 to get a diff - - name: Run black - run: | - black --version - black --check --diff . - - # Lint code. - - name: Run flake8 - run: flake8 - - - name: Run extra lint checks - run: "[ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh" - - # check for any files unintentionally left out of MANIFEST.in - - name: Check manifest - run: check-manifest - - - name: Check for pulpcore imports outside of pulpcore.plugin - run: sh .ci/scripts/check_pulpcore_imports.sh + uses: "./.github/workflows/lint.yml" - - name: Check for gettext problems - run: sh .ci/scripts/check_gettext.sh - - test: - runs-on: ubuntu-latest - # run only after lint finishes + build: needs: lint - strategy: - fail-fast: false - matrix: - env: - - TEST: pulp - - TEST: docs - - TEST: azure - - TEST: s3 - - TEST: lowerbounds - outputs: - deprecations-pulp: ${{ steps.deprecations.outputs.deprecations-pulp }} - deprecations-azure: ${{ steps.deprecations.outputs.deprecations-azure }} - deprecations-s3: ${{ steps.deprecations.outputs.deprecations-s3 }} - deprecations-lowerbounds: ${{ steps.deprecations.outputs.deprecations-lowerbounds }} - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 1 + uses: "./.github/workflows/build.yml" - - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - - name: Install httpie - run: | - echo ::group::HTTPIE - pip install httpie - echo ::endgroup:: - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV - - - name: Set environment variables - run: | - echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV - - - name: Before Install - run: .github/workflows/scripts/before_install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - name: Install - run: .github/workflows/scripts/install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - name: Before Script - run: .github/workflows/scripts/before_script.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - REDIS_DISABLED: ${{ contains('', matrix.env.TEST) }} - - - name: Setting secrets - if: github.event_name != 'pull_request' - run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - - - name: Script - run: .github/workflows/scripts/script.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - name: Extract Deprecations from Logs - id: deprecations - run: echo deprecations-${{ matrix.env.TEST }}=$(docker logs pulp 2>&1 | grep -i pulpcore.deprecation | base64 -w 0) >> $GITHUB_OUTPUT - - - name: Logs - if: always() - run: | - echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true - docker images || true - docker ps -a || true - docker logs pulp || true - docker exec pulp ls -latr /etc/yum.repos.d/ || true - docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" + test: + needs: build + uses: "./.github/workflows/test.yml" deprecations: + defaults: + run: + working-directory: "." runs-on: ubuntu-latest if: github.base_ref == 'main' needs: test diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml index ea8de8c09..ffd5ba6d8 100644 --- a/.github/workflows/create-branch.yml +++ b/.github/workflows/create-branch.yml @@ -24,7 +24,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - path: pulp_deb + path: "pulp_deb" - uses: actions/setup-python@v4 with: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 000000000..48f274219 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,56 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_deb' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: "Lint" +on: + workflow_call: + +defaults: + run: + working-directory: "pulp_deb" + +jobs: + lint: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + path: "pulp_deb" + + - uses: actions/setup-python@v4 + with: + python-version: "3.8" + + # lint_requirements contains tools needed for flake8, etc. + - name: Install requirements + run: pip3 install -r lint_requirements.txt + + # run black separately from flake8 to get a diff + - name: Run black + run: | + black --version + black --check --diff . + + # Lint code. + - name: Run flake8 + run: flake8 + + - name: Run extra lint checks + run: "[ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh" + + # check for any files unintentionally left out of MANIFEST.in + - name: Check manifest + run: check-manifest + + - name: Check for pulpcore imports outside of pulpcore.plugin + run: sh .ci/scripts/check_pulpcore_imports.sh + + - name: Check for gettext problems + run: sh .ci/scripts/check_gettext.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 530bfc4da..e413d788b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -6,7 +6,7 @@ # For more info visit https://github.com/pulp/plugin_template --- -name: Deb Nightly CI +name: "Deb Nightly CI" on: schedule: # * is a special character in YAML so you have to quote this string @@ -14,156 +14,22 @@ on: - cron: '00 3 * * *' workflow_dispatch: +defaults: + run: + working-directory: "pulp_deb" + concurrency: - group: ${{ github.ref_name }}-${{ github.workflow }} + group: "${{ github.ref_name }}-${{ github.workflow }}" cancel-in-progress: true jobs: - test: - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - env: - - TEST: pulp - - TEST: docs - - TEST: azure - - TEST: s3 - - - TEST: generate-bindings - - TEST: lowerbounds - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - - name: Install httpie - run: | - echo ::group::HTTPIE - pip install httpie - echo ::endgroup:: - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV - - - name: Set environment variables - run: | - echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV - - - name: Before Install - run: .github/workflows/scripts/before_install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - uses: ruby/setup-ruby@v1 - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - with: - ruby-version: "2.6" - - - name: Install - run: .github/workflows/scripts/install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - name: Before Script - run: .github/workflows/scripts/before_script.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - REDIS_DISABLED: ${{ contains('', matrix.env.TEST) }} - - - name: Setting secrets - run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - - - name: Install Python client - run: .github/workflows/scripts/install_python_client.sh - shell: bash - - - name: Install Ruby client - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - run: .github/workflows/scripts/install_ruby_client.sh - shell: bash - - - name: Script - run: .github/workflows/scripts/script.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - name: Upload python client packages - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - uses: actions/upload-artifact@v3 - with: - name: python-client.tar - path: python-client.tar - - - name: Upload python client docs - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - uses: actions/upload-artifact@v3 - with: - name: python-client-docs.tar - path: python-client-docs.tar - - - name: Upload ruby client packages - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - uses: actions/upload-artifact@v3 - with: - name: ruby-client.tar - path: ruby-client.tar - - name: Upload built docs - if: ${{ env.TEST == 'docs' }} - uses: actions/upload-artifact@v3 - with: - name: docs.tar - path: docs/docs.tar + build: + uses: "./.github/workflows/build.yml" - - name: Logs - if: always() - run: | - echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true - docker images || true - docker ps -a || true - docker logs pulp || true - docker exec pulp ls -latr /etc/yum.repos.d/ || true - docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" + test: + runs-on: "ubuntu-latest" + needs: "build" + uses: "./.github/workflows/test.yml" changelog: runs-on: ubuntu-latest @@ -171,6 +37,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 + path: "pulp_deb" - uses: actions/setup-python@v4 with: @@ -194,111 +61,58 @@ jobs: uses: peter-evans/create-pull-request@v4 with: token: ${{ secrets.RELEASE_TOKEN }} - title: 'Update Changelog' - body: '' - branch: 'changelog/update' + title: "Update Changelog" + body: "" + branch: "changelog/update" delete-branch: true + path: "pulp_deb" publish: runs-on: ubuntu-latest needs: test - env: - TEST: publish - steps: - uses: actions/checkout@v4 with: fetch-depth: 1 + path: "pulp_deb" - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 with: - python-version: "3.8" + fetch-depth: 1 + repository: "pulp/pulp-openapi-generator" + path: "pulp-openapi-generator" - - uses: ruby/setup-ruby@v1 + - uses: actions/download-artifact@v3 with: - ruby-version: "2.6" + name: "plugin_package" + path: "pulp_deb/dist/" - - name: Install httpie - run: | - echo ::group::HTTPIE - pip install httpie - echo ::endgroup:: - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV + - uses: actions/setup-python@v4 + with: + python-version: "3.8" - name: Set environment variables run: | echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV - - name: Install python dependencies - run: | - echo ::group::PYDEPS - pip install wheel - echo ::endgroup:: - - - name: Before Install - run: .github/workflows/scripts/before_install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - name: Install - run: .github/workflows/scripts/install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - name: Install Python client - run: .github/workflows/scripts/install_python_client.sh - shell: bash - - - name: Install Ruby client - run: .github/workflows/scripts/install_ruby_client.sh - shell: bash - - - name: Before Script - run: .github/workflows/scripts/before_script.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - REDIS_DISABLED: ${{ contains('', matrix.env.TEST) }} - - - name: Setting secrets - run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - - name: Download built docs uses: actions/download-artifact@v3 with: - name: docs.tar + name: "docs.tar" + path: "pulp_deb" - name: Download Python client docs uses: actions/download-artifact@v3 with: - name: python-client-docs.tar + name: "python-client-docs.tar" + path: "pulp_deb" + + - name: Setting secrets + run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + env: + SECRETS_CONTEXT: ${{ toJson(secrets) }} - name: Publish docs to pulpproject.org run: | diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml new file mode 100644 index 000000000..5fbebd7ee --- /dev/null +++ b/.github/workflows/pr_checks.yml @@ -0,0 +1,62 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_deb' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: Deb PR static checks +on: + pull_request_target: + types: [opened, synchronize, reopened] + +# This workflow runs with elevated permissions. +# Do not even think about running a single bit of code from the PR. +# Static analysis should be fine however. + +concurrency: + group: ${{ github.event.pull_request.number }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + single_commit: + runs-on: ubuntu-latest + name: Label multiple commit PR + permissions: + pull-requests: write + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Commit Count Check + run: | + git fetch origin ${{ github.event.pull_request.head.sha }} + echo "COMMIT_COUNT=$(git log --oneline --no-merges origin/${{ github.base_ref }}..${{ github.event.pull_request.head.sha }} | wc -l)" >> "$GITHUB_ENV" + - uses: actions/github-script@v7 + with: + script: | + const labelName = "multi-commit"; + const { COMMIT_COUNT } = process.env; + + if (COMMIT_COUNT == 1) + { + try { + await github.rest.issues.removeLabel({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName, + }); + } catch(err) { + } + } + else + { + await github.rest.issues.addLabels({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + labels: [labelName], + }); + } diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f0ee5a0da..f560676e1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,18 +12,14 @@ on: inputs: release: description: "Release tag (e.g. 3.2.1)" - before_script: - description: | - Bash code to run before bindings and docs are built. This should only be used when re-running - a workflow to correct some aspect of the docs. e.g.: git checkout origin/3.14 CHANGES.rst - required: false -env: - RELEASE_WORKFLOW: true +defaults: + run: + working-directory: "pulp_deb" jobs: build-artifacts: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" strategy: fail-fast: false @@ -32,6 +28,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 + path: "pulp_deb" - uses: actions/setup-python@v4 with: @@ -56,14 +53,15 @@ jobs: - name: Create the release commit, tag it, create a post-release commit, and build plugin package run: python .github/workflows/scripts/release.py ${{ github.event.inputs.release }} - - name: 'Tar files' - run: tar -cvf pulp_deb.tar . + - name: "Tar files" + run: | + tar -cvf pulp_deb.tar . - - name: 'Upload Artifact' + - name: "Upload Artifact" uses: actions/upload-artifact@v3 with: - name: pulp_deb.tar - path: pulp_deb.tar + name: "pulp_deb.tar" + path: "pulp_deb/pulp_deb.tar" build-bindings-docs: @@ -76,7 +74,14 @@ jobs: steps: - uses: actions/download-artifact@v3 with: - name: pulp_deb.tar + name: "pulp_deb.tar" + path: "pulp_deb/" + + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + repository: "pulp/pulp-openapi-generator" + path: "pulp-openapi-generator" - uses: actions/setup-python@v4 with: @@ -85,24 +90,16 @@ jobs: with: ruby-version: "2.6" - - name: Untar repository + - name: "Untar repository" run: | tar -xf pulp_deb.tar - # update to the branch's latest ci files rather than the ones from the release tag. this is - # helpful when there was a problem with the ci files during the release which needs to be - # fixed after the release tag has been created - - name: Update ci files - run: | - git checkout "origin/${GITHUB_REF##*/}" -- .ci - git checkout "origin/${GITHUB_REF##*/}" -- .github - - name: Install httpie run: | echo ::group::HTTPIE pip install httpie echo ::endgroup:: - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV + echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp_deb/.ci/assets/httpie/" >> $GITHUB_ENV # Building the bindings and docs requires accessing the OpenAPI specs endpoint, so we need to # setup the Pulp instance. @@ -112,10 +109,6 @@ jobs: env: PY_COLORS: '1' ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} @@ -126,41 +119,35 @@ jobs: env: PY_COLORS: '1' ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} shell: bash - - name: Additional before_script - run: ${{ github.event.inputs.before_script }} - shell: bash - - name: Install Python client run: .github/workflows/scripts/install_python_client.sh shell: bash - name: Install Ruby client run: .github/workflows/scripts/install_ruby_client.sh shell: bash - - - name: Upload python client packages + - name: "Upload python client packages" uses: actions/upload-artifact@v3 with: - name: python-client.tar - path: python-client.tar + name: "python-client.tar" + path: "pulp_deb/deb-python-client.tar" + if-no-files-found: "error" - - name: Upload python client docs + - name: "Upload python client docs" uses: actions/upload-artifact@v3 with: - name: python-client-docs.tar - path: python-client-docs.tar - - name: Upload ruby client packages + name: "python-client-docs.tar" + path: "pulp_deb/deb-python-client-docs.tar" + if-no-files-found: "error" + - name: "Upload ruby client packages" uses: actions/upload-artifact@v3 with: - name: ruby-client.tar - path: ruby-client.tar + name: "ruby-client.tar" + path: "pulp_deb/deb-ruby-client.tar" + if-no-files-found: "error" - name: Build docs run: | export DJANGO_SETTINGS_MODULE=pulpcore.app.settings @@ -168,11 +155,11 @@ jobs: make -C docs/ PULP_URL="https://pulp" diagrams html tar -cvf docs/docs.tar docs/_build - - name: Upload built docs + - name: "Upload built docs" uses: actions/upload-artifact@v3 with: - name: docs.tar - path: docs/docs.tar + name: "docs.tar" + path: "pulp_deb/docs/docs.tar" - name: Logs if: always() @@ -198,7 +185,8 @@ jobs: steps: - uses: actions/download-artifact@v3 with: - name: pulp_deb.tar + name: "pulp_deb.tar" + path: "pulp_deb/" - uses: actions/setup-python@v4 with: @@ -217,14 +205,6 @@ jobs: run: | tar -xf pulp_deb.tar - # update to the branch's latest ci files rather than the ones from the release tag. this is - # helpful when there was a problem with the ci files during the release which needs to be - # fixed after the release tag has been created - - name: Update ci files - run: | - git checkout "origin/${GITHUB_REF##*/}" -- .ci - git checkout "origin/${GITHUB_REF##*/}" -- .github - - name: Setting secrets run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" env: @@ -233,20 +213,22 @@ jobs: - name: Install python dependencies run: | echo ::group::PYDEPS - pip install gitpython requests packaging~=21.3 tweepy + pip install gitpython requests packaging~=21.3 echo ::endgroup:: - name: Push branch and tag to GitHub run: bash .github/workflows/scripts/push_branch_and_tag_to_github.sh ${{ github.event.inputs.release }} - - name: Download built docs + - name: "Download built docs" uses: actions/download-artifact@v3 with: - name: docs.tar + name: "docs.tar" + path: "pulp_deb/" - - name: Download Python client docs + - name: "Download Python client docs" uses: actions/download-artifact@v3 with: - name: python-client-docs.tar + name: "python-client-docs.tar" + path: "pulp_deb/" - name: Publish docs to pulpproject.org run: | @@ -254,37 +236,44 @@ jobs: .github/workflows/scripts/publish_docs.sh tag ${{ github.event.inputs.release }} - name: Deploy plugin to pypi run: bash .github/workflows/scripts/publish_plugin_pypi.sh ${{ github.event.inputs.release }} - - name: Download Python client + - name: "Download Python client" uses: actions/download-artifact@v3 with: - name: python-client.tar + name: "python-client.tar" + path: "pulp_deb/" - - name: Untar python client packages - run: tar -xvf python-client.tar + - name: "Untar python client packages" + run: | + tar -xvf deb-python-client.tar - - name: Publish client to pypi - run: bash .github/workflows/scripts/publish_client_pypi.sh - - name: Download Ruby client + - name: "Publish client to pypi" + run: | + bash .github/workflows/scripts/publish_client_pypi.sh + - name: "Download Ruby client" uses: actions/download-artifact@v3 with: - name: ruby-client.tar + name: "ruby-client.tar" + path: "pulp_deb/" - - name: Untar Ruby client packages - run: tar -xvf ruby-client.tar - - - name: Publish client to rubygems - run: bash .github/workflows/scripts/publish_client_gem.sh - - - name: Update GitHub - continue-on-error: true + - name: "Untar Ruby client packages" run: | - set -euv - export COMMIT_MSG=$(git log --format=%B --no-merges -1) - export GH_ISSUES=$(echo $COMMIT_MSG | grep -o "GH Issues: .*" | awk '{print $3}') - pip install pygithub + tar -xvf deb-ruby-client.tar - echo "GH Issues $GH_ISSUES" - python .ci/scripts/update_github.py + - name: "Publish client to rubygems" + run: | + bash .github/workflows/scripts/publish_client_gem.sh - name: Create release on GitHub - run: bash .github/workflows/scripts/create_release_from_tag.sh ${{ github.event.inputs.release }} + uses: actions/github-script@v7 + env: + TAG_NAME: ${{ inputs.release }} + with: + const { TAG_NAME } = process.env; + + script: | + await github.rest.repos.createRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + tag_name: TAG_NAME, + make_latest: "legacy", + }); diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index 8cd9eb0b9..dcd23456c 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -55,11 +55,6 @@ then echo $COMMIT_MSG | sed -n -e 's/.*CI Base Image:\s*\([-_/[:alnum:]]*:[-_[:alnum:]]*\).*/ci_base: "\1"/p' >> .ci/ansible/vars/main.yaml fi - -cd .. - -git clone --depth=1 https://github.com/pulp/pulp-openapi-generator.git - # Intall requirements for ansible playbooks pip install docker netaddr boto3 ansible @@ -73,8 +68,6 @@ then exit $s fi -cd pulp_deb - if [[ "$TEST" = "lowerbounds" ]]; then python3 .ci/scripts/calc_deps_lowerbounds.py > lowerbounds_requirements.txt mv lowerbounds_requirements.txt requirements.txt diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 95df78908..ab8ea0a64 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -15,6 +15,9 @@ set -euv source .github/workflows/scripts/utils.sh +PLUGIN_VERSION="$(sed -n -e 's/^\s*current_version\s*=\s*//p' .bumpversion.cfg | python -c 'from packaging.version import Version; print(Version(input()))')" +PLUGIN_NAME="./pulp_deb/dist/pulp_deb-${PLUGIN_VERSION}-py3-none-any.whl" + export PULP_API_ROOT="/pulp/" PIP_REQUIREMENTS=("pulp-cli") @@ -29,11 +32,6 @@ pip install ${PIP_REQUIREMENTS[*]} cd .ci/ansible/ -if [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then - PLUGIN_NAME=./pulp_deb/dist/pulp_deb-$PLUGIN_VERSION-py3-none-any.whl -else - PLUGIN_NAME=./pulp_deb -fi cat >> vars/main.yaml << VARSYAML image: name: pulp diff --git a/.github/workflows/scripts/install_python_client.sh b/.github/workflows/scripts/install_python_client.sh index f9a007778..dcf0b4a7b 100755 --- a/.github/workflows/scripts/install_python_client.sh +++ b/.github/workflows/scripts/install_python_client.sh @@ -9,45 +9,36 @@ set -mveuo pipefail -export PULP_URL="${PULP_URL:-https://pulp}" - # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. +source .github/workflows/scripts/utils.sh + +export PULP_URL="${PULP_URL:-https://pulp}" + pip install twine wheel -export REPORTED_VERSION=$(http $PULP_URL/pulp/api/v3/status/ | jq --arg plugin deb --arg legacy_plugin pulp_deb -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') -export DESCRIPTION="$(git describe --all --exact-match `git rev-parse HEAD`)" -if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then - export VERSION=${REPORTED_VERSION} -else - export EPOCH="$(date +%s)" - export VERSION=${REPORTED_VERSION}${EPOCH} -fi - -export response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulp-deb-client/$VERSION/) - -if [ "$response" == "200" ]; -then - echo "pulp_deb client $VERSION has already been released. Installing from PyPI." - docker exec pulp pip3 install pulp-deb-client==$VERSION - mkdir -p dist - tar cvf python-client.tar ./dist - exit -fi - -cd ../pulp-openapi-generator +REPORTED_STATUS="$(pulp status)" +REPORTED_VERSION="$(echo "$REPORTED_STATUS" | jq --arg plugin "deb" -r '.versions[] | select(.component == $plugin) | .version')" +VERSION="$(echo "$REPORTED_VERSION" | python -c 'from packaging.version import Version; print(Version(input()))')" + +pushd ../pulp-openapi-generator rm -rf pulp_deb-client -./generate.sh pulp_deb python $VERSION -cd pulp_deb-client +./generate.sh pulp_deb python "$VERSION" +pushd pulp_deb-client python setup.py sdist bdist_wheel --python-tag py3 -find . -name "*.whl" -exec docker exec pulp pip3 install /root/pulp-openapi-generator/pulp_deb-client/{} \; -tar cvf ../../pulp_deb/python-client.tar ./dist + +twine check "dist/pulp_deb_client-$VERSION-py3-none-any.whl" || exit 1 +twine check "dist/pulp_deb-client-$VERSION.tar.gz" || exit 1 + +cmd_prefix pip3 install "/root/pulp-openapi-generator/pulp_deb-client/dist/pulp_deb_client-${VERSION}-py3-none-any.whl" +tar cvf ../../pulp_deb/deb-python-client.tar ./dist find ./docs/* -exec sed -i 's/Back to README/Back to HOME/g' {} \; find ./docs/* -exec sed -i 's/README//g' {} \; cp README.md docs/index.md sed -i 's/docs\///g' docs/index.md find ./docs/* -exec sed -i 's/\.md//g' {} \; -tar cvf ../../pulp_deb/python-client-docs.tar ./docs -exit $? +tar cvf ../../pulp_deb/deb-python-client-docs.tar ./docs +popd +popd diff --git a/.github/workflows/scripts/install_ruby_client.sh b/.github/workflows/scripts/install_ruby_client.sh index ac1b37b96..b95a5b67b 100755 --- a/.github/workflows/scripts/install_ruby_client.sh +++ b/.github/workflows/scripts/install_ruby_client.sh @@ -7,37 +7,26 @@ # # For more info visit https://github.com/pulp/plugin_template -set -euv +set -mveuo pipefail # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. +source .github/workflows/scripts/utils.sh + export PULP_URL="${PULP_URL:-https://pulp}" -export REPORTED_VERSION=$(http $PULP_URL/pulp/api/v3/status/ | jq --arg plugin deb --arg legacy_plugin pulp_deb -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') -export DESCRIPTION="$(git describe --all --exact-match `git rev-parse HEAD`)" -if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then - export VERSION=${REPORTED_VERSION} -else - export EPOCH="$(date +%s)" - export VERSION=${REPORTED_VERSION}${EPOCH} -fi - -export response=$(curl --write-out %{http_code} --silent --output /dev/null https://rubygems.org/gems/pulp_deb_client/versions/$VERSION) - -if [ "$response" == "200" ]; -then - echo "pulp_deb client $VERSION has already been released. Installing from RubyGems.org." - gem install pulp_deb_client -v $VERSION - touch pulp_deb_client-$VERSION.gem - tar cvf ruby-client.tar ./pulp_deb_client-$VERSION.gem - exit -fi - -cd ../pulp-openapi-generator + +REPORTED_STATUS="$(pulp status)" +REPORTED_VERSION="$(echo "$REPORTED_STATUS" | jq --arg plugin "deb" -r '.versions[] | select(.component == $plugin) | .version')" +VERSION="$(echo "$REPORTED_VERSION" | python -c 'from packaging.version import Version; print(Version(input()))')" + +pushd ../pulp-openapi-generator rm -rf pulp_deb-client -./generate.sh pulp_deb ruby $VERSION -cd pulp_deb-client +./generate.sh pulp_deb ruby "$VERSION" +pushd pulp_deb-client gem build pulp_deb_client -gem install --both ./pulp_deb_client-$VERSION.gem -tar cvf ../../pulp_deb/ruby-client.tar ./pulp_deb_client-$VERSION.gem +gem install --both "./pulp_deb_client-$VERSION.gem" +tar cvf ../../pulp_deb/deb-ruby-client.tar "./pulp_deb_client-$VERSION.gem" +popd +popd diff --git a/.github/workflows/scripts/publish_client_pypi.sh b/.github/workflows/scripts/publish_client_pypi.sh index b6819acc5..0c187e977 100755 --- a/.github/workflows/scripts/publish_client_pypi.sh +++ b/.github/workflows/scripts/publish_client_pypi.sh @@ -10,28 +10,26 @@ set -euv # make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. +cd "$(dirname "$(realpath -e "$0")")/../../.." pip install twine -export VERSION=$(ls dist | sed -rn 's/pulp_deb-client-(.*)\.tar.gz/\1/p') +VERSION="$(ls dist | sed -rn 's/pulp_deb-client-(.*)\.tar.gz/\1/p')" if [[ -z "$VERSION" ]]; then echo "No client package found." exit fi -export response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulp-deb-client/$VERSION/) +RESPONSE="$(curl --write-out '%{http_code}' --silent --output /dev/null "https://pypi.org/project/pulp-deb-client/$VERSION/")" -if [ "$response" == "200" ]; +if [ "$RESPONSE" == "200" ]; then echo "pulp_deb client $VERSION has already been released. Skipping." exit fi -twine check dist/pulp_deb_client-$VERSION-py3-none-any.whl || exit 1 -twine check dist/pulp_deb-client-$VERSION.tar.gz || exit 1 -twine upload dist/pulp_deb_client-$VERSION-py3-none-any.whl -u pulp -p $PYPI_PASSWORD -twine upload dist/pulp_deb-client-$VERSION.tar.gz -u pulp -p $PYPI_PASSWORD - -exit $? +twine upload -u pulp -p "$PYPI_PASSWORD" \ +"dist/pulp_deb_client-$VERSION-py3-none-any.whl" \ +"dist/pulp_deb-client-$VERSION.tar.gz" \ +; diff --git a/.github/workflows/scripts/publish_docs.sh b/.github/workflows/scripts/publish_docs.sh index abe7ee62a..aca28035a 100755 --- a/.github/workflows/scripts/publish_docs.sh +++ b/.github/workflows/scripts/publish_docs.sh @@ -10,7 +10,7 @@ set -euv # make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. +cd "$(dirname "$(realpath -e "$0")")/../../.." mkdir ~/.ssh touch ~/.ssh/pulp-infra @@ -27,10 +27,11 @@ export DJANGO_SETTINGS_MODULE=pulpcore.app.settings export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py export WORKSPACE=$PWD -eval "$(ssh-agent -s)" #start the ssh agent +# start the ssh agent +eval "$(ssh-agent -s)" ssh-add ~/.ssh/pulp-infra -python3 .github/workflows/scripts/docs-publisher.py --build-type $1 --branch $2 +python3 .github/workflows/scripts/docs-publisher.py --build-type "$1" --branch "$2" if [[ "$GITHUB_WORKFLOW" == "Deb changelog update" ]]; then # Do not build bindings docs on changelog update @@ -39,9 +40,9 @@ fi pip install mkdocs pymdown-extensions "Jinja2<3.1" -mkdir -p ../bindings -tar -xvf python-client-docs.tar --directory ../bindings -cd ../bindings +mkdir -p ../deb-bindings +tar -xvf deb-python-client-docs.tar --directory ../deb-bindings +pushd ../deb-bindings cat >> mkdocs.yml << DOCSYAML --- site_name: PulpDeb Client @@ -61,3 +62,4 @@ rsync -avzh site/ doc_builder_pulp_deb@docs.pulpproject.org:/var/www/docs.pulppr # publish to docs.pulpproject.org/pulp_deb_client/en/{release} rsync -avzh site/ doc_builder_pulp_deb@docs.pulpproject.org:/var/www/docs.pulpproject.org/pulp_deb_client/en/"$2" +popd diff --git a/.github/workflows/scripts/push_branch_and_tag_to_github.sh b/.github/workflows/scripts/push_branch_and_tag_to_github.sh index 2f13c78d8..09c20228d 100755 --- a/.github/workflows/scripts/push_branch_and_tag_to_github.sh +++ b/.github/workflows/scripts/push_branch_and_tag_to_github.sh @@ -1,21 +1,16 @@ #!/bin/sh -set -e -BRANCH_NAME=$(echo $GITHUB_REF | sed -rn 's/refs\/heads\/(.*)/\1/p') +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_deb' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template -ref_string=$(git show-ref --tags | grep refs/tags/$1) +set -eu -SHA=${ref_string:0:40} +BRANCH_NAME="$(echo "$GITHUB_REF" | sed -rn 's/refs\/heads\/(.*)/\1/p')" -remote_repo=https://pulpbot:${RELEASE_TOKEN}@github.com/${GITHUB_REPOSITORY}.git +remote_repo="https://pulpbot:${RELEASE_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" -git push "${remote_repo}" $BRANCH_NAME - -curl -s -X POST https://api.github.com/repos/$GITHUB_REPOSITORY/git/refs \ --H "Authorization: token $RELEASE_TOKEN" \ --d @- << EOF -{ - "ref": "refs/tags/$1", - "sha": "$SHA" -} -EOF +git push "${remote_repo}" "$BRANCH_NAME" "$1" diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 18939ae28..4db3b23cb 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -12,7 +12,6 @@ set -mveuo pipefail # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. -REPO_ROOT="$PWD" source .github/workflows/scripts/utils.sh @@ -38,64 +37,57 @@ if [[ "$TEST" = "docs" ]]; then tar -cvf docs.tar ./_build cd .. - if [ -f $POST_DOCS_TEST ]; then - source $POST_DOCS_TEST + if [ -f "$POST_DOCS_TEST" ]; then + source "$POST_DOCS_TEST" fi exit fi REPORTED_STATUS="$(pulp status)" -if [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then - REPORTED_VERSION="$(echo $REPORTED_STATUS | jq --arg plugin deb --arg legacy_plugin pulp_deb -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version')" - response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulp-deb/$REPORTED_VERSION/) - if [ "$response" == "200" ]; - then - echo "pulp_deb $REPORTED_VERSION has already been released. Skipping running tests." - exit - fi -fi - echo "machine pulp login admin password password " | cmd_user_stdin_prefix bash -c "cat >> ~pulp/.netrc" # Some commands like ansible-galaxy specifically require 600 -cmd_user_stdin_prefix bash -c "chmod 600 ~pulp/.netrc" +cmd_prefix bash -c "chmod 600 ~pulp/.netrc" -cd ../pulp-openapi-generator -if [ "$(echo "$REPORTED_STATUS" | jq -r '.versions[0].package')" = "null" ] +# Infer the client name from the package name by replacing "-" with "_". +# Use the component to infer the package name on older versions of pulpcore. + +if [ "$(echo "$REPORTED_STATUS" | jq -r '.domain_enabled')" = "true" ] then - # We are on an old version of pulpcore without package in the status report - for app_label in $(echo "$REPORTED_STATUS" | jq -r '.versions[].component') + # Workaround: Domains are not supported by the published bindings. + # Generate new bindings for all packages. + pushd ../pulp-openapi-generator + for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[]|(.package // ("pulp_" + .component)|sub("pulp_core"; "pulpcore"))|sub("-"; "_")') do - if [ "$app_label" = "core" ] - then - item=pulpcore - else - item="pulp_${app_label}" - fi ./generate.sh "${item}" python cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" sudo rm -rf "./${item}-client" done + popd else - for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[].package|sub("-"; "_")') + # Sadly: Different pulpcore-versions aren't either... + pushd ../pulp-openapi-generator + for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.component!="deb")|(.package // ("pulp_" + .component)|sub("pulp_core"; "pulpcore"))|sub("-"; "_")') do ./generate.sh "${item}" python cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" sudo rm -rf "./${item}-client" done + popd fi -cd $REPO_ROOT - -cat unittest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" -cat functest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt" -cmd_prefix pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt +# At this point, this is a safeguard only, so let's not make too much fuzz about the old status format. +echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.package)|(.package|sub("_"; "-")) + "-client==" + .version' > bindings_requirements.txt +cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" < unittest_requirements.txt +cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt" < functest_requirements.txt +cmd_stdin_prefix bash -c "cat > /tmp/bindings_requirements.txt" < bindings_requirements.txt +cmd_prefix pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt -r /tmp/bindings_requirements.txt CERTIFI=$(cmd_prefix python3 -c 'import certifi; print(certifi.where())') -cmd_prefix bash -c "cat /etc/pulp/certs/pulp_webserver.crt | tee -a "$CERTIFI" > /dev/null" +cmd_prefix bash -c "cat /etc/pulp/certs/pulp_webserver.crt >> '$CERTIFI'" # check for any uncommitted migrations echo "Checking for uncommitted migrations..." @@ -109,29 +101,25 @@ if [[ "$TEST" == "performance" ]]; then if [[ -z ${PERFORMANCE_TEST+x} ]]; then cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_deb.tests.performance" else - cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_deb.tests.performance.test_$PERFORMANCE_TEST" + cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_deb.tests.performance.test_${PERFORMANCE_TEST}" fi exit fi -if [ -f $FUNC_TEST_SCRIPT ]; then - source $FUNC_TEST_SCRIPT +if [ -f "$FUNC_TEST_SCRIPT" ]; then + source "$FUNC_TEST_SCRIPT" else - - if [[ "$GITHUB_WORKFLOW" == "Deb Nightly CI/CD" ]] || [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then + if [[ "$GITHUB_WORKFLOW" == "Deb Nightly CI/CD" ]] + then cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_deb.tests.functional -m parallel -n 8 --nightly" cmd_user_prefix bash -c "pytest -v -r sx --color=yes --pyargs pulp_deb.tests.functional -m 'not parallel' --nightly" - else cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_deb.tests.functional -m parallel -n 8" cmd_user_prefix bash -c "pytest -v -r sx --color=yes --pyargs pulp_deb.tests.functional -m 'not parallel'" - - fi - fi -if [ -f $POST_SCRIPT ]; then - source $POST_SCRIPT +if [ -f "$POST_SCRIPT" ]; then + source "$POST_SCRIPT" fi diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..80d0201f4 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,161 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_deb' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: "Test" +on: + workflow_call: + +defaults: + run: + working-directory: "pulp_deb" + +jobs: + test: + runs-on: "ubuntu-latest" + strategy: + fail-fast: false + matrix: + env: + - TEST: pulp + - TEST: docs + - TEST: azure + - TEST: s3 + - TEST: lowerbounds + outputs: + deprecations-pulp: ${{ steps.deprecations.outputs.deprecations-pulp }} + deprecations-azure: ${{ steps.deprecations.outputs.deprecations-azure }} + deprecations-s3: ${{ steps.deprecations.outputs.deprecations-s3 }} + deprecations-lowerbounds: ${{ steps.deprecations.outputs.deprecations-lowerbounds }} + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + path: "pulp_deb" + + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + repository: "pulp/pulp-openapi-generator" + path: "pulp-openapi-generator" + + - uses: actions/setup-python@v4 + with: + python-version: "3.8" + + - uses: "actions/download-artifact@v3" + with: + name: "plugin_package" + path: "pulp_deb/dist/" + + - name: Install httpie + run: | + echo ::group::HTTPIE + pip install httpie + echo ::endgroup:: + echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp_deb/.ci/assets/httpie/" >> $GITHUB_ENV + + - name: Set environment variables + run: | + echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV + + - name: Before Install + run: .github/workflows/scripts/before_install.sh + shell: bash + env: + PY_COLORS: '1' + ANSIBLE_FORCE_COLOR: '1' + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} + - uses: ruby/setup-ruby@v1 + if: ${{ env.TEST == 'pulp' }} + with: + ruby-version: "2.6" + + - name: Install + run: .github/workflows/scripts/install.sh + shell: bash + env: + PY_COLORS: '1' + ANSIBLE_FORCE_COLOR: '1' + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} + + - name: Before Script + run: .github/workflows/scripts/before_script.sh + shell: bash + env: + PY_COLORS: '1' + ANSIBLE_FORCE_COLOR: '1' + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} + REDIS_DISABLED: ${{ contains('', matrix.env.TEST) }} + + - name: Install Python client + run: .github/workflows/scripts/install_python_client.sh + shell: bash + - name: Install Ruby client + if: ${{ env.TEST == 'pulp' }} + run: .github/workflows/scripts/install_ruby_client.sh + shell: bash + + - name: Script + run: .github/workflows/scripts/script.sh + shell: bash + env: + PY_COLORS: '1' + ANSIBLE_FORCE_COLOR: '1' + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} + + - name: Extract Deprecations from Logs + id: deprecations + run: echo deprecations-${{ matrix.env.TEST }}=$(docker logs pulp 2>&1 | grep -i pulpcore.deprecation | base64 -w 0) >> $GITHUB_OUTPUT + - name: Upload python client packages + if: ${{ env.TEST == 'pulp' }} + uses: actions/upload-artifact@v3 + with: + name: "python-client.tar" + path: "pulp_deb/deb-python-client.tar" + if-no-files-found: "error" + retention-days: 5 + + - name: Upload python client docs + if: ${{ env.TEST == 'pulp' }} + uses: actions/upload-artifact@v3 + with: + name: "python-client-docs.tar" + path: "pulp_deb/deb-python-client-docs.tar" + if-no-files-found: "error" + retention-days: 5 + - name: Upload Ruby client + if: ${{ env.TEST == 'pulp' }} + uses: actions/upload-artifact@v3 + with: + name: "ruby-client.tar" + path: "pulp_deb/deb-ruby-client.tar" + if-no-files-found: "error" + retention-days: 5 + - name: Upload built docs + if: ${{ env.TEST == 'docs' }} + uses: actions/upload-artifact@v3 + with: + name: "docs.tar" + path: "pulp_deb/docs/docs.tar" + + - name: Logs + if: always() + run: | + echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" + http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true + docker images || true + docker ps -a || true + docker logs pulp || true + docker exec pulp ls -latr /etc/yum.repos.d/ || true + docker exec pulp cat /etc/yum.repos.d/* || true + docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" diff --git a/template_config.yml b/template_config.yml index 7258aa76e..dd3337ff8 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,7 +1,7 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. -# generated with plugin_template@2021.08.26-248-g4bfc3e1 +# generated with plugin_template@2021.08.26-276-g45ce134 additional_repos: [] api_root: /pulp/ @@ -38,15 +38,11 @@ lint_requirements: true noissue_marker: '[noissue]' parallel_test_workers: 8 plugin_app_label: deb -plugin_camel: PulpDeb -plugin_camel_short: Deb -plugin_caps: PULP_DEB -plugin_caps_short: DEB -plugin_dash: pulp-deb -plugin_dash_short: deb plugin_default_branch: main plugin_name: pulp_deb -plugin_snake: pulp_deb +plugins: +- app_label: deb + name: pulp_deb post_job_template: null pre_job_template: null publish_docs_to_pulpprojectdotorg: true @@ -74,8 +70,6 @@ pypi_username: pulp python_version: '3.8' release_email: pulp-infra@redhat.com release_user: pulpbot -run_pulpcore_tests_for_plugins: false -single_commit_check: false stalebot: true stalebot_days_until_close: 30 stalebot_days_until_stale: 90 @@ -89,6 +83,5 @@ test_lowerbounds: true test_performance: false test_reroute: true test_s3: true -update_github: true use_issue_template: true