Add downstream test job to CI (#855)

Fixes #856

This makes ci-mgmt run acceptance-tests on
[pulumi-xyz](https://github.com/pulumi/pulumi-xyz) and verifies that
they all succeed.

It should prevent errors like #837

.github/workflows/pull-request.yml

@@ -4,6 +4,9 @@ on:
       - master
 env:
   GITHUB_TOKEN: ${{ secrets.PULUMI_BOT_TOKEN }}
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 jobs:
   lint:
     name: Run actionlint and shellcheck
@@ -47,3 +50,37 @@ jobs:
         run: cd provider-ci && make test-workflow-generation
       - name: Check worktree clean
         run: ./ci-scripts/ci/check-worktree-is-clean
+
+  deploy:
+    uses: ./.github/workflows/update-workflows.yml
+    secrets: inherit
+    with:
+      bridged: true
+      provider_name: xyz
+      automerge: false
+      downstream_test: true
+      skip_closing_prs: true
+      caller_workflow: "pull-request"
+
+  downstream:
+    name: Test xyz
+    timeout-minutes: 240
+    runs-on: ubuntu-latest
+    needs: deploy
+    env:
+      GITHUB_TOKEN: ${{ secrets.PULUMI_BOT_TOKEN }}
+    strategy:
+      fail-fast: false
+    steps:
+      - name: Await PR opened for pulumi-xyz
+        run: |
+          echo Await PR opened for pulumi-xyz
+          until gh search prs --repo pulumi/pulumi-xyz --match body "This PR was automatically generated by the pull-request workflow in the pulumi/ci-mgmt repo, from commit ${{ github.sha }}."  --json url | grep url; do sleep 30; done;
+      - name: Await PR codegen tests succeed.
+        run: |
+          echo "Await PR opened for pulumi-xyz"
+          number=$(gh search prs --repo pulumi/pulumi-xyz --match body "This PR was automatically generated by the pull-request workflow in the pulumi/ci-mgmt repo, from commit ${{ github.sha }}."  --json number --jq '.[0].number')
+          # Ensure that expected checks are pending before checking the status passes.
+          # We use 'test' as all workflows have tests.
+          until gh pr checks --repo "pulumi/pulumi-xyz" "$number" | grep 'test'; do sleep 30; done;
+          gh pr checks --repo "pulumi/pulumi-xyz" "$number" --watch --fail-fast

.github/workflows/update-workflows.yml

@@ -19,6 +19,14 @@ on:
         description: "Whether the provider is bridged or native"
         type: boolean
         required: true
+      downstream_test:
+        description: "Whether the upgrade is a downstream test"
+        type: boolean
+        default: false
+      skip_closing_prs:
+        description: "Whether to skip closing PRs"
+        type: boolean
+        default: false
 
 env:
   GITHUB_TOKEN: ${{ secrets.PULUMI_BOT_TOKEN }}
@@ -87,6 +95,7 @@ jobs:
           npx ts-node index.ts "$DIR"
       - name: Close obsolete PRs started by this workflow
         uses: actions/github-script@v6
+        if: ${{ !inputs.skip_closing_prs }}
         with:
           github-token: ${{ secrets.PULUMI_BOT_TOKEN }}
           script: |
@@ -112,15 +121,15 @@ jobs:
             }
       - name: Create PR
         id: create-pr
-        uses: peter-evans/create-pull-request@v3
+        uses: peter-evans/create-pull-request@v6
         with:
           author: Pulumi Bot <[email protected]>
           body: "This PR was automatically generated by the ${{ inputs.caller_workflow }} workflow in the pulumi/ci-mgmt repo, from commit ${{ github.sha }}."
           branch: "update-github-actions-workflows-${{ github.run_number }}"
           committer: Pulumi Bot <[email protected]>
           commit-message: "[internal] Update GitHub Actions workflow files"
           labels: "impact/no-changelog-required"
-          title: "Update GitHub Actions workflows."
+          title: "${{ inputs.downstream_test == true && '[DOWNSTREAM TEST] '}}Update GitHub Actions workflows."
           path: pulumi-${{ inputs.provider_name }}
           token: ${{ secrets.PULUMI_BOT_TOKEN }}
       - name: "Set PR to auto-merge"

provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/prerelease.yml

@@ -347,3 +347,6 @@ on:
   push:
     tags:
     - v*.*.*-**
+#{{- if .Config.testMasterAndReleaseWorkflows }}#
+  pull_request:
+#{{ end }}#

provider-ci/providers.json

@@ -65,6 +65,5 @@
     "vault",
     "venafi",
     "vsphere",
-    "wavefront",
-    "xyz"
+    "wavefront"
 ]