Generalize bridged template for reuse with NoUpstream providers (#1278)

This is #1268 without the sharding. This should be safe as it is not affecting any providers per se yet, just building up a capability. --------- Co-authored-by: Florian Stadler <[email protected]>
pulumi · Jan 13, 2025 · 5cf2bc8 · 5cf2bc8
1 parent 7926c46
commit 5cf2bc8
Show file tree

Hide file tree

Showing 36 changed files with 2,379 additions and 3 deletions.
diff --git a/provider-ci/Makefile b/provider-ci/Makefile
@@ -37,7 +37,7 @@ format:
 # files for other bridged provider repositories should be ephemeral.
 .PHONY: test-providers test-provider/%
 
-test-providers: test-provider/aws test-provider/docker test-provider/cloudflare test-provider/acme
+test-providers: test-provider/aws test-provider/docker test-provider/cloudflare test-provider/acme test-provider/eks
 
 # 1. Delete all files except the .ci-mgmt.yaml file and run the provider-ci generate command.
 # 2. Copy the generated provider repository to a temporary git repo and run actionlint on it.

diff --git a/provider-ci/internal/pkg/config.go b/provider-ci/internal/pkg/config.go
@@ -265,6 +265,10 @@ type Config struct {
 	// MakeTemplate has no effect but is set by 78 providers.
 	// https://github.com/search?q=org%3Apulumi+path%3A.ci-mgmt.yaml+%22makeTemplate%3A%22&type=code
 	MakeTemplate string `yaml:"makeTemplate"`
+
+	// NoUpstream is a temporary hack to disable bridge-specific workflow steps
+	// as part of the work to consolidate these with native providers.
+	NoUpstream bool
 }
 
 // LoadLocalConfig loads the provider configuration at the given path with

diff --git a/provider-ci/internal/pkg/generate.go b/provider-ci/internal/pkg/generate.go
@@ -45,6 +45,11 @@ func GeneratePackage(opts GenerateOpts) error {
 	if err != nil {
 		return fmt.Errorf("error getting template directories: %w", err)
 	}
+	if opts.Config.Template == "generic" {
+		opts.Config.NoUpstream = true
+		opts.Config.CheckUpstreamUpgrade = false
+	}
+
 	// Clean up old workflows if requested
 	if opts.Config.CleanGithubWorkflows {
 		workflows, err := os.ReadDir(filepath.Join(opts.OutDir, ".github", "workflows"))
@@ -102,6 +107,8 @@ func getTemplateDirs(templateName string) ([]string, error) {
 	case "external-bridged-provider":
 		// Render more specific templates last to allow them to override more general templates.
 		return []string{"dev-container", "provider", "external-provider", "bridged-provider"}, nil
+	case "generic":
+		return []string{"provider", "pulumi-provider", "bridged-provider"}, nil
 	default:
 		return nil, fmt.Errorf("unknown template: %s", templateName)
 	}

diff --git a/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/upgrade-bridge.yml b/provider-ci/internal/pkg/templates/bridged-provider/.github/workflows/upgrade-bridge.yml
@@ -1,3 +1,4 @@
+#{{ if not .Config.NoUpstream -}}#
 # WARNING: This file is autogenerated - changes will be overwritten when regenerated by https://github.com/pulumi/ci-mgmt
 
 name: Upgrade bridge
@@ -113,3 +114,4 @@ jobs:
         pr-reviewers: ${{ github.event.client_payload.pr-reviewers }}
         pr-description: ${{ github.event.client_payload.pr-description }}
         pr-title-prefix: ${{ github.event.client_payload.pr-title-prefix }}
+#{{ end -}}#
diff --git a/provider-ci/internal/pkg/templates/bridged-provider/Makefile b/provider-ci/internal/pkg/templates/bridged-provider/Makefile
@@ -35,7 +35,7 @@ VERSION_GENERIC = $(shell pulumictl convert-version --language generic --version
 # Strips debug information from the provider binary to reduce its size and speed up builds
 LDFLAGS_STRIP_SYMBOLS=-s -w
 LDFLAGS_PROJ_VERSION=-X $(PROJECT)/$(VERSION_PATH)=$(VERSION_GENERIC)#{{if .Config.ProviderVersion}}# -X #{{ .Config.ProviderVersion }}#=$(VERSION_GENERIC)#{{end}}#
-#{{- if .Config.ProviderVersion }}#
+#{{- if and (not (eq .Config.ProviderVersion "")) (not .Config.NoUpstream) }}#
 LDFLAGS_UPSTREAM_VERSION=-X #{{ .Config.ProviderVersion }}#=v$(VERSION_GENERIC)
 #{{- else }}#
 LDFLAGS_UPSTREAM_VERSION=
@@ -287,7 +287,11 @@ tfgen_build_only: bin/$(TFGEN)
 bin/$(TFGEN): provider/*.go provider/go.* .make/upstream
 	(cd provider && go build $(PULUMI_PROVIDER_BUILD_PARALLELISM) -o $(WORKING_DIR)/bin/$(TFGEN) -ldflags "$(LDFLAGS_PROJ_VERSION) $(LDFLAGS_EXTRAS)" $(PROJECT)/$(PROVIDER_PATH)/cmd/$(TFGEN))
 .PHONY: tfgen schema tfgen_no_deps tfgen_build_only
-
+#{{ if .Config.NoUpstream }}#
+upstream: .make/upstream
+.make/upstream:
+	@touch $@
+#{{- else }}#
 # Apply patches to the upstream submodule, if it exists
 upstream: .make/upstream
 # Re-run if the upstream commit or the patches change
@@ -302,6 +306,7 @@ endif
 	cd upstream-tools && yarn --silent run apply
 #{{- end }}#
 	@touch $@
+#{{- end }}#
 .PHONY: upstream
 
 bin/pulumi-java-gen: .pulumi-java-gen.version

diff --git a/provider-ci/internal/pkg/templates/bridged-provider/upstream.sh b/provider-ci/internal/pkg/templates/bridged-provider/upstream.sh
@@ -1,3 +1,4 @@
+#{{ if not .Config.NoUpstream -}}#
 #!/usr/bin/env bash
 # WARNING: This file is autogenerated - changes will be overwritten when regenerated by https://github.com/pulumi/ci-mgmt
 
@@ -400,3 +401,4 @@ case ${original_cmd} in
     exit 1
     ;;
 esac
+#{{ end -}}#
diff --git a/provider-ci/providers.json b/provider-ci/providers.json
@@ -23,6 +23,7 @@
     "dnsimple",
     "docker",
     "ec",
+    "eks",
     "external",
     "f5bigip",
     "fastly",

diff --git a/provider-ci/test-providers/eks/.ci-mgmt.yaml b/provider-ci/test-providers/eks/.ci-mgmt.yaml
@@ -0,0 +1,25 @@
+provider: eks
+major-version: 3
+aws: true
+env:
+  ALT_AWS_ACCESS_KEY_ID: ${{ secrets.ALT_AWS_ACCESS_KEY_ID }}
+  ALT_AWS_SECRET_ACCESS_KEY: ${{ secrets.ALT_AWS_SECRET_ACCESS_KEY }}
+  AWS_REGION: us-west-2
+  GITHUB_TOKEN: ${{ secrets.PULUMI_BOT_TOKEN }}
+  GOLANGCI_LINT_VERSION: v1.61.0
+  NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+  NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+  PROVIDER: eks
+  PUBLISH_REPO_PASSWORD: ${{ secrets.OSSRH_PASSWORD }}
+  PUBLISH_REPO_USERNAME: ${{ secrets.OSSRH_USERNAME }}
+  PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
+  PULUMI_API: https://api.pulumi-staging.io
+  PULUMI_GO_DEP_ROOT: ${{ github.workspace }}/..
+  SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+  SIGNING_KEY: ${{ secrets.JAVA_SIGNING_KEY }}
+  SIGNING_KEY_ID: ${{ secrets.JAVA_SIGNING_KEY_ID }}
+  SIGNING_PASSWORD: ${{ secrets.JAVA_SIGNING_PASSWORD }}
+  PULUMI_ENABLE_RESOURCE_REFERENCES: 1
+  PULUMI_LOCAL_NUGET: ${{ github.workspace }}/nuget
+template: generic
+freeDiskSpaceBeforeTest: true # TODO: https://github.com/pulumi/pulumi/issues/17718
diff --git a/provider-ci/test-providers/eks/.gitattributes b/provider-ci/test-providers/eks/.gitattributes
@@ -0,0 +1 @@
+sdk/**/* linguist-generated=true
diff --git a/provider-ci/test-providers/eks/.github/ISSUE_TEMPLATE/bug.yaml b/provider-ci/test-providers/eks/.github/ISSUE_TEMPLATE/bug.yaml
@@ -0,0 +1,69 @@
+name: Bug Report
+description: Report something that's not working correctly
+labels: ["kind/bug", "needs-triage"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this bug report! 
+        You can also ask questions on our [Community Slack](https://slack.pulumi.com/).
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: Describe what happened
+      description: Please summarize what happened, including what Pulumi commands you ran, as well as
+        an inline snippet of any relevant error or console output.
+    validations:
+      required: true
+  - type: textarea
+    id: sample-program
+    attributes:
+      label: Sample program
+      description: |
+        <details><summary>Provide a reproducible sample program</summary>
+        If this is a bug you encountered while running a Pulumi command, please provide us with a minimal, 
+        self-contained Pulumi program that reproduces this behavior so that we can investigate on our end. 
+        Without a functional reproduction, we will not be able to prioritize this bug.
+        **Note:** If the program output is more than a few lines, please send us a Gist or a link to a file.
+        </details>
+    validations:
+      required: true
+  - type: textarea
+    id: log-output
+    attributes:
+      label: Log output
+      description: |
+        <details><summary>How to Submit Logs</summary>
+        If this is something that is dependent on your environment, please also provide us with the output of
+        `pulumi up --logtostderr --logflow -v=10` from the root of your project.
+        We may also ask you to supply us with debug output following [these steps](https://www.pulumi.com/docs/using-pulumi/pulumi-packages/debugging-provider-packages/).
+        **Note:** If the log output is more than a few lines, please send us a Gist or a link to a file.
+        </details>
+  - type: textarea
+    id: resources
+    attributes:
+      label: Affected Resource(s)
+      description: Please list the affected Pulumi Resource(s) or Function(s).
+    validations:
+      required: false
+  - type: textarea
+    id: versions
+    attributes:
+      label: Output of `pulumi about`
+      description: Provide the output of `pulumi about` from the root of your project.
+    validations:
+      required: true
+  - type: textarea
+    id: ctx
+    attributes:
+      label: Additional context
+      description: Anything else you would like to add?
+    validations:
+      required: false
+  - type: textarea
+    id: voting
+    attributes:
+      label: Contributing
+      value: |
+        Vote on this issue by adding a 👍 reaction. 
+        To contribute a fix for this issue, leave a comment (and link to your pull request, if you've opened one already).
diff --git a/provider-ci/test-providers/eks/.github/ISSUE_TEMPLATE/epic.md b/provider-ci/test-providers/eks/.github/ISSUE_TEMPLATE/epic.md
@@ -0,0 +1,35 @@
+---
+name: Epic
+about: Tracks a shippable unit of work
+title: '[Epic] {your-title-here}'
+labels: kind/epic
+projects: ['pulumi/32']
+assignees: ''
+type: Epic
+---
+
+## Overview
+<!-- Start with a one- to three-sentence summary that should be understandable by any Pulumian or community member, even those without any context on the work. -->
+
+## Key KPIs
+<!-- What KPIs should this Epic will move; what could we measure to observe that this project was successful? -->
+
+## Key Stakeholders
+- Product and Engineering: <!-- Teams and individuals involved in the design and implementation -->
+- Documentation: <!-- Representative from the docs team -->
+- Marketing/Partnerships: <!-- Representative from the Marketing team -->
+- Customers: <!-- [Tracking Issue](add-link-and-uncomment) -->
+
+## Key Deliverables
+<!-- List any discrete chunks of work or milestones that are planned in the epic (eg. subcomponent A, dogfood release, beta, etc.) -->
+
+### References 📔
+
+<!-- Any project that is more than one iteration should have a Project Board using this [template](https://github.com/orgs/pulumi/projects/131).  -->
+- [ ] Project View <!-- [Link](add-link-and-uncomment) -->
+- [ ] PR/FAQ <!-- [Link](add-link-and-uncomment) -->
+- [ ] Design Doc <!-- [Link](add-link-and-uncomment) -->
+- [ ] UX Designs <!-- [Link](add-link-and-uncomment) -->
+- [ ] Decision Log <!-- [Link](add-link-and-uncomment) -->
+
+<!-- Work items should be added to the project board linked above  -->
diff --git a/provider-ci/test-providers/eks/.github/actions/download-bin/action.yml b/provider-ci/test-providers/eks/.github/actions/download-bin/action.yml
@@ -0,0 +1,23 @@
+name: Download the provider binary
+description: Downloads the provider binary to `bin/`.
+
+runs:
+  using: "composite"
+  steps:
+
+    - name: Download pulumi-resource-eks
+      uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+      with:
+        pattern: pulumi-resource-eks-*-linux-amd64.tar.gz
+        path: ${{ github.workspace }}/bin
+        merge-multiple: true
+
+    - name: Untar pulumi-resource-eks
+      shell: bash
+      run: |
+        tar -zxf ${{ github.workspace }}/bin/*amd64.tar.gz -C ${{ github.workspace}}/bin
+
+    - name: Mark pulumi-resource-eks as executable
+      shell: bash
+      run: |
+        find ${{ github.workspace }} -name "pulumi-*-eks" -print -exec chmod +x {} \;
diff --git a/provider-ci/test-providers/eks/.github/actions/download-sdk/action.yml b/provider-ci/test-providers/eks/.github/actions/download-sdk/action.yml
@@ -0,0 +1,19 @@
+name: Download SDK asset
+description: Restores the SDK asset for a language.
+
+inputs:
+  language:
+    required: true
+    description: One of nodejs, python, dotnet, go, java
+
+runs:
+  using: "composite"
+  steps:
+    - name: Download ${{ inputs.language }} SDK
+      uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+      with:
+        name: ${{ inputs.language }}-sdk.tar.gz
+        path: ${{ github.workspace}}/sdk/
+    - name: Uncompress SDK folder
+      shell: bash
+      run: tar -zxf ${{ github.workspace }}/sdk/${{ inputs.language }}.tar.gz -C ${{ github.workspace }}/sdk/${{ inputs.language }}
diff --git a/provider-ci/test-providers/eks/.github/actions/download-tfgen/action.yml b/provider-ci/test-providers/eks/.github/actions/download-tfgen/action.yml
@@ -0,0 +1,17 @@
+name: Download the tfgen binary
+description: Downloads the tfgen binary to `bin/`.
+
+runs:
+  using: "composite"
+  steps:
+
+    - name: Download pulumi-tfgen-eks
+      uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+      with:
+        name: pulumi-tfgen-eks
+        path: ${{ github.workspace }}/bin
+
+    - name: Ensure pulumi-tfgen-eks is executable
+      shell: bash
+      run: |
+        find ${{ github.workspace }} -name "pulumi-*-eks" -print -exec chmod +x {} \;
diff --git a/provider-ci/test-providers/eks/.github/actions/setup-tools/action.yml b/provider-ci/test-providers/eks/.github/actions/setup-tools/action.yml
@@ -0,0 +1,91 @@
+name: Setup tools
+description: Installs Go, Pulumi, pulumictl, schema-tools, Node.JS, Python, dotnet and Java.
+
+inputs:
+  tools:
+    description: |
+      Comma separated list of tools to install. The default of "all" installs all tools. Available tools are:
+        go
+        pulumicli
+        pulumictl
+        schema-tools
+        nodejs
+        python
+        dotnet
+        java
+    default: all
+  cache-go:
+    description: |
+      Whether to enable the GitHub cache for Go. Appropriate for disabling in
+      smaller jobs that typically completely before the "real" job has an
+      opportunity to populate the cache.
+    default: "true"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install Go
+      if: inputs.tools == 'all' || contains(inputs.tools, 'go')
+      uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5
+      with:
+        go-version: "1.21.x"
+        cache-dependency-path: |
+          provider/*.sum
+          upstream/*.sum
+          sdk/go/*.sum
+          sdk/*.sum
+          *.sum
+        # TODO(https://github.com/actions/setup-go/issues/316): Restore but don't save the cache.
+        cache: ${{ inputs.cache-go }}
+
+    - name: Install pulumictl
+      if: inputs.tools == 'all' || contains(inputs.tools, 'pulumictl')
+      uses: jaxxstorm/action-install-gh-release@cd6b2b78ad38bdd294341cda064ec0692b06215b # v1.14.0
+      with:
+        tag: v0.0.46
+        repo: pulumi/pulumictl
+
+    - name: Install Pulumi CLI
+      if: inputs.tools == 'all' || contains(inputs.tools, 'pulumicli')
+      uses: pulumi/actions@c7fad9e2f0b79653172b36538b8b34b3c0291952 # v6
+      with:
+        pulumi-version: "dev"
+
+    - name: Install Schema Tools
+      if: inputs.tools == 'all' || contains(inputs.tools, 'schema-tools')
+      uses: jaxxstorm/action-install-gh-release@cd6b2b78ad38bdd294341cda064ec0692b06215b # v1.14.0
+      with:
+        repo: pulumi/schema-tools
+
+    - name: Setup Node
+      if: inputs.tools == 'all' || contains(inputs.tools, 'nodejs')
+      uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4
+      with:
+        node-version: 20.x
+        registry-url: https://registry.npmjs.org
+
+    - name: Setup DotNet
+      if: inputs.tools == 'all' || contains(inputs.tools, 'dotnet')
+      uses: actions/setup-dotnet@87b7050bc53ea08284295505d98d2aa94301e852 # v4.2.0
+      with:
+        dotnet-version: 6.0.x
+
+    - name: Setup Python
+      if: inputs.tools == 'all' || contains(inputs.tools, 'python')
+      uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
+      with:
+        python-version: 3.11.8
+
+    - name: Setup Java
+      if: inputs.tools == 'all' || contains(inputs.tools, 'java')
+      uses: actions/setup-java@7a6d8a8234af8eb26422e24e3006232cccaa061b # v4.6.0
+      with:
+        cache: gradle
+        distribution: temurin
+        java-version: 11
+
+    - name: Setup Gradle
+      if: inputs.tools == 'all' || contains(inputs.tools, 'java')
+      uses: gradle/gradle-build-action@ac2d340dc04d9e1113182899e983b5400c17cda1 # v3
+      with:
+        gradle-version: 7.6