Merge remote-tracking branch 'origin' into mhotan/configurable-secret…

…-name
flyteorg · Jan 23, 2025 · 706502d · 706502d
2 parents 9b7a65c + 8125ae1
commit 706502d
Show file tree

Hide file tree

Showing 1,150 changed files with 46,259 additions and 24,377 deletions.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -33,6 +33,17 @@ If tests were added, say they were added here. Please make sure to add some test
 If it was tested in a way different from regular unit tests, please clarify how you tested step by step, ideally copy and paste-able, so that other reviewers can test and check, and descendants can verify in the future.
 If tests were not added, please describe why they were not added and/or why it was difficult to add.
 -->
+### Labels
+
+Please add one or more of the following labels to categorize your PR:
+- **added**: For new features.
+- **changed**: For changes in existing functionality.
+- **deprecated**: For soon-to-be-removed features.
+- **removed**: For features being removed.
+- **fixed**: For any bug fixed.
+- **security**: In case of vulnerabilities
+
+This is important to improve the readability of release notes.
 
 ### Setup process
 

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
@@ -9,6 +9,7 @@ on:
   push:
     branches:
       - master
+      - 'release-v**'
 env:
   GO_VERSION: "1.22"
   PRIORITIES: "P0"
@@ -32,8 +33,7 @@ jobs:
         component:
           - datacatalog
           - flyteadmin
-          # TODO(monorepo): Enable lint flytecopilot
-          # - flytecopilot
+          - flytecopilot
           - flytectl
           - flyteidl
           - flyteplugins

diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml
@@ -0,0 +1,18 @@
+name: 'Dependency Review'
+on: [pull_request]
+permissions:
+  contents: read
+  pull-requests: write
+jobs:
+  dependency-review:
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Checkout Repository'
+        uses: actions/checkout@v4
+      - name: Dependency Review
+        uses: actions/dependency-review-action@v4
+        with:
+          comment-summary-in-pr: on-failure
+          # Licenses need to come from https://spdx.org/licenses/
+          deny-licenses: GPL-1.0-only, GPL-1.0-or-later, GPL-2.0-only, GPL-2.0-or-later, GPL-3.0-only, GPL-3.0-or-later
+
diff --git a/.github/workflows/flytectl-install.yml b/.github/workflows/flytectl-install.yml
@@ -8,6 +8,7 @@ on:
   pull_request:
     paths:
       - flytectl/**
+      - 'release-v**'
   push:
     branches:
       - master

diff --git a/.github/workflows/flyteidl-buf-publish.yml b/.github/workflows/flyteidl-buf-publish.yml
@@ -6,6 +6,7 @@ on:
       - artifacts-shell-2
       - artifacts
       - master
+      - 'release-v**'
     paths:
       - 'flyteidl/**'
 jobs:

diff --git a/.github/workflows/flyteidl-checks.yml b/.github/workflows/flyteidl-checks.yml
@@ -9,6 +9,7 @@ on:
   push:
     branches:
       - master
+      - 'release-v**'
 env:
   GO_VERSION: "1.22"
 jobs:

diff --git a/.github/workflows/helm-charts.yaml b/.github/workflows/helm-charts.yaml
@@ -10,6 +10,7 @@ on:
     branches:
       - master
       - rc/*
+      - 'release-v**'
   workflow_dispatch:
 
 jobs:

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -59,7 +59,7 @@ jobs:
       - name: Before Build
         run: ${{ inputs.before-build }}
       - name: Build and Push Image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: ${{ inputs.dockerfile }}

diff --git a/.github/workflows/sandbox.yml b/.github/workflows/sandbox.yml
@@ -53,7 +53,7 @@ jobs:
           username: "${{ secrets.FLYTE_BOT_USERNAME }}"
           password: "${{ secrets.FLYTE_BOT_PAT }}"
       - name: Build and push DIND Image
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/arm64, linux/amd64

diff --git a/.github/workflows/single-binary.yml b/.github/workflows/single-binary.yml
@@ -10,6 +10,7 @@ on:
     branches:
       - master
       - rc/*
+      - "release-v**"
   workflow_dispatch:
 
 jobs:
@@ -72,7 +73,7 @@ jobs:
         run: |
           mkdir -p docker/sandbox-bundled/images/tar/{arm64,amd64}
       - name: Export ARM64 Image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/arm64
@@ -83,7 +84,7 @@ jobs:
           file: Dockerfile
           outputs: type=docker,dest=docker/sandbox-bundled/images/tar/arm64/flyte-binary.tar
       - name: Export AMD64 Image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/amd64
@@ -94,7 +95,7 @@ jobs:
           file: Dockerfile
           outputs: type=docker,dest=docker/sandbox-bundled/images/tar/amd64/flyte-binary.tar
       - name: Upload single binary image
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: single-binary-image
           path: docker/sandbox-bundled/images/tar
@@ -107,7 +108,7 @@ jobs:
           password: "${{ secrets.FLYTE_BOT_PAT }}"
       - name: Build and push Image
         if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/arm64, linux/amd64
@@ -133,7 +134,7 @@ jobs:
           echo "FLYTESNACKS_VERSION=${FLYTESNACKS_VERSION}" >> ${GITHUB_ENV}
       - name: Checkout
         uses: actions/checkout@v4
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: single-binary-image
           path: docker/sandbox-bundled/images/tar
@@ -143,7 +144,7 @@ jobs:
           driver-opts: image=moby/buildkit:master
           buildkitd-flags: "--allow-insecure-entitlement security.insecure"
       - name: Build sandbox image for functional tests
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: docker/sandbox-bundled
           load: true
@@ -175,7 +176,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install uv
-          uv pip install --system flytekit flytekitplugins-deck-standard flytekitplugins-envd "numpy<2.0.0" pyarrow
+          uv pip install --system flytekit flytekitplugins-deck-standard "numpy<2.0.0" pyarrow pandas
           uv pip freeze
       - name: Checkout flytesnacks
         uses: actions/checkout@v4
@@ -188,14 +189,107 @@ jobs:
           flytekit_version=$(pip show flytekit | grep -i version | awk '{ print $2 }')
           while read -r line;
           do
-              pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \
+              pyflyte -vv --config ./boilerplate/flyte/end2end/functional-test-config.yaml \
                   register \
                   --project flytesnacks \
                   --domain development \
                   --image cr.flyte.org/flyteorg/flytekit:py3.12-${flytekit_version} \
                   --version ${{ env.FLYTESNACKS_VERSION }} \
                   flytesnacks/$line;
           done < flytesnacks/flyte_tests.txt
+      - name: Install Pytest
+        run: |
+          pip install pytest
+      - name: End2End
+        run: |
+          make end2end_execute
+
+  sandbox-bundled-offloaded-functional-tests:
+    runs-on: ubuntu-latest
+    env:
+      FLYTESNACKS_PRIORITIES: "P0"
+      FLYTESNACKS_VERSION: ""
+    timeout-minutes: 60
+    needs: [build-and-push-single-binary-image]
+    steps:
+      - name: Set latest Flytesnacks release
+        if: ${{ env.FLYTESNACKS_VERSION == '' }}
+        run: |
+          FLYTESNACKS_VERSION="$(curl --silent https://api.github.com/repos/flyteorg/flytesnacks/releases/latest | jq -r .tag_name)"
+          echo "FLYTESNACKS_VERSION=${FLYTESNACKS_VERSION}" >> ${GITHUB_ENV}
+      - name: Checkout
+        uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: single-binary-image
+          path: docker/sandbox-bundled/images/tar
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          driver-opts: image=moby/buildkit:master
+          buildkitd-flags: "--allow-insecure-entitlement security.insecure"
+      - name: Build sandbox image for functional tests
+        uses: docker/build-push-action@v6
+        with:
+          context: docker/sandbox-bundled
+          load: true
+          allow: "security.insecure"
+          tags: flyte-sandbox-bundled:local
+      # Without this, the GHA runner is under disk pressure and evicts all the pods.
+      # Buildx cache uses roughly 50% (7gb) of the GHA runner's disk (14gb).
+      - name: Prune Docker Buildx cache to reclaim storage
+        run: docker buildx prune --all --force
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.12"
+      - uses: unionai/[email protected]
+      - name: Setup sandbox
+        run: |
+          mkdir -p ~/.flyte/sandbox
+          cat << EOF > ~/.flyte/sandbox/config.yaml
+          propeller:
+            literal-offloading-config:
+              enabled: true
+              min-size-in-mb-for-offloading: 0.000001
+          task_resources:
+            defaults:
+              cpu: "0"
+              memory: "0"
+            limits:
+              cpu: "0"
+              memory: "0"
+          EOF
+          flytectl demo start --image flyte-sandbox-bundled:local --imagePullPolicy Never
+        # By setting min-size-in-mb-for-offloading to 0.000001, we ensure that all tasks are offloaded.
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install uv
+          uv pip install --system flytekit flytekitplugins-deck-standard "numpy<2.0.0" pyarrow pandas
+          uv pip freeze
+      - name: Checkout flytesnacks
+        uses: actions/checkout@v4
+        with:
+          repository: flyteorg/flytesnacks
+          path: flytesnacks
+          ref: ${{ env.FLYTESNACKS_VERSION }}
+      - name: Register specific tests
+        run: |
+          flytekit_version=$(pip show flytekit | grep -i version | awk '{ print $2 }')
+          while read -r line;
+          do
+              pyflyte -vv --config ./boilerplate/flyte/end2end/functional-test-config.yaml \
+                  register \
+                  --project flytesnacks \
+                  --domain development \
+                  --image cr.flyte.org/flyteorg/flytekit:py3.12-${flytekit_version} \
+                  --version ${{ env.FLYTESNACKS_VERSION }} \
+                  flytesnacks/$line;
+          done < flytesnacks/flyte_tests.txt
+      - name: Install Pytest
+        run: |
+          pip install pytest
       - name: End2End
         run: |
           make end2end_execute
@@ -207,7 +301,7 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: single-binary-image
           path: docker/sandbox-bundled/images/tar
@@ -239,7 +333,7 @@ jobs:
           username: "${{ secrets.FLYTE_BOT_USERNAME }}"
           password: "${{ secrets.FLYTE_BOT_PAT }}"
       - name: Build and push multi-arch image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: docker/sandbox-bundled
           allow: "security.insecure"

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -8,6 +8,7 @@ on:
   push:
     branches:
       - master
+      - 'release-v**'
   pull_request:
 jobs:
   compile:
@@ -69,6 +70,9 @@ jobs:
       - name: Build the documentation
         working-directory: ${{ github.workspace }}/flyte
         shell: bash -el {0}
+        env:
+          # this is a fake key for algolia docsearch to make docs build process pass
+          DOCSEARCH_API_KEY: fake_docsearch_api_key
         run: |
           conda activate monodocs-env
           make docs

diff --git a/.github/workflows/validate-helm-charts.yaml b/.github/workflows/validate-helm-charts.yaml
@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
       - master
+      - 'release-v**'
     paths:
       - "charts/**"
       - "deployment/**"

diff --git a/.gitignore b/.gitignore
@@ -30,12 +30,12 @@ vendor/
 /docker/sandbox-bundled/images/tar
 **/bin/
 docs/_tags/
-docs/flytectl
-docs/protos
-docs/flytekit
+docs/api/flytectl
+docs/api/flytekit
+docs/api/flyteidl
 docs/flytesnacks
 docs/examples
 docs/_src
 docs/_projects
-docs/api
 docs/tests
+empty-config.yaml
diff --git a/CHANGELOG/CHANGELOG-v1.13.2.md b/CHANGELOG/CHANGELOG-v1.13.2.md
@@ -0,0 +1,53 @@
+# Flyte 1.13.2 Release Notes
+
+## What's Changed
+* Enable echo plugin by default by @pingsutw in https://github.com/flyteorg/flyte/pull/5679
+* Do not emit execution id label by default in single binary by @eapolinario in https://github.com/flyteorg/flyte/pull/5704
+* Using new offloaded metadata literal message for literal offloading by @pmahindrakar-oss in https://github.com/flyteorg/flyte/pull/5705
+* Improve error message for MismatchingTypes by @pingsutw in https://github.com/flyteorg/flyte/pull/5639
+* [Docs] Echo Task by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5707
+* Improve execution name readability by @wayner0628 in https://github.com/flyteorg/flyte/pull/5637
+* Configure imagePullPolicy to be Always pull on flyte sandbox environment by @pmahindrakar-oss in https://github.com/flyteorg/flyte/pull/5709
+* Set IsDefault to False for echo plugin by @pingsutw in https://github.com/flyteorg/flyte/pull/5713
+* Move default execution name generation to flyteadmin by @wayner0628 in https://github.com/flyteorg/flyte/pull/5714
+* Update helm/docs per changes in supported task discovery by @Sovietaced in https://github.com/flyteorg/flyte/pull/5694
+* [flyteagent] Add Logging for Agent Supported Task Types by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5718
+* extend pod customization to include init containers by @samhita-alla in https://github.com/flyteorg/flyte/pull/5685
+* Update "Try Serverless" language in Quickstart guide by @neverett in https://github.com/flyteorg/flyte/pull/5698
+* Refactor flyteadmin to pass proto structs as pointers by @Sovietaced in https://github.com/flyteorg/flyte/pull/5717
+* fix: Use deterministic execution names in scheduler by @pingsutw in https://github.com/flyteorg/flyte/pull/5724
+* [flyteagent] Enable `agent-service` in every configuration  by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5730
+* [Docs][flyteagent] Remove `agent-service` config in flyte agent documentation by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5731
+* Fix flytectl returning the oldest workflow when using --latest flag by @RRap0so in https://github.com/flyteorg/flyte/pull/5716
+* Remove explicit go toolchain versions by @ddl-ebrown in https://github.com/flyteorg/flyte/pull/5723
+* Add listing api to stow storage by @bgedik in https://github.com/flyteorg/flyte/pull/5741
+* Use latest upload/download-artifact action version by @Sovietaced in https://github.com/flyteorg/flyte/pull/5743
+* Introduced SMTP notification by @robert-ulbrich-mercedes-benz in https://github.com/flyteorg/flyte/pull/5535
+* Added literal offloading for array node map tasks by @pmahindrakar-oss in https://github.com/flyteorg/flyte/pull/5697
+* pytorch object.inv moved by @wild-endeavor in https://github.com/flyteorg/flyte/pull/5755
+* [RFC] Binary IDL With MessagePack Bytes by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5742
+* Rename literal offloading flags and enable offloading in local config by @eapolinario in https://github.com/flyteorg/flyte/pull/5754
+* Add semver check for dev and beta version by @pmahindrakar-oss in https://github.com/flyteorg/flyte/pull/5757
+* Add comments for semver regex by @pmahindrakar-oss in https://github.com/flyteorg/flyte/pull/5758
+* Enable caching in flyte-core helmchart by @cpaulik in https://github.com/flyteorg/flyte/pull/5739
+* [Flyte][1][IDL] Binary IDL With MessagePack by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5751
+* [Flyte][2][Literal Type For Scalar] Binary IDL With MessagePack by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5761
+* Backoff on etcd errors by @EngHabu in https://github.com/flyteorg/flyte/pull/5710
+* [Docs][flyteagent] Update Databricks Agent Setup to V2 by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5766
+* [Docs][flyteagent] Improve Agent Secret Setup by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5765
+* [RFC] Offloaded Raw Literals by @wild-endeavor in https://github.com/flyteorg/flyte/pull/5103
+* Use pluggable clock for auto refresh cache and make unit tests fast by @Sovietaced in https://github.com/flyteorg/flyte/pull/5767
+* [Docs] Flyte Deck example by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5777
+* Update ImageSpec documentation by @pingsutw in https://github.com/flyteorg/flyte/pull/5748
+* Revert "Improve execution name readability" by @pingsutw in https://github.com/flyteorg/flyte/pull/5740
+* [Docs] Flyte Deck example V2 by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5781
+* [flytepropeller][flyteadmin] Compiler unknown literal type error handling by @Future-Outlier in https://github.com/flyteorg/flyte/pull/5651
+* [Temporary fix] Pin flytekit version for single binary builds by @pmahindrakar-oss in https://github.com/flyteorg/flyte/pull/5798
+* Fix propeller crash when inferring literal type for an offloaded literal by @pmahindrakar-oss in https://github.com/flyteorg/flyte/pull/5771
+
+## New Contributors
+* @wayner0628 made their first contribution in https://github.com/flyteorg/flyte/pull/5637
+* @robert-ulbrich-mercedes-benz made their first contribution in https://github.com/flyteorg/flyte/pull/5535
+* @cpaulik made their first contribution in https://github.com/flyteorg/flyte/pull/5739
+
+**Full Changelog**: https://github.com/flyteorg/flyte/compare/v1.13.1...v1.13.2
-Original file line number
+Diff line change
@@ Expand Up / @@ -9,6 +9,7 @@ on: @@
       push:
         branches:
           - master
+          - 'release-v**'
     env:
       GO_VERSION: "1.22"
     jobs:
@@ Expand Down @@