From 7b9728d76fc8c6282ccdf3e95288ce6add08d08b Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 1 Dec 2023 13:15:26 +0100 Subject: [PATCH 1/9] Format Makefile to look uniform to duckdb/extension-template one --- Makefile | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 8ce5142..c772cda 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,7 @@ ifeq (${STATIC_LIBCPP}, 1) STATIC_LIBCPP=-DSTATIC_LIBCPP=TRUE endif +#### VCPKG config VCPKG_TOOLCHAIN_PATH?= ifneq ("${VCPKG_TOOLCHAIN_PATH}", "") TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DVCPKG_MANIFEST_DIR='${PROJ_DIR}' -DVCPKG_BUILD=1 -DCMAKE_TOOLCHAIN_FILE='${VCPKG_TOOLCHAIN_PATH}' @@ -21,6 +22,7 @@ ifneq ("${VCPKG_TARGET_TRIPLET}", "") TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}' endif +#### Enable Ninja as generator ifeq ($(GEN),ninja) GENERATOR=-G "Ninja" FORCE_COLOR=-DFORCE_COLORED_OUTPUT=1 @@ -28,10 +30,19 @@ endif BUILD_FLAGS=-DEXTENSION_STATIC_BUILD=1 -DBUILD_EXTENSIONS="httpfs" ${OSX_BUILD_UNIVERSAL_FLAG} ${STATIC_LIBCPP} ${TOOLCHAIN_FLAGS} -CLIENT_FLAGS := +EXT_NAME=iceberg + +#### Configuration for this extension +EXTENSION_NAME=ICEBERG +EXTENSION_FLAGS=\ +-DDUCKDB_EXTENSION_NAMES="iceberg" \ +-DDUCKDB_EXTENSION_${EXTENSION_NAME}_PATH="$(PROJ_DIR)" \ +-DDUCKDB_EXTENSION_${EXTENSION_NAME}_LOAD_TESTS=1 \ +-DDUCKDB_EXTENSION_${EXTENSION_NAME}_INCLUDE_PATH="$(PROJ_DIR)src/include" \ +-DDUCKDB_EXTENSION_${EXTENSION_NAME}_TEST_PATH="$(PROJ_DIR)test/sql" -# These flags will make DuckDB build the extension -EXTENSION_FLAGS=-DDUCKDB_EXTENSION_NAMES="iceberg" -DDUCKDB_EXTENSION_ICEBERG_PATH="$(PROJ_DIR)" -DDUCKDB_EXTENSION_ICEBERG_SHOULD_LINK=1 -DDUCKDB_EXTENSION_ICEBERG_INCLUDE_PATH="$(PROJ_DIR)src/include" + +CLIENT_FLAGS := pull: git submodule init From 2b94e369093d66edbd3eeb7c8647b202e215160b Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 1 Dec 2023 13:15:40 +0100 Subject: [PATCH 2/9] Add duckdb-wasm specifics to Makefile --- Makefile | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c772cda..1dfe07e 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all clean format debug release duckdb_debug duckdb_release pull update +.PHONY: all clean format debug release duckdb_debug duckdb_release pull update wasm_mvp wasm_eh wasm_threads all: release @@ -127,4 +127,24 @@ data_large: data data_clean python3 scripts/test_data_generator/generate_iceberg.py 1 data/iceberg/generated_spec2_1 2 data_clean: - rm -rf data/iceberg/generated_* \ No newline at end of file + rm -rf data/iceberg/generated_* + +WASM_LINK_TIME_FLAGS= + +wasm_mvp: + mkdir -p build/wasm_mvp + emcmake cmake $(GENERATOR) -DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -Bbuild/wasm_mvp -DCMAKE_CXX_FLAGS="-DDUCKDB_CUSTOM_PLATFORM=wasm_mvp" -DSKIP_EXTENSIONS="parquet" -S duckdb $(TOOLCHAIN_FLAGS) $(EXTENSION_FLAGS) -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake + emmake make -j8 -Cbuild/wasm_mvp + cd build/wasm_mvp/extension/${EXT_NAME} && emcc $f -sSIDE_MODULE=1 -o ../../${EXT_NAME}.duckdb_extension.wasm -O3 ${EXT_NAME}.duckdb_extension $(WASM_LINK_TIME_FLAGS) + +wasm_eh: + mkdir -p build/wasm_eh + emcmake cmake $(GENERATOR) -DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -Bbuild/wasm_eh -DCMAKE_CXX_FLAGS="-fwasm-exceptions -DWEBDB_FAST_EXCEPTIONS=1 -DDUCKDB_CUSTOM_PLATFORM=wasm_eh" -DSKIP_EXTENSIONS="parquet" -S duckdb $(TOOLCHAIN_FLAGS) $(EXTENSION_FLAGS) -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake + emmake make -j8 -Cbuild/wasm_eh + cd build/wasm_eh/extension/${EXT_NAME} && emcc $f -sSIDE_MODULE=1 -o ../../${EXT_NAME}.duckdb_extension.wasm -O3 ${EXT_NAME}.duckdb_extension $(WASM_LINK_TIME_FLAGS) + +wasm_threads: + mkdir -p ./build/wasm_threads + emcmake cmake $(GENERATOR) -DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -Bbuild/wasm_threads -DCMAKE_CXX_FLAGS="-fwasm-exceptions -DWEBDB_FAST_EXCEPTIONS=1 -DWITH_WASM_THREADS=1 -DWITH_WASM_SIMD=1 -DWITH_WASM_BULK_MEMORY=1 -DDUCKDB_CUSTOM_PLATFORM=wasm_threads" -DSKIP_EXTENSIONS="parquet" -S duckdb $(TOOLCHAIN_FLAGS) $(EXTENSION_FLAGS) -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake + emmake make -j8 -Cbuild/wasm_threads + cd build/wasm_threads/extension/${EXT_NAME} && emcc $f -sSIDE_MODULE=1 -o ../../${EXT_NAME}.duckdb_extension.wasm -O3 ${EXT_NAME}.duckdb_extension $(WASM_LINK_TIME_FLAGS) From 33490d56a749fd6415ce0740b2e3634626b9f014 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 1 Dec 2023 14:26:52 +0100 Subject: [PATCH 3/9] Unittest on Windows --- Makefile | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 1dfe07e..89c32a0 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,12 @@ all: release MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) PROJ_DIR := $(dir $(MKFILE_PATH)) +ifeq ($(OS),Windows_NT) + TEST_PATH="/test/Release/unittest.exe" +else + TEST_PATH="/test/unittest" +endif + OSX_BUILD_UNIVERSAL_FLAG= ifneq (${OSX_BUILD_ARCH}, "") OSX_BUILD_UNIVERSAL_FLAG=-DOSX_BUILD_ARCH=${OSX_BUILD_ARCH} @@ -90,12 +96,10 @@ release_python: release # Main tests test: test_release - test_release: release - ./build/release/test/unittest --test-dir . "[sql]" - + ./build/release/$(TEST_PATH) "$(PROJ_DIR)test/*" test_debug: debug - ./build/debug/test/unittest --test-dir . "[sql]" + ./build/debug/$(TEST_PATH) "$(PROJ_DIR)test/*" # Client tests test_js: test_debug_js From b5a525d07b230bc9d310d01ef0e907325951acf6 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 1 Dec 2023 12:59:15 +0100 Subject: [PATCH 4/9] CI: use script extension deploy (enabling also duckdb-wasm builds and deploy) --- .../workflows/MainDistributionPipeline.yml | 32 +++++ .github/workflows/_extension_deploy.yml | 123 ++++++++++++++++++ scripts/extension-upload-ci.sh | 90 +++++++++++++ 3 files changed, 245 insertions(+) create mode 100644 .github/workflows/MainDistributionPipeline.yml create mode 100644 .github/workflows/_extension_deploy.yml create mode 100755 scripts/extension-upload-ci.sh diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml new file mode 100644 index 0000000..81db2db --- /dev/null +++ b/.github/workflows/MainDistributionPipeline.yml @@ -0,0 +1,32 @@ +# +# This workflow calls the main distribution pipeline from DuckDB to build, test and (optionally) release the extension +# +name: Main Extension Distribution Pipeline +on: + push: + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} + cancel-in-progress: true + +jobs: + duckdb-stable-build: + name: Build extension binaries + uses: duckdb/duckdb/.github/workflows/_extension_distribution.yml@6812703823d1d66566bc7eaac2b6e4b273c85333 + with: + vcpkg_commit: a42af01b72c28a8e1d7b48107b33e4f286a55ef6 + duckdb_version: v0.9.2 + extension_name: iceberg + + duckdb-stable-deploy: + name: Deploy extension binaries + needs: duckdb-stable-build + uses: ./.github/workflows/_extension_deploy.yml + secrets: inherit + with: + duckdb_version: v0.9.2 + extension_name: iceberg + deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} + deploy_versioned: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} diff --git a/.github/workflows/_extension_deploy.yml b/.github/workflows/_extension_deploy.yml new file mode 100644 index 0000000..e1caa57 --- /dev/null +++ b/.github/workflows/_extension_deploy.yml @@ -0,0 +1,123 @@ +# +# Reusable workflow that deploys the artifacts produced by github.com/duckdb/duckdb/.github/workflows/_extension_distribution.yml +# +# note: this workflow needs to be located in the extension repository, as it requires secrets to be passed to the +# deploy script. However, it should generally not be necessary to modify this workflow in your extension repository, as +# this workflow can be configured to use a custom deploy script. + + +name: Extension Deployment +on: + workflow_call: + inputs: + # The name of the extension + extension_name: + required: true + type: string + # DuckDB version to build against + duckdb_version: + required: true + type: string + # ';' separated list of architectures to exclude, for example: 'linux_amd64;osx_arm64' + exclude_archs: + required: false + type: string + default: "" + # Whether to upload this deployment as the latest. This may overwrite a previous deployment. + deploy_latest: + required: false + type: boolean + default: false + # Whether to upload this deployment under a versioned path. These will not be deleted automatically + deploy_versioned: + required: false + type: boolean + default: false + # Postfix added to artifact names. Can be used to guarantee unique names when this workflow is called multiple times + artifact_postfix: + required: false + type: string + default: "" + # Override the default deploy script with a custom script + deploy_script: + required: false + type: string + default: "./scripts/extension-upload-ci.sh*" + # Override the default matrix parse script with a custom script + matrix_parse_script: + required: false + type: string + default: "./duckdb/scripts/modify_distribution_matrix.py" + +jobs: + generate_matrix: + name: Generate matrix + runs-on: ubuntu-latest + outputs: + deploy_matrix: ${{ steps.parse-matrices.outputs.deploy_matrix }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: 'true' + + - name: Checkout DuckDB to version + run: | + cd duckdb + git checkout ${{ inputs.duckdb_version }} + + - id: parse-matrices + run: | + cat ./duckdb/.github/config/distribution_matrix.json > distribution_matrix.json + grep wasm distribution_matrix.json || (head -n -1 ./duckdb/.github/config/distribution_matrix.json > distribution_matrix.json && echo ',"wasm":{"include":[{"duckdb_arch":"wasm_mvp","vcpkg_triplet":"wasm32-emscripten"},{"duckdb_arch":"wasm_eh","vcpkg_triplet":"wasm32-emscripten"},{"duckdb_arch":"wasm_threads","vcpkg_triplet":"wasm32-emscripten"}]}}' >> distribution_matrix.json) + python3 ${{ inputs.matrix_parse_script }} --input distribution_matrix.json --deploy_matrix --output deploy_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty + deploy_matrix="`cat deploy_matrix.json`" + echo deploy_matrix=$deploy_matrix >> $GITHUB_OUTPUT + echo `cat $GITHUB_OUTPUT` + + deploy: + name: Deploy + runs-on: ubuntu-latest + needs: generate_matrix + if: ${{ needs.generate_matrix.outputs.deploy_matrix != '{}' && needs.generate_matrix.outputs.deploy_matrix != '' }} + strategy: + matrix: ${{fromJson(needs.generate_matrix.outputs.deploy_matrix)}} + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: 'true' + + - name: Checkout DuckDB to version + run: | + cd duckdb + git checkout ${{ inputs.duckdb_version }} + + - uses: actions/download-artifact@v2 + with: + name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}}${{startsWith(matrix.duckdb, 'wasm') && '.wasm' || ''}} + path: | + /tmp/extension + + - name: Deploy + shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_ORG_DEPLOY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_ORG_DEPLOY_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.S3_DUCKDB_ORG_REGION }} + BUCKET_NAME: ${{ secrets.S3_DUCKDB_ORG_BUCKET }} + DUCKDB_EXTENSION_SIGNING_PK: ${{ secrets.S3_DUCKDB_ORG_EXTENSION_SIGNING_PK }} + run: | + pwd + python3 -m pip install pip awscli + git config --global --add safe.directory '*' + cd duckdb + git fetch --tags + export DUCKDB_VERSION=`git tag --points-at HEAD` + export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} + cd .. + git fetch --tags + export EXT_VERSION=`git tag --points-at HEAD` + export EXT_VERSION=${EXT_VERSION:=`git log -1 --format=%h`} + ${{ inputs.deploy_script }} ${{ inputs.extension_name }} $EXT_VERSION $DUCKDB_VERSION ${{ matrix.duckdb_arch }} $BUCKET_NAME ${{inputs.deploy_latest || 'true' && 'false'}} ${{inputs.deploy_versioned || 'true' && 'false'}} diff --git a/scripts/extension-upload-ci.sh b/scripts/extension-upload-ci.sh new file mode 100755 index 0000000..32097f2 --- /dev/null +++ b/scripts/extension-upload-ci.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +# Extension upload script + +# Usage: ./extension-upload.sh +# : Name of the extension +# : Version (commit / version tag) of the extension +# : Version (commit / version tag) of DuckDB +# : Architecture target of the extension binary +# : S3 bucket to upload to +# : Set this as the latest version ("true" / "false", default: "false") +# : Set this as a versioned version that will prevent its deletion + +set -e + +if [[ $4 == wasm* ]]; then + ext="/tmp/extension/$1.duckdb_extension.wasm" +else + ext="/tmp/extension/$1.duckdb_extension" +fi + +echo $ext + +script_dir="$(dirname "$(readlink -f "$0")")" + +# calculate SHA256 hash of extension binary +cat $ext > $ext.append + +if [[ $4 == wasm* ]]; then + # 0 for custom section + # 113 in hex = 275 in decimal, total lenght of what follows (1 + 16 + 2 + 256) + # [1(continuation) + 0010011(payload) = \x93, 0(continuation) + 10(payload) = \x02] + echo -n -e '\x00' >> $ext.append + echo -n -e '\x93\x02' >> $ext.append + # 10 in hex = 16 in decimal, lenght of name, 1 byte + echo -n -e '\x10' >> $ext.append + echo -n -e 'duckdb_signature' >> $ext.append + # the name of the WebAssembly custom section, 16 bytes + # 100 in hex, 256 in decimal + # [1(continuation) + 0000000(payload) = ff, 0(continuation) + 10(payload)], + # for a grand total of 2 bytes + echo -n -e '\x80\x02' >> $ext.append +fi + +# (Optionally) Sign binary +if [ "$DUCKDB_EXTENSION_SIGNING_PK" != "" ]; then + echo "$DUCKDB_EXTENSION_SIGNING_PK" > private.pem + $script_dir/../duckdb/scripts/compute-extension-hash.sh $ext.append > $ext.hash + openssl pkeyutl -sign -in $ext.hash -inkey private.pem -pkeyopt digest:sha256 -out $ext.sign + rm -f private.pem +fi + +# Signature is always there, potentially defaulting to 256 zeros +truncate -s 256 $ext.sign + +# append signature to extension binary +cat $ext.sign >> $ext.append + +# compress extension binary +if [[ $4 == wasm_* ]]; then + gzip < $ext.append > "$ext.compressed" +else + brotli < $ext.append > "$ext.compressed" +fi + +set -e + +# Abort if AWS key is not set +if [ -z "$AWS_ACCESS_KEY_ID" ]; then + echo "No AWS key found, skipping.." + exit 0 +fi + +# upload versioned version +if [[ $7 = 'true' ]]; then + if [[ $4 == wasm* ]]; then + aws s3 cp $ext.compressed s3://$5/duckdb-wasm/$1/$2/duckdb-wasm/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" + else + aws s3 cp $ext.compressed s3://$5/$1/$2/$3/$4/$1.duckdb_extension.gz --acl public-read + fi +fi + +# upload to latest version +if [[ $6 = 'true' ]]; then + if [[ $4 == wasm* ]]; then + aws s3 cp $ext.compressed s3://$5/duckdb-wasm/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" + else + aws s3 cp $ext.compressed s3://$5/$3/$4/$1.duckdb_extension.gz --acl public-read + fi +fi From c3571d561580560fa8bab20b76e0f5234bd13723 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Fri, 1 Dec 2023 14:27:48 +0100 Subject: [PATCH 5/9] Remove wasm targets This is blocked on https://github.com/microsoft/vcpkg/issues/34594 --- .github/workflows/MainDistributionPipeline.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index 81db2db..6cf4e76 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -19,6 +19,7 @@ jobs: vcpkg_commit: a42af01b72c28a8e1d7b48107b33e4f286a55ef6 duckdb_version: v0.9.2 extension_name: iceberg + exclude_archs: "wasm_mvp;wasm_eh;wasm_threads" duckdb-stable-deploy: name: Deploy extension binaries @@ -28,5 +29,6 @@ jobs: with: duckdb_version: v0.9.2 extension_name: iceberg + exclude_archs: "wasm_mvp;wasm_eh;wasm_threads" deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} deploy_versioned: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} From 7b14d93de4c5c39ed98c1889ebdd2f9f9685ba08 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Wed, 14 Feb 2024 11:05:49 +0100 Subject: [PATCH 6/9] bump to v0.10 --- .github/workflows/Linux.yml | 45 +------ .github/workflows/MacOS.yml | 34 +---- .../workflows/MainDistributionPipeline.yml | 16 +-- .github/workflows/Windows.yml | 90 ------------- .github/workflows/_extension_deploy.yml | 123 ------------------ duckdb | 2 +- scripts/extension-upload-ci.sh | 90 ------------- scripts/extension-upload.sh | 36 ----- 8 files changed, 11 insertions(+), 425 deletions(-) delete mode 100644 .github/workflows/Windows.yml delete mode 100644 .github/workflows/_extension_deploy.yml delete mode 100755 scripts/extension-upload-ci.sh delete mode 100755 scripts/extension-upload.sh diff --git a/.github/workflows/Linux.yml b/.github/workflows/Linux.yml index aa42fd5..cab4e5d 100644 --- a/.github/workflows/Linux.yml +++ b/.github/workflows/Linux.yml @@ -15,9 +15,7 @@ jobs: strategy: matrix: # Add commits/tags to build against other DuckDB versions - duckdb_version: [ '' ] - arch: ['linux_amd64', 'linux_arm64', 'linux_amd64_gcc4'] - vcpkg_version: [ '2023.04.15' ] + arch: ['linux_amd64', 'linux_amd64_gcc4'] include: - arch: 'linux_amd64_gcc4' container: 'quay.io/pypa/manylinux2014_x86_64' @@ -25,9 +23,6 @@ jobs: - arch: 'linux_amd64' container: 'ubuntu:18.04' vcpkg_triplet: 'x64-linux' - - arch: 'linux_arm64' - container: 'ubuntu:18.04' - vcpkg_triplet: 'arm64-linux' env: VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }} GEN: Ninja @@ -59,12 +54,6 @@ jobs: fetch-depth: 0 submodules: 'true' - - name: Checkout DuckDB to version - if: ${{ matrix.duckdb_version != ''}} - run: | - cd duckdb - git checkout ${{ matrix.duckdb_version }} - - name: Setup ManyLinux2014 if: ${{ matrix.arch == 'linux_amd64_gcc4' }} run: | @@ -94,34 +83,4 @@ jobs: - name: Test extension if: ${{ matrix.arch == 'linux_amd64_gcc4' || matrix.arch == 'linux_amd64' }} run: | - make test - - - uses: actions/upload-artifact@v2 - with: - name: ${{matrix.arch}}-extensions - path: | - build/release/extension/iceberg/iceberg.duckdb_extension - - - name: Deploy - env: - AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }} - BUCKET_NAME: ${{ secrets.S3_BUCKET }} - DUCKDB_EXTENSION_SIGNING_PK: ${{ secrets.DUCKDB_EXTENSION_SIGNING_KEY }} - run: | - git config --global --add safe.directory '*' - cd duckdb - git fetch --tags - export DUCKDB_VERSION=`git tag --points-at HEAD` - export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} - cd .. - if [[ "$AWS_ACCESS_KEY_ID" == "" ]] ; then - echo 'No key set, skipping' - elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then - python3 -m pip install pip awscli - ./scripts/extension-upload.sh iceberg ${{ github.ref_name }} $DUCKDB_VERSION ${{matrix.arch}} $BUCKET_NAME true - elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then - python3 -m pip install pip awscli - ./scripts/extension-upload.sh iceberg `git log -1 --format=%h` $DUCKDB_VERSION ${{matrix.arch}} $BUCKET_NAME false - fi \ No newline at end of file + make test \ No newline at end of file diff --git a/.github/workflows/MacOS.yml b/.github/workflows/MacOS.yml index 4a522bb..baa4b23 100644 --- a/.github/workflows/MacOS.yml +++ b/.github/workflows/MacOS.yml @@ -14,8 +14,6 @@ jobs: strategy: matrix: # Add commits/tags to build against other DuckDB versions - duckdb_version: [ '' ] - vcpkg_version: [ '2023.04.15' ] vcpkg_triplet: [ 'x64-osx', 'arm64-osx' ] include: - vcpkg_triplet: 'x64-osx' @@ -64,34 +62,4 @@ jobs: if: ${{ matrix.osx_build_arch == 'x86_64'}} shell: bash run: | - make test_release - - - uses: actions/upload-artifact@v2 - with: - name: osx-${{ matrix.osx_build_arch }}-extension - path: | - build/release/extension/iceberg/iceberg.duckdb_extension - - - name: Deploy - env: - AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }} - BUCKET_NAME: ${{ secrets.S3_BUCKET }} - DUCKDB_EXTENSION_SIGNING_PK: ${{ secrets.DUCKDB_EXTENSION_SIGNING_KEY }} - run: | - git config --global --add safe.directory '*' - cd duckdb - git fetch --tags - export DUCKDB_VERSION=`git tag --points-at HEAD` - export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} - cd .. - if [[ "$AWS_ACCESS_KEY_ID" == "" ]] ; then - echo 'No key set, skipping' - elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then - python3 -m pip install pip awscli - ./scripts/extension-upload.sh iceberg ${{ github.ref_name }} $DUCKDB_VERSION ${{matrix.duckdb_arch}} $BUCKET_NAME true - elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then - python3 -m pip install pip awscli - ./scripts/extension-upload.sh iceberg `git log -1 --format=%h` $DUCKDB_VERSION ${{matrix.duckdb_arch}} $BUCKET_NAME false - fi \ No newline at end of file + make test_release \ No newline at end of file diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index 6cf4e76..0d3dec2 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -14,21 +14,19 @@ concurrency: jobs: duckdb-stable-build: name: Build extension binaries - uses: duckdb/duckdb/.github/workflows/_extension_distribution.yml@6812703823d1d66566bc7eaac2b6e4b273c85333 + uses: duckdb/duckdb/.github/workflows/_extension_distribution.yml@v0.10.0 with: - vcpkg_commit: a42af01b72c28a8e1d7b48107b33e4f286a55ef6 - duckdb_version: v0.9.2 extension_name: iceberg - exclude_archs: "wasm_mvp;wasm_eh;wasm_threads" + duckdb_version: 'v0.10.0' + exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools' duckdb-stable-deploy: name: Deploy extension binaries needs: duckdb-stable-build - uses: ./.github/workflows/_extension_deploy.yml + uses: duckdb/duckdb/.github/workflows/_extension_deploy.yml@v0.10.0 secrets: inherit with: - duckdb_version: v0.9.2 extension_name: iceberg - exclude_archs: "wasm_mvp;wasm_eh;wasm_threads" - deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} - deploy_versioned: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} + duckdb_version: 'v0.10.0' + exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools' + deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} \ No newline at end of file diff --git a/.github/workflows/Windows.yml b/.github/workflows/Windows.yml deleted file mode 100644 index e387449..0000000 --- a/.github/workflows/Windows.yml +++ /dev/null @@ -1,90 +0,0 @@ -name: Windows -on: [push, pull_request,repository_dispatch] -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} - cancel-in-progress: true -defaults: - run: - shell: bash - -jobs: - windows: - name: Release - runs-on: windows-latest - strategy: - matrix: - # Add commits/tags to build against other DuckDB versions - duckdb_version: [ '' ] - vcpkg_version: [ '2023.04.15' ] - vcpkg_triplet: ['x64-windows-static-md'] - env: - VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }} - GEN: Ninja - VCPKG_ROOT: ${{ github.workspace }}\vcpkg - VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}\vcpkg\scripts\buildsystems\vcpkg.cmake - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'true' - - - name: Setup vcpkg - uses: lukka/run-vcpkg@v11.1 - with: - vcpkgGitCommitId: a42af01b72c28a8e1d7b48107b33e4f286a55ef6 - - - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - - name: Checkout DuckDB to version - # Add commits/tags to build against other DuckDB versions - if: ${{ matrix.duckdb_version != ''}} - run: | - cd duckdb - git checkout ${{ matrix.duckdb_version }} - - - name: Build extension - run: | - make release - - # TODO: Windows tests currently failing due to path separator issue. -# - name: Test Extension -# shell: bash -# run: | -# build/release/test/Release/unittest.exe --test-dir . "*.test" - - - uses: actions/upload-artifact@v2 - with: - name: windows-extension - path: | - build/release/extension/iceberg/iceberg.duckdb_extension - - - name: error log - if: failure() - shell: bash - run: cat D:/a/duckdb_iceberg/duckdb_iceberg/vcpkg/buildtrees/avro-cpp/install-x64-windows-static-md-dbg-out.log - - - name: Deploy - env: - AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }} - BUCKET_NAME: ${{ secrets.S3_BUCKET }} - DUCKDB_EXTENSION_SIGNING_PK: ${{ secrets.DUCKDB_EXTENSION_SIGNING_KEY }} - run: | - cd duckdb - git fetch --tags - export DUCKDB_VERSION=`git tag --points-at HEAD` - export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} - cd .. - if [[ "$AWS_ACCESS_KEY_ID" == "" ]] ; then - echo 'No key set, skipping' - elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then - python -m pip install awscli - ./scripts/extension-upload.sh iceberg ${{ github.ref_name }} $DUCKDB_VERSION windows_amd64 $BUCKET_NAME true - elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then - python -m pip install awscli - ./scripts/extension-upload.sh iceberg `git log -1 --format=%h` $DUCKDB_VERSION windows_amd64 $BUCKET_NAME false - fi \ No newline at end of file diff --git a/.github/workflows/_extension_deploy.yml b/.github/workflows/_extension_deploy.yml deleted file mode 100644 index e1caa57..0000000 --- a/.github/workflows/_extension_deploy.yml +++ /dev/null @@ -1,123 +0,0 @@ -# -# Reusable workflow that deploys the artifacts produced by github.com/duckdb/duckdb/.github/workflows/_extension_distribution.yml -# -# note: this workflow needs to be located in the extension repository, as it requires secrets to be passed to the -# deploy script. However, it should generally not be necessary to modify this workflow in your extension repository, as -# this workflow can be configured to use a custom deploy script. - - -name: Extension Deployment -on: - workflow_call: - inputs: - # The name of the extension - extension_name: - required: true - type: string - # DuckDB version to build against - duckdb_version: - required: true - type: string - # ';' separated list of architectures to exclude, for example: 'linux_amd64;osx_arm64' - exclude_archs: - required: false - type: string - default: "" - # Whether to upload this deployment as the latest. This may overwrite a previous deployment. - deploy_latest: - required: false - type: boolean - default: false - # Whether to upload this deployment under a versioned path. These will not be deleted automatically - deploy_versioned: - required: false - type: boolean - default: false - # Postfix added to artifact names. Can be used to guarantee unique names when this workflow is called multiple times - artifact_postfix: - required: false - type: string - default: "" - # Override the default deploy script with a custom script - deploy_script: - required: false - type: string - default: "./scripts/extension-upload-ci.sh*" - # Override the default matrix parse script with a custom script - matrix_parse_script: - required: false - type: string - default: "./duckdb/scripts/modify_distribution_matrix.py" - -jobs: - generate_matrix: - name: Generate matrix - runs-on: ubuntu-latest - outputs: - deploy_matrix: ${{ steps.parse-matrices.outputs.deploy_matrix }} - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'true' - - - name: Checkout DuckDB to version - run: | - cd duckdb - git checkout ${{ inputs.duckdb_version }} - - - id: parse-matrices - run: | - cat ./duckdb/.github/config/distribution_matrix.json > distribution_matrix.json - grep wasm distribution_matrix.json || (head -n -1 ./duckdb/.github/config/distribution_matrix.json > distribution_matrix.json && echo ',"wasm":{"include":[{"duckdb_arch":"wasm_mvp","vcpkg_triplet":"wasm32-emscripten"},{"duckdb_arch":"wasm_eh","vcpkg_triplet":"wasm32-emscripten"},{"duckdb_arch":"wasm_threads","vcpkg_triplet":"wasm32-emscripten"}]}}' >> distribution_matrix.json) - python3 ${{ inputs.matrix_parse_script }} --input distribution_matrix.json --deploy_matrix --output deploy_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty - deploy_matrix="`cat deploy_matrix.json`" - echo deploy_matrix=$deploy_matrix >> $GITHUB_OUTPUT - echo `cat $GITHUB_OUTPUT` - - deploy: - name: Deploy - runs-on: ubuntu-latest - needs: generate_matrix - if: ${{ needs.generate_matrix.outputs.deploy_matrix != '{}' && needs.generate_matrix.outputs.deploy_matrix != '' }} - strategy: - matrix: ${{fromJson(needs.generate_matrix.outputs.deploy_matrix)}} - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'true' - - - name: Checkout DuckDB to version - run: | - cd duckdb - git checkout ${{ inputs.duckdb_version }} - - - uses: actions/download-artifact@v2 - with: - name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}}${{startsWith(matrix.duckdb, 'wasm') && '.wasm' || ''}} - path: | - /tmp/extension - - - name: Deploy - shell: bash - env: - AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_ORG_DEPLOY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_ORG_DEPLOY_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.S3_DUCKDB_ORG_REGION }} - BUCKET_NAME: ${{ secrets.S3_DUCKDB_ORG_BUCKET }} - DUCKDB_EXTENSION_SIGNING_PK: ${{ secrets.S3_DUCKDB_ORG_EXTENSION_SIGNING_PK }} - run: | - pwd - python3 -m pip install pip awscli - git config --global --add safe.directory '*' - cd duckdb - git fetch --tags - export DUCKDB_VERSION=`git tag --points-at HEAD` - export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} - cd .. - git fetch --tags - export EXT_VERSION=`git tag --points-at HEAD` - export EXT_VERSION=${EXT_VERSION:=`git log -1 --format=%h`} - ${{ inputs.deploy_script }} ${{ inputs.extension_name }} $EXT_VERSION $DUCKDB_VERSION ${{ matrix.duckdb_arch }} $BUCKET_NAME ${{inputs.deploy_latest || 'true' && 'false'}} ${{inputs.deploy_versioned || 'true' && 'false'}} diff --git a/duckdb b/duckdb index 275f4a7..20b1486 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit 275f4a7e9564db08cc3cda211e1f63a7967308cf +Subproject commit 20b1486d1192f9fbd2328d1122b5afe5f1747fce diff --git a/scripts/extension-upload-ci.sh b/scripts/extension-upload-ci.sh deleted file mode 100755 index 32097f2..0000000 --- a/scripts/extension-upload-ci.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash - -# Extension upload script - -# Usage: ./extension-upload.sh -# : Name of the extension -# : Version (commit / version tag) of the extension -# : Version (commit / version tag) of DuckDB -# : Architecture target of the extension binary -# : S3 bucket to upload to -# : Set this as the latest version ("true" / "false", default: "false") -# : Set this as a versioned version that will prevent its deletion - -set -e - -if [[ $4 == wasm* ]]; then - ext="/tmp/extension/$1.duckdb_extension.wasm" -else - ext="/tmp/extension/$1.duckdb_extension" -fi - -echo $ext - -script_dir="$(dirname "$(readlink -f "$0")")" - -# calculate SHA256 hash of extension binary -cat $ext > $ext.append - -if [[ $4 == wasm* ]]; then - # 0 for custom section - # 113 in hex = 275 in decimal, total lenght of what follows (1 + 16 + 2 + 256) - # [1(continuation) + 0010011(payload) = \x93, 0(continuation) + 10(payload) = \x02] - echo -n -e '\x00' >> $ext.append - echo -n -e '\x93\x02' >> $ext.append - # 10 in hex = 16 in decimal, lenght of name, 1 byte - echo -n -e '\x10' >> $ext.append - echo -n -e 'duckdb_signature' >> $ext.append - # the name of the WebAssembly custom section, 16 bytes - # 100 in hex, 256 in decimal - # [1(continuation) + 0000000(payload) = ff, 0(continuation) + 10(payload)], - # for a grand total of 2 bytes - echo -n -e '\x80\x02' >> $ext.append -fi - -# (Optionally) Sign binary -if [ "$DUCKDB_EXTENSION_SIGNING_PK" != "" ]; then - echo "$DUCKDB_EXTENSION_SIGNING_PK" > private.pem - $script_dir/../duckdb/scripts/compute-extension-hash.sh $ext.append > $ext.hash - openssl pkeyutl -sign -in $ext.hash -inkey private.pem -pkeyopt digest:sha256 -out $ext.sign - rm -f private.pem -fi - -# Signature is always there, potentially defaulting to 256 zeros -truncate -s 256 $ext.sign - -# append signature to extension binary -cat $ext.sign >> $ext.append - -# compress extension binary -if [[ $4 == wasm_* ]]; then - gzip < $ext.append > "$ext.compressed" -else - brotli < $ext.append > "$ext.compressed" -fi - -set -e - -# Abort if AWS key is not set -if [ -z "$AWS_ACCESS_KEY_ID" ]; then - echo "No AWS key found, skipping.." - exit 0 -fi - -# upload versioned version -if [[ $7 = 'true' ]]; then - if [[ $4 == wasm* ]]; then - aws s3 cp $ext.compressed s3://$5/duckdb-wasm/$1/$2/duckdb-wasm/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" - else - aws s3 cp $ext.compressed s3://$5/$1/$2/$3/$4/$1.duckdb_extension.gz --acl public-read - fi -fi - -# upload to latest version -if [[ $6 = 'true' ]]; then - if [[ $4 == wasm* ]]; then - aws s3 cp $ext.compressed s3://$5/duckdb-wasm/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" - else - aws s3 cp $ext.compressed s3://$5/$3/$4/$1.duckdb_extension.gz --acl public-read - fi -fi diff --git a/scripts/extension-upload.sh b/scripts/extension-upload.sh deleted file mode 100755 index 2304642..0000000 --- a/scripts/extension-upload.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -# Usage: ./extension-upload.sh -# : Name of the extension -# : Version (commit / version tag) of the extension -# : Version (commit / version tag) of DuckDB -# : Architecture target of the extension binary -# : S3 bucket to upload to -# : Set this as the latest version ("true" / "false", default: "false") - -set -e - -ext="build/release/extension/$1/$1.duckdb_extension" - -# (Optionally) Sign binary -if [ "$DUCKDB_EXTENSION_SIGNING_PK" != "" ]; then - echo "$DUCKDB_EXTENSION_SIGNING_PK" > private.pem - duckdb/scripts/compute-extension-hash.sh $ext > $ext.hash - openssl pkeyutl -sign -in $ext.hash -inkey private.pem -pkeyopt digest:sha256 -out $ext.sign - cat $ext.sign >> $ext -fi - -# compress extension binary -gzip < "${ext}" > "$ext.gz" - -# upload compressed extension binary to S3 -aws s3 cp $ext.gz s3://$5/$1/$2/$3/$4/$1.duckdb_extension.gz --acl public-read - -# upload to latest if copy_to_latest is set to true -if [[ $6 = 'true' ]]; then - aws s3 cp $ext.gz s3://$5/$1/latest/$3/$4/$1.duckdb_extension.gz --acl public-read -fi - -if [ "$DUCKDB_EXTENSION_SIGNING_PK" != "" ]; then - rm private.pem -fi \ No newline at end of file From 5572ffc3c605e6f180dbe910ed2d476897f60f4d Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Wed, 14 Feb 2024 12:44:28 +0100 Subject: [PATCH 7/9] fixes related to bumping duckdb to v0.10.0 --- Makefile | 1 + scripts/start-rest-catalog.sh | 14 +++++++++----- test/sql/iceberg_snapshots.test | 2 ++ 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index b3202da..9d2c2c6 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,7 @@ EXTENSION_FLAGS=\ -DDUCKDB_EXTENSION_NAMES="iceberg" \ -DDUCKDB_EXTENSION_${EXTENSION_NAME}_PATH="$(PROJ_DIR)" \ -DDUCKDB_EXTENSION_${EXTENSION_NAME}_LOAD_TESTS=1 \ +-DDUCKDB_EXTENSION_${EXTENSION_NAME}_SHOULD_LINK=1 \ -DDUCKDB_EXTENSION_${EXTENSION_NAME}_INCLUDE_PATH="$(PROJ_DIR)src/include" \ -DDUCKDB_EXTENSION_${EXTENSION_NAME}_TEST_PATH="$(PROJ_DIR)test/sql" diff --git a/scripts/start-rest-catalog.sh b/scripts/start-rest-catalog.sh index 393caca..ba8fca8 100755 --- a/scripts/start-rest-catalog.sh +++ b/scripts/start-rest-catalog.sh @@ -14,11 +14,15 @@ python3 provision.py UNPARTITIONED_TABLE_PATH=$(curl -s http://127.0.0.1:8181/v1/namespaces/default/tables/table_unpartitioned | jq -r '."metadata-location"') SQL=$(cat <<-END -SET s3_access_key_id='admin'; -SET s3_secret_access_key='password'; -SET s3_endpoint='127.0.0.1:9000'; -SET s3_url_style='path'; -SET s3_use_ssl=false; + +CREATE SECRET ( + TYPE S3, + KEY_ID 'admin', + SECRET 'password', + ENDPOINT '127.0.0.1:9000', + URL_STYLE 'path', + USE_SSL 0 +); SELECT * FROM iceberg_scan('${UNPARTITIONED_TABLE_PATH}'); END diff --git a/test/sql/iceberg_snapshots.test b/test/sql/iceberg_snapshots.test index 943f68b..6b93f12 100644 --- a/test/sql/iceberg_snapshots.test +++ b/test/sql/iceberg_snapshots.test @@ -8,6 +8,8 @@ SELECT * FROM ICEBERG_SNAPSHOTS('data/iceberg/lineitem_iceberg'); ---- Catalog Error +require notwindows + require iceberg query IIII From dcb2e013c01af156d04143941eea4e5a985d480e Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Wed, 14 Feb 2024 13:03:06 +0100 Subject: [PATCH 8/9] add platform param --- Makefile | 2 +- README.md | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 9d2c2c6..6757fa6 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ ifeq ($(GEN),ninja) FORCE_COLOR=-DFORCE_COLORED_OUTPUT=1 endif -BUILD_FLAGS=-DEXTENSION_STATIC_BUILD=1 -DBUILD_EXTENSIONS="httpfs" ${OSX_BUILD_UNIVERSAL_FLAG} ${STATIC_LIBCPP} ${TOOLCHAIN_FLAGS} +BUILD_FLAGS=-DEXTENSION_STATIC_BUILD=1 -DBUILD_EXTENSIONS="httpfs" ${OSX_BUILD_UNIVERSAL_FLAG} ${STATIC_LIBCPP} ${TOOLCHAIN_FLAGS} -DDUCKDB_EXPLICIT_PLATFORM='${DUCKDB_PLATFORM}' EXT_NAME=iceberg diff --git a/README.md b/README.md index 364ac68..30c7c1a 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,4 @@ -> **Disclaimer:** This extension is currently in an experimental state. Feel free to try it out, but be aware that minimal testing and -benchmarking was done. - -> Warning: This extension currently builds with the main branch of DuckDB. A PR is being worked on. When the PR is merged, -this extension will be updated and usable from (nightly) DuckDB releases. +> **Disclaimer:** This extension is currently in an experimental state. Feel free to try it out, but be aware that things may not work as expected # DuckDB extension for Apache Iceberg From 8f9d78a9212323f523de5f714022ad23d9dd5e1e Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Wed, 14 Feb 2024 15:45:22 +0100 Subject: [PATCH 9/9] remove unnecessary workflows --- .github/workflows/Linux.yml | 86 ------------------------------------- .github/workflows/MacOS.yml | 65 ---------------------------- 2 files changed, 151 deletions(-) delete mode 100644 .github/workflows/Linux.yml delete mode 100644 .github/workflows/MacOS.yml diff --git a/.github/workflows/Linux.yml b/.github/workflows/Linux.yml deleted file mode 100644 index cab4e5d..0000000 --- a/.github/workflows/Linux.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: Linux -on: [push, pull_request,repository_dispatch] -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} - cancel-in-progress: true -defaults: - run: - shell: bash - -jobs: - linux: - name: Linux Release - runs-on: ubuntu-latest - container: ${{ matrix.container }} - strategy: - matrix: - # Add commits/tags to build against other DuckDB versions - arch: ['linux_amd64', 'linux_amd64_gcc4'] - include: - - arch: 'linux_amd64_gcc4' - container: 'quay.io/pypa/manylinux2014_x86_64' - vcpkg_triplet: 'x64-linux' - - arch: 'linux_amd64' - container: 'ubuntu:18.04' - vcpkg_triplet: 'x64-linux' - env: - VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }} - GEN: Ninja - VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake - - steps: - - name: Install required ubuntu packages - if: ${{ matrix.arch == 'linux_amd64' || matrix.arch == 'linux_arm64' }} - run: | - apt-get update -y -qq - apt-get install -y -qq software-properties-common - add-apt-repository ppa:git-core/ppa - apt-get update -y -qq - apt-get install -y -qq ninja-build make gcc-multilib g++-multilib libssl-dev wget openjdk-8-jdk zip maven unixodbc-dev libc6-dev-i386 lib32readline6-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext unzip build-essential checkinstall libffi-dev curl libz-dev openssh-client - apt-get install -y -qq tar pkg-config - - - name: Install Git 2.18.5 - if: ${{ matrix.arch == 'linux_amd64' || matrix.arch == 'linux_arm64' }} - run: | - wget https://github.com/git/git/archive/refs/tags/v2.18.5.tar.gz - tar xvf v2.18.5.tar.gz - cd git-2.18.5 - make - make prefix=/usr install - git --version - - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'true' - - - name: Setup ManyLinux2014 - if: ${{ matrix.arch == 'linux_amd64_gcc4' }} - run: | - ./duckdb/scripts/setup_manylinux2014.sh general aws-cli ccache ssh openssl python_alias - - - name: Setup Ubuntu - if: ${{ matrix.arch == 'linux_amd64' || matrix.arch == 'linux_arm64' }} - uses: ./duckdb/.github/actions/ubuntu_18_setup - with: - aarch64_cross_compile: 1 - - - name: Setup vcpkg - uses: lukka/run-vcpkg@v11.1 - with: - vcpkgGitCommitId: a42af01b72c28a8e1d7b48107b33e4f286a55ef6 - - # Build extension - - name: Build extension - env: - GEN: ninja - STATIC_LIBCPP: 1 - CC: ${{ matrix.arch == 'linux_arm64' && 'aarch64-linux-gnu-gcc' || '' }} - CXX: ${{ matrix.arch == 'linux_arm64' && 'aarch64-linux-gnu-g++' || '' }} - run: | - make release - - - name: Test extension - if: ${{ matrix.arch == 'linux_amd64_gcc4' || matrix.arch == 'linux_amd64' }} - run: | - make test \ No newline at end of file diff --git a/.github/workflows/MacOS.yml b/.github/workflows/MacOS.yml deleted file mode 100644 index baa4b23..0000000 --- a/.github/workflows/MacOS.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: MacOS -on: [push, pull_request,repository_dispatch] -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} - cancel-in-progress: true -defaults: - run: - shell: bash - -jobs: - macos: - name: MacOS Release (${{ matrix.osx_build_arch }}) - runs-on: macos-latest - strategy: - matrix: - # Add commits/tags to build against other DuckDB versions - vcpkg_triplet: [ 'x64-osx', 'arm64-osx' ] - include: - - vcpkg_triplet: 'x64-osx' - osx_build_arch: 'x86_64' - duckdb_arch: 'osx_amd64' - - vcpkg_triplet: 'arm64-osx' - osx_build_arch: 'arm64' - duckdb_arch: 'osx_arm64' - - env: - VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }} - OSX_BUILD_ARCH: ${{ matrix.osx_build_arch }} - GEN: Ninja - VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'true' - - - name: Install Ninja - run: brew install ninja - - - name: Setup Ccache - uses: hendrikmuhs/ccache-action@main - with: - key: ${{ github.job }}-${{ matrix.duckdb_version }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} - - - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - - name: Setup vcpkg - uses: lukka/run-vcpkg@v11.1 - with: - vcpkgGitCommitId: a42af01b72c28a8e1d7b48107b33e4f286a55ef6 - - - name: Build extension - shell: bash - run: | - make release - - - name: Test Extension - if: ${{ matrix.osx_build_arch == 'x86_64'}} - shell: bash - run: | - make test_release \ No newline at end of file