Merge pull request #39 from samansmink/add-azure-auth

Add Azure support

.github/workflows/CloudTesting.yml

@@ -0,0 +1,80 @@
+name: Cloud functional tests
+on: [push, repository_dispatch]
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
+  cancel-in-progress: true
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  azure-tests-linux:
+    name: Azure tests (Linux)
+    runs-on: ubuntu-latest
+    env:
+      VCPKG_TARGET_TRIPLET: x64-linux
+      VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
+      GEN: Ninja
+      DUCKDB_PLATFORM: linux_amd64
+
+    steps:
+      - name: Install required ubuntu packages
+        run: |
+          sudo apt-get update -y -qq
+          sudo apt-get install -y -qq software-properties-common
+          sudo add-apt-repository ppa:git-core/ppa
+          sudo apt-get update -y -qq
+          sudo apt-get install -y -qq ninja-build make gcc-multilib g++-multilib zip unzip build-essential checkinstall curl libz-dev openssh-client
+
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: 'true'
+
+      - name: Setup Ccache
+        uses: hendrikmuhs/[email protected] # Note: pinned due to GLIBC incompatibility in later releases
+        with:
+          key: ${{ github.job }}-${{ matrix.duckdb_arch }}
+
+      - name: Setup vcpkg
+        uses: lukka/[email protected]
+        with:
+          vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6
+
+      - name: Setup Rust
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Build extension
+        env:
+          GEN: ninja
+        run: |
+          make release
+
+      - name: Test with Service Principal (SPN) in env vars
+        env:
+          AZURE_CLIENT_ID: ${{secrets.AZURE_CLIENT_ID}}
+          AZURE_CLIENT_SECRET: ${{secrets.AZURE_CLIENT_SECRET}}
+          AZURE_TENANT_ID: ${{secrets.AZURE_TENANT_ID}}
+          AZURE_STORAGE_ACCOUNT: ${{secrets.AZURE_STORAGE_ACCOUNT}}
+        run: |
+          python3 duckdb/scripts/run_tests_one_by_one.py ./build/release/test/unittest "*test/sql/cloud/*"
+
+      - name: Test with SPN logged in in azure-cli
+        env:
+          AZURE_STORAGE_ACCOUNT: ${{secrets.AZURE_STORAGE_ACCOUNT}}
+          DUCKDB_AZ_CLI_LOGGED_IN: 1
+        run: |
+          az login --service-principal -u ${{secrets.AZURE_CLIENT_ID}} -p ${{secrets.AZURE_CLIENT_SECRET}} --tenant ${{secrets.AZURE_TENANT_ID}}
+          python3 duckdb/scripts/run_tests_one_by_one.py ./build/release/test/unittest "*test/sql/cloud/*"
+
+      - name: Log out azure-cli
+        if: always()
+        run: |
+          az logout
+
+      - name: Tests that focus on public non-authenticated requests
+        env:
+          AZURE_STORAGE_ACCOUNT: ${{secrets.AZURE_STORAGE_ACCOUNT}}
+          DUCKDB_AZURE_PUBLIC_CONTAINER_AVAILABLE: 1
+        run: |
+          python3 duckdb/scripts/run_tests_one_by_one.py ./build/release/test/unittest "*test/sql/cloud/*"

.github/workflows/LocalTesting.yml

@@ -0,0 +1,117 @@
+name: Local functional tests
+on: [push, pull_request,repository_dispatch]
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
+  cancel-in-progress: true
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  azurite-tests-linux:
+    name: Azurite (local azure test server) tests (Linux)
+    runs-on: ubuntu-latest
+    container: 'quay.io/pypa/manylinux2014_x86_64'
+    env:
+      VCPKG_TARGET_TRIPLET: 'x64-linux'
+      GEN: Ninja
+      VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
+      AZURE_STORAGE_CONNECTION_STRING: 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;'
+      AZURE_STORAGE_ACCOUNT: devstoreaccount1
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        submodules: 'true'
+
+    - name: install Azure test service
+      run: |
+        yum install -y nodejs npm
+        npm install -g azurite
+        echo -e "[azure-cli]\nname=Azure CLI\nbaseurl=https://packages.microsoft.com/yumrepos/azure-cli\nenabled=1\ngpgcheck=1\ngpgkey=https://packages.microsoft.com/keys/microsoft.asc" | tee /etc/yum.repos.d/azure-cli.repo
+        yum install -y azure-cli
+
+    - name: Setup ManyLinux2014
+      run: |
+        ./duckdb/scripts/setup_manylinux2014.sh general aws-cli ccache ssh python_alias openssl
+
+    - name: Setup Rust for manylinux (dtolnay/rust-toolchain doesn't work due to curl being old here)
+      run: |
+        curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+        echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+
+    - name: Setup vcpkg
+      uses: lukka/[email protected]
+      with:
+        vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6
+
+    - name: Handle OpenSSL dependency for rust build
+      run: |
+        echo "OPENSSL_ROOT_DIR=`pwd`/build/release/vcpkg_installed/x64-linux" >> $GITHUB_ENV
+        echo "OPENSSL_DIR=`pwd`/build/release/vcpkg_installed/x64-linux" >> $GITHUB_ENV
+        echo "OPENSSL_USE_STATIC_LIBS=true" >> $GITHUB_ENV
+
+    # Build extension
+    - name: Build extension
+      env:
+        GEN: ninja
+      run: |
+        make release
+
+    - name: Launch & populate Azure test service
+      run: |
+        azurite > azurite_log.txt 2>&1 &
+        sleep 10
+        ./scripts/upload_test_files_to_azurite.sh
+
+    - name: Test extension
+      run: |
+        make test
+
+    - name: Azure test server log
+      if: always()
+      shell: bash
+      run: |
+        echo "## azurite"
+        cat azurite_log.txt
+
+  generated-tests-linux:
+    name: Generated Tests (Linux)
+    runs-on: ubuntu-latest
+    env:
+      GEN: ninja
+      VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: 'true'
+
+      - name: Install
+        shell: bash
+        run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build
+
+      - name: Setup Ccache
+        uses: hendrikmuhs/ccache-action@main
+        with:
+          key: ${{ github.job }}
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Setup vcpkg
+        uses: lukka/[email protected]
+        with:
+          vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6
+
+      - name: Build
+        shell: bash
+        run: make generate-data
+
+      - name: Test
+        shell: bash
+        run: |
+          GENERATED_DATA_AVAILABLE=1 make test

.gitignore

@@ -7,3 +7,7 @@ testext
 test/python/__pycache__/
 .Rhistory
 data/generated
+__azurite*__.json
+__blobstorage__
+.venv
+.vscode

.gitmodules

@@ -5,4 +5,4 @@
 [submodule "extension-ci-tools"]
 	path = extension-ci-tools
 	url = [email protected]:duckdb/extension-ci-tools.git
-	branch = main
+	branch = main

extension_config.cmake

@@ -9,6 +9,13 @@ duckdb_extension_load(delta
 # Build the httpfs extension to test with s3/http
 duckdb_extension_load(httpfs)
 
+# Build the azure extension to test with azure
+duckdb_extension_load(azure
+        LOAD_TESTS
+        GIT_URL https://github.com/duckdb/duckdb_azure
+        GIT_TAG 49b63dc8cd166952a0a34dfd54e6cfe5b823e05e
+)
+
 # Build the tpch and tpcds extension for testing/benchmarking
 duckdb_extension_load(tpch)
 duckdb_extension_load(tpcds)

scripts/upload_test_files_to_azurite.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# Default Azurite connection string (see: https://github.com/Azure/Azurite)
+conn_string="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;"
+
+# Create container
+az storage container create -n delta-testing-private --connection-string "${conn_string}"
+az storage container create -n delta-testing-public  --connection-string "${conn_string}" --public-access blob
+
+copy_file() {
+  local from="${1}"
+  local to="${2}"
+  az storage blob upload --file "${from}" --name "${to}" --container-name "delta-testing-private" --connection-string "${conn_string}"
+  az storage blob upload --file "${from}" --name "${to}" --container-name "delta-testing-public"  --connection-string "${conn_string}"
+}
+
+cd ./build/release/rust/src/delta_kernel/acceptance/tests/dat/out/reader_tests/generated &&
+while read filepath; do
+    remote_filepath=dat/"$(echo "${filepath}" | cut -c 3-)"
+    copy_file "${filepath}" "${remote_filepath}"
+done < <(find . -type f)