Merge pull request #49 from samansmink/add-minio-support

add support for minio, r2, gcs

.github/workflows/LocalTesting.yml

@@ -18,6 +18,7 @@ jobs:
       VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
       AZURE_STORAGE_CONNECTION_STRING: 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;'
       AZURE_STORAGE_ACCOUNT: devstoreaccount1
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
 
     steps:
     - uses: actions/checkout@v3
@@ -76,6 +77,92 @@ jobs:
         echo "## azurite"
         cat azurite_log.txt
 
+  minio-tests-linux:
+    name: Minio (local S3 test server) tests (Linux)
+    runs-on: ubuntu-latest
+    env:
+      S3_TEST_SERVER_AVAILABLE: 1
+      GEN: ninja
+      VCPKG_TARGET_TRIPLET: x64-linux
+      VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: 'true'
+
+      - name: Checkout DuckDB to version
+        if: ${{ matrix.duckdb_version != '<submodule_version>'}}
+        run: |
+          cd duckdb
+          git checkout ${{ matrix.duckdb_version }}
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install Ninja
+        shell: bash
+        run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build
+
+      - name: Setup Ccache
+        uses: hendrikmuhs/ccache-action@main
+        with:
+          key: ${{ github.job }}
+          save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }}
+
+      - name: Setup vcpkg
+        uses: lukka/[email protected]
+        with:
+          vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6
+
+      - name: Build
+        shell: bash
+        run: make
+
+      - name: Start S3/HTTP test server
+        shell: bash
+        run: |
+          cd duckdb
+          mkdir data/attach_test
+          touch data/attach_test/attach.db
+          sudo ./scripts/install_s3_test_server.sh
+          source ./scripts/run_s3_test_server.sh
+          sleep 30
+
+      - name: Write AWS credentials file
+        shell: bash
+        run: |
+          ./scripts/create_minio_credential_file.sh
+
+      - name: Copy files to minio
+        shell: bash
+        env:
+          DUCKDB_MINIO_TEST_SERVER_AVAILABLE: 1
+          AWS_ACCESS_KEY_ID: minio_duckdb_user
+          AWS_SECRET_ACCESS_KEY: minio_duckdb_user_password
+          AWS_DEFAULT_REGION: eu-west-1
+          AWS_ENDPOINT: duckdb-minio.com:9000
+        run: |
+          ./scripts/upload_test_files_to_minio.sh
+
+      - name: Test
+        shell: bash
+        run: |
+          make test
+
+      - name: Run Env tests
+        shell: bash
+        env:
+          DUCKDB_MINIO_TEST_SERVER_AVAILABLE: 1
+          AWS_ACCESS_KEY_ID: minio_duckdb_user
+          AWS_SECRET_ACCESS_KEY: minio_duckdb_user_password
+          AWS_DEFAULT_REGION: eu-west-1
+          AWS_ENDPOINT: duckdb-minio.com:9000
+        run: |
+          ./build/release/test/unittest "*/test/sql/cloud/minio_local/*"
+
   generated-tests-linux:
     name: Generated Tests (Linux)
     runs-on: ubuntu-latest

extension_config.cmake

@@ -16,6 +16,13 @@ duckdb_extension_load(azure
         GIT_TAG 49b63dc8cd166952a0a34dfd54e6cfe5b823e05e
 )
 
+# Build the aws extension to test with credential providers
+duckdb_extension_load(aws
+        LOAD_TESTS
+        GIT_URL https://github.com/duckdb/duckdb_aws
+        GIT_TAG 3d1f5c8d0127ff7aaf127935721b197e5fdd95e5
+)
+
 # Build the tpch and tpcds extension for testing/benchmarking
 duckdb_extension_load(tpch)
 duckdb_extension_load(tpcds)

scripts/create_minio_credential_file.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+# Warning: overwrites your existing aws credentials file!
+
+# Set the file path for the credentials file
+credentials_file=~/.aws/credentials
+
+# Set the file path for the config file
+config_file=~/.aws/config
+
+# create dir if not already exists
+mkdir -p ~/.aws
+
+# Create the credentials configuration
+credentials_str="[default]
+aws_access_key_id=minio_duckdb_user
+aws_secret_access_key=minio_duckdb_user_password
+
+[minio-testing-2]
+aws_access_key_id=minio_duckdb_user_2
+aws_secret_access_key=minio_duckdb_user_2_password
+
+[minio-testing-invalid]
+aws_access_key_id=minio_duckdb_user_invalid
+aws_secret_access_key=thispasswordiscompletelywrong
+aws_session_token=completelybogussessiontoken
+"
+
+# Write the credentials configuration to the file
+echo "$credentials_str" > "$credentials_file"
+
+# Create the credentials configuration
+config_str="[default]
+region=eu-west-1
+
+[profile minio-testing-2]
+region=eu-west-1
+
+[profile minio-testing-invalid]
+region=the-moon-123
+"
+
+# Write the config to the file
+echo "$config_str" > "$config_file"

scripts/upload_test_files_to_minio.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+aws s3 cp --endpoint-url http://duckdb-minio.com:9000 --recursive ./build/release/rust/src/delta_kernel/acceptance/tests/dat/out/reader_tests/generated "s3://test-bucket/dat"
+aws s3 cp --endpoint-url http://duckdb-minio.com:9000 --recursive ./build/release/rust/src/delta_kernel/acceptance/tests/dat/out/reader_tests/generated "s3://test-bucket-public/dat"

src/functions/delta_scan.cpp

@@ -112,7 +112,14 @@ static ffi::EngineBuilder* CreateBuilder(ClientContext &context, const string &p
     ffi::EngineBuilder* builder;
 
     // For "regular" paths we early out with the default builder config
-    if (!StringUtil::StartsWith(path, "s3://") && !StringUtil::StartsWith(path, "azure://") && !StringUtil::StartsWith(path, "az://") && !StringUtil::StartsWith(path, "abfs://") && !StringUtil::StartsWith(path, "abfss://")) {
+    if (!StringUtil::StartsWith(path, "s3://") &&
+        !StringUtil::StartsWith(path, "gcs://") &&
+        !StringUtil::StartsWith(path, "gs://") &&
+        !StringUtil::StartsWith(path, "r2://") &&
+        !StringUtil::StartsWith(path, "azure://") &&
+        !StringUtil::StartsWith(path, "az://") &&
+        !StringUtil::StartsWith(path, "abfs://") &&
+        !StringUtil::StartsWith(path, "abfss://")) {
         auto interface_builder_res = ffi::get_engine_builder(KernelUtils::ToDeltaString(path), DuckDBEngineError::AllocateError);
         return KernelUtils::UnpackResult(interface_builder_res, "get_engine_interface_builder for path " + path);
     }
@@ -130,6 +137,33 @@ static ffi::EngineBuilder* CreateBuilder(ClientContext &context, const string &p
         bucket = path.substr(5, end_of_container-5);
         path_in_bucket = path.substr(end_of_container);
         secret_type = "s3";
+    } else if (StringUtil::StartsWith(path, "gcs://")) {
+        auto end_of_container = path.find('/',6);
+
+        if(end_of_container == string::npos) {
+            throw IOException("Invalid gcs url passed to delta scan: %s", path);
+        }
+        bucket = path.substr(6, end_of_container-6);
+        path_in_bucket = path.substr(end_of_container);
+        secret_type = "gcs";
+    } else if (StringUtil::StartsWith(path, "gs://")) {
+        auto end_of_container = path.find('/',5);
+
+        if(end_of_container == string::npos) {
+            throw IOException("Invalid gcs url passed to delta scan: %s", path);
+        }
+        bucket = path.substr(5, end_of_container-5);
+        path_in_bucket = path.substr(end_of_container);
+        secret_type = "gcs";
+    } else if (StringUtil::StartsWith(path, "r2://")) {
+        auto end_of_container = path.find('/',5);
+
+        if(end_of_container == string::npos) {
+            throw IOException("Invalid gcs url passed to delta scan: %s", path);
+        }
+        bucket = path.substr(5, end_of_container-5);
+        path_in_bucket = path.substr(end_of_container);
+        secret_type = "r2";
     } else if ((StringUtil::StartsWith(path, "azure://")) || (StringUtil::StartsWith(path, "abfss://"))) {
         auto end_of_container = path.find('/',8);
 
@@ -159,8 +193,18 @@ static ffi::EngineBuilder* CreateBuilder(ClientContext &context, const string &p
         secret_type = "azure";
     }
 
-    auto interface_builder_res = ffi::get_engine_builder(KernelUtils::ToDeltaString(path), DuckDBEngineError::AllocateError);
-    builder = KernelUtils::UnpackResult(interface_builder_res, "get_engine_interface_builder for path " + path);
+    // We need to substitute DuckDB's usage of s3 and r2 paths because delta kernel needs to just interpret them as s3 protocol servers.
+    string cleaned_path;
+    if (StringUtil::StartsWith(path, "r2://") || StringUtil::StartsWith(path, "gs://") ) {
+        cleaned_path = "s3://" + path.substr(5);
+    } else if (StringUtil::StartsWith(path, "gcs://")) {
+        cleaned_path = "s3://" + path.substr(6);
+    } else {
+        cleaned_path = path;
+    }
+
+    auto interface_builder_res = ffi::get_engine_builder(KernelUtils::ToDeltaString(cleaned_path), DuckDBEngineError::AllocateError);
+    builder = KernelUtils::UnpackResult(interface_builder_res, "get_engine_interface_builder for path " + cleaned_path);
 
     // For S3 or Azure paths we need to trim the url, set the container, and fetch a potential secret
     auto &secret_manager = SecretManager::Get(context);
@@ -170,18 +214,24 @@ static ffi::EngineBuilder* CreateBuilder(ClientContext &context, const string &p
 
     // No secret: nothing left to do here!
     if (!secret_match.HasMatch()) {
+        if (StringUtil::StartsWith(path, "r2://") || StringUtil::StartsWith(path, "gs://") || StringUtil::StartsWith(path, "gcs://")) {
+            throw NotImplementedException("Can not scan a gcs:// gs:// or r2:// url without a secret providing its endpoint currently. Please create an R2 or GCS secret containing the credentials for this endpoint and try again.");
+        }
+
         return builder;
     }
     const auto &kv_secret = dynamic_cast<const KeyValueSecret &>(*secret_match.secret_entry->secret);
 
-
     // Here you would need to add the logic for setting the builder options for Azure
     // This is just a placeholder and will need to be replaced with the actual logic
-    if (secret_type == "s3") {
+    if (secret_type == "s3" || secret_type == "gcs" || secret_type == "r2") {
         auto key_id = kv_secret.TryGetValue("key_id").ToString();
         auto secret = kv_secret.TryGetValue("secret").ToString();
         auto session_token = kv_secret.TryGetValue("session_token").ToString();
         auto region = kv_secret.TryGetValue("region").ToString();
+        auto endpoint = kv_secret.TryGetValue("endpoint").ToString();
+        auto use_ssl = kv_secret.TryGetValue("use_ssl").ToString();
+        auto url_style = kv_secret.TryGetValue("url_style").ToString();
 
         if (key_id.empty() && secret.empty()) {
             ffi::set_builder_option(builder, KernelUtils::ToDeltaString("skip_signature"), KernelUtils::ToDeltaString("true"));
@@ -196,6 +246,21 @@ static ffi::EngineBuilder* CreateBuilder(ClientContext &context, const string &p
         if (!session_token.empty()) {
             ffi::set_builder_option(builder, KernelUtils::ToDeltaString("aws_session_token"), KernelUtils::ToDeltaString(session_token));
         }
+        if (!endpoint.empty() && endpoint != "s3.amazonaws.com") {
+            if(!StringUtil::StartsWith(endpoint, "https://") && !StringUtil::StartsWith(endpoint, "http://")) {
+                if(use_ssl == "1" || use_ssl == "NULL") {
+                    endpoint = "https://" + endpoint;
+                } else {
+                    endpoint = "http://" + endpoint;
+                }
+            }
+
+            if (StringUtil::StartsWith(endpoint, "http://")) {
+                ffi::set_builder_option(builder, KernelUtils::ToDeltaString("allow_http"), KernelUtils::ToDeltaString("true"));
+            }
+            ffi::set_builder_option(builder, KernelUtils::ToDeltaString("aws_endpoint"), KernelUtils::ToDeltaString(endpoint));
+        }
+
         ffi::set_builder_option(builder, KernelUtils::ToDeltaString("aws_region"), KernelUtils::ToDeltaString(region));
 
     } else if (secret_type == "azure") {

test/sql/cloud/minio_local/gcs_r2.test

@@ -0,0 +1,93 @@
+# name: test/sql/cloud/minio_local/gcs_r2.test
+# description: test delta extension with GCS and R2
+# group: [aws]
+
+require httpfs
+
+require parquet
+
+require delta
+
+require aws
+
+require-env DUCKDB_MINIO_TEST_SERVER_AVAILABLE
+
+require-env AWS_ACCESS_KEY_ID
+
+require-env AWS_SECRET_ACCESS_KEY
+
+require-env AWS_DEFAULT_REGION
+
+require-env AWS_ENDPOINT
+
+statement ok
+set secret_directory='__TEST_DIR__/minio_local_gcs_env'
+
+statement error
+FROM delta_scan('gcs://test-bucket/dat/all_primitive_types/delta')
+----
+Can not scan a gcs:// gs:// or r2:// url without a secret providing its endpoint currently. Please create an R2 or GCS secret containing the credentials for this endpoint and try again.
+
+statement error
+FROM delta_scan('gs://test-bucket/dat/all_primitive_types/delta')
+----
+Can not scan a gcs:// gs:// or r2:// url without a secret providing its endpoint currently. Please create an R2 or GCS secret containing the credentials for this endpoint and try again.
+
+statement error
+FROM delta_scan('r2://test-bucket/dat/all_primitive_types/delta')
+----
+Can not scan a gcs:// gs:// or r2:// url without a secret providing its endpoint currently. Please create an R2 or GCS secret containing the credentials for this endpoint and try again.
+
+# create a fake gcs secret
+statement ok
+CREATE SECRET (
+    TYPE GCS,
+	KEY_ID '${AWS_ACCESS_KEY_ID}',
+	SECRET '${AWS_SECRET_ACCESS_KEY}',
+	REGION '${AWS_DEFAULT_REGION}',
+	ENDPOINT '${AWS_ENDPOINT}',
+	USE_SSL false 
+)
+
+query I
+SELECT int32
+FROM delta_scan('gcs://test-bucket-public/dat/all_primitive_types/delta')
+----
+0
+1
+2
+3
+4
+
+query I
+SELECT int32
+FROM delta_scan('gs://test-bucket-public/dat/all_primitive_types/delta')
+----
+0
+1
+2
+3
+4
+
+# create a fake r2 secret
+statement ok
+CREATE SECRET s1 (
+    TYPE R2,
+    PROVIDER config,
+    account_id 'some_bogus_account',
+    KEY_ID '${AWS_ACCESS_KEY_ID}',
+	SECRET '${AWS_SECRET_ACCESS_KEY}',
+	REGION '${AWS_DEFAULT_REGION}',
+	ENDPOINT '${AWS_ENDPOINT}',
+	USE_SSL false 
+)
+
+query I
+SELECT int32
+FROM delta_scan('r2://test-bucket-public/dat/all_primitive_types/delta')
+----
+0
+1
+2
+3
+4