From 6810aa3c24fb28146ff3c26ed2d26cf86f6b18f9 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Mon, 13 Jan 2025 10:50:45 +0100 Subject: [PATCH 1/8] bump duckdb to main --- .github/workflows/MainDistributionPipeline.yml | 6 ++---- duckdb | 2 +- extension-ci-tools | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index 3449ca6..a42cbc0 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -16,14 +16,12 @@ jobs: name: Build extension binaries uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main with: - # pip install duckdb==1.1.4.dev2005 - duckdb_version: b470dea7ee + duckdb_version: main ci_tools_version: main extension_name: delta enable_rust: true exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64_mingw' extra_toolchains: 'python3' - vcpkg_commit: c82f74667287d3dc386bce81e44964370c91a289 duckdb-stable-deploy: name: Deploy extension binaries @@ -32,6 +30,6 @@ jobs: secrets: inherit with: extension_name: delta + duckdb_version: main ci_tools_version: main - duckdb_version: b470dea7ee exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64_mingw' diff --git a/duckdb b/duckdb index b470dea..89bcc3e 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit b470dea7ee47dc2debcc37a4e94976f8eff6670c +Subproject commit 89bcc3e2ce739b1b470afa79818ee03c8cf96fe8 diff --git a/extension-ci-tools b/extension-ci-tools index 916d4ef..4317e39 160000 --- a/extension-ci-tools +++ b/extension-ci-tools @@ -1 +1 @@ -Subproject commit 916d4ef4371068ca98a007378b52582c3e46b4e5 +Subproject commit 4317e39099f4b71d614f00d044aaec651bec6fc9 From e39f308b148923db424e5184f4472d174abad85d Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Thu, 12 Dec 2024 14:26:37 +0100 Subject: [PATCH 2/8] poc hooking up logging --- src/delta_extension.cpp | 4 ++++ src/delta_utils.cpp | 48 +++++++++++++++++++++++++++++++++++++ src/include/delta_utils.hpp | 20 ++++++++++++++++ 3 files changed, 72 insertions(+) diff --git a/src/delta_extension.cpp b/src/delta_extension.cpp index 50ce93d..596f9ab 100644 --- a/src/delta_extension.cpp +++ b/src/delta_extension.cpp @@ -2,6 +2,7 @@ #include "delta_extension.hpp" +#include "delta_utils.hpp" #include "delta_functions.hpp" #include "duckdb.hpp" #include "duckdb/common/exception.hpp" @@ -62,6 +63,9 @@ static void LoadInternal(DatabaseInstance &instance) { "Adds the filtered files to the explain output. Warning: this may change performance of " "delta scan during explain analyze queries.", LogicalType::BOOLEAN, Value(true)); + + LoggerCallback::Initialize(instance); + ffi::enable_event_tracing(LoggerCallback::CallbackEvent, ffi::Level::TRACE); } void DeltaExtension::Load(DuckDB &db) { diff --git a/src/delta_utils.cpp b/src/delta_utils.cpp index d47d99f..89d7e5c 100644 --- a/src/delta_utils.cpp +++ b/src/delta_utils.cpp @@ -4,6 +4,7 @@ #include "duckdb.hpp" #include "duckdb/main/extension_util.hpp" +#include "duckdb/main/database.hpp" #include #include @@ -634,4 +635,51 @@ uintptr_t PredicateVisitor::VisitFilter(const string &col_name, const TableFilte } } +void LoggerCallback::Initialize(DatabaseInstance &db_p) { + auto &instance = GetInstance(); + unique_lock lck(instance.lock); + instance.db = db_p.shared_from_this(); +} + +void LoggerCallback::CallbackEvent(ffi::Event event) { + auto &instance = GetInstance(); + auto db_locked = instance.db.lock(); + if (db_locked) { + auto transformed_log_level = GetDuckDBLogLevel(event.level); + auto log_type = KernelUtils::FromDeltaString(event.target); + string constructed_log_message; + Logger::Log(log_type.c_str(), *db_locked, transformed_log_level, [&]() { + auto message = KernelUtils::FromDeltaString(event.message); + auto file = KernelUtils::FromDeltaString(event.file); + + if (!file.empty()) { + constructed_log_message = StringUtil::Format("%s@%u : %s ", file, event.line, message); + } else { + constructed_log_message = message; + } + + return constructed_log_message.c_str(); + }); + } +} + +LogLevel LoggerCallback::GetDuckDBLogLevel(ffi::Level level) { + switch (level) { + case ffi::Level::TRACE: + case ffi::Level::DEBUGGING: + return LogLevel::DEBUGGING; + case ffi::Level::INFO: + return LogLevel::INFO; + case ffi::Level::WARN: + return LogLevel::WARN; + case ffi::Level::ERROR: + return LogLevel::ERROR; + } +} + +LoggerCallback &LoggerCallback::GetInstance() { + static LoggerCallback instance; + return instance; +} + }; // namespace duckdb diff --git a/src/include/delta_utils.hpp b/src/include/delta_utils.hpp index 540bdb6..b34c197 100644 --- a/src/include/delta_utils.hpp +++ b/src/include/delta_utils.hpp @@ -15,6 +15,7 @@ // TODO: clean up this file as we go namespace duckdb { +class DatabaseInstance; class ExpressionVisitor : public ffi::EngineExpressionVisitor { using FieldList = vector>; @@ -336,4 +337,23 @@ class PredicateVisitor : public ffi::EnginePredicate { uintptr_t VisitFilter(const string &col_name, const TableFilter &filter, ffi::KernelExpressionVisitorState *state); }; +// Singleton class to forward logs to DuckDB +class LoggerCallback { +public: + //! Singleton GetInstance + static LoggerCallback& GetInstance(); + static void Initialize(DatabaseInstance &db); + static void CallbackLogLine(ffi::KernelStringSlice log_line); + static void CallbackEvent(ffi::Event log_line); + + + static LogLevel GetDuckDBLogLevel(ffi::Level); + +protected: + LoggerCallback() {} + + mutex lock; + weak_ptr db; +}; + } // namespace duckdb From 020eb46b3165c2e31842b9486694492340d2a78b Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Mon, 13 Jan 2025 13:16:18 +0100 Subject: [PATCH 3/8] finish delta kernel logging forwarding --- src/delta_extension.cpp | 9 +++-- src/delta_utils.cpp | 37 +++++++++++++++----- src/include/delta_utils.hpp | 5 ++- test/sql/delta_kernel_rs/logging.test | 49 +++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 12 deletions(-) create mode 100644 test/sql/delta_kernel_rs/logging.test diff --git a/src/delta_extension.cpp b/src/delta_extension.cpp index 596f9ab..2d9c03f 100644 --- a/src/delta_extension.cpp +++ b/src/delta_extension.cpp @@ -10,6 +10,7 @@ #include "duckdb/storage/storage_extension.hpp" #include "storage/delta_catalog.hpp" #include "storage/delta_transaction_manager.hpp" +#include "duckdb/main/config.hpp" namespace duckdb { @@ -60,12 +61,16 @@ static void LoadInternal(DatabaseInstance &instance) { config.storage_extensions["delta"] = make_uniq(); config.AddExtensionOption("delta_scan_explain_files_filtered", - "Adds the filtered files to the explain output. Warning: this may change performance of " + "Adds the filtered files to the explain output. Warning: this may impact performance of " "delta scan during explain analyze queries.", LogicalType::BOOLEAN, Value(true)); + config.AddExtensionOption("delta_kernel_logging", + "Forwards the internal logging of the Delta Kernel to the duckdb logger. Warning: this may impact " + "performance even with DuckDB logging disabled.", + LogicalType::BOOLEAN, Value(false), LoggerCallback::DuckDBSettingCallBack); + LoggerCallback::Initialize(instance); - ffi::enable_event_tracing(LoggerCallback::CallbackEvent, ffi::Level::TRACE); } void DeltaExtension::Load(DuckDB &db) { diff --git a/src/delta_utils.cpp b/src/delta_utils.cpp index 89d7e5c..e8ed7c9 100644 --- a/src/delta_utils.cpp +++ b/src/delta_utils.cpp @@ -172,7 +172,7 @@ void ExpressionVisitor::VisitPrimitiveLiteralDouble(void *state, uintptr_t sibli } void ExpressionVisitor::VisitTimestampLiteral(void *state, uintptr_t sibling_list_id, int64_t value) { - auto expression = make_uniq(Value::TIMESTAMPTZ(static_cast(value))); + auto expression = make_uniq(Value::TIMESTAMPTZ(timestamp_tz_t(value))); static_cast(state)->AppendToList(sibling_list_id, std::move(expression)); } @@ -646,14 +646,13 @@ void LoggerCallback::CallbackEvent(ffi::Event event) { auto db_locked = instance.db.lock(); if (db_locked) { auto transformed_log_level = GetDuckDBLogLevel(event.level); - auto log_type = KernelUtils::FromDeltaString(event.target); string constructed_log_message; - Logger::Log(log_type.c_str(), *db_locked, transformed_log_level, [&]() { + Logger::Log("delta.Kernel", *db_locked, transformed_log_level, [&]() { + auto log_type = KernelUtils::FromDeltaString(event.target); auto message = KernelUtils::FromDeltaString(event.message); auto file = KernelUtils::FromDeltaString(event.file); - if (!file.empty()) { - constructed_log_message = StringUtil::Format("%s@%u : %s ", file, event.line, message); + constructed_log_message = StringUtil::Format("[%s] %s@%u : %s ", log_type, file, event.line, message); } else { constructed_log_message = message; } @@ -666,14 +665,15 @@ void LoggerCallback::CallbackEvent(ffi::Event event) { LogLevel LoggerCallback::GetDuckDBLogLevel(ffi::Level level) { switch (level) { case ffi::Level::TRACE: + return LogLevel::LOG_TRACE; case ffi::Level::DEBUGGING: - return LogLevel::DEBUGGING; + return LogLevel::LOG_DEBUG; case ffi::Level::INFO: - return LogLevel::INFO; + return LogLevel::LOG_INFO; case ffi::Level::WARN: - return LogLevel::WARN; + return LogLevel::LOG_WARN; case ffi::Level::ERROR: - return LogLevel::ERROR; + return LogLevel::LOG_ERROR; } } @@ -682,4 +682,23 @@ LoggerCallback &LoggerCallback::GetInstance() { return instance; } +void LoggerCallback::DuckDBSettingCallBack(ClientContext &context, SetScope scope, Value ¶meter) { + Value current_setting; + auto res = context.TryGetCurrentSetting("delta_kernel_logging", current_setting); + + if (res.GetScope() == SettingScope::INVALID) { + throw InternalException("Failed to find setting 'delta_kernel_logging'"); + } + + if (current_setting.GetValue() && !parameter.GetValue()) { + throw InvalidInputException("Can not disable 'delta_kernel_logging' after enabling it. You can disable DuckDB " + "logging with SET enable_logging=false, but there will still be some performance overhead from 'delta_kernel_logging'" + "that can only be mitigated by restarting DuckDB"); + } + + if (!current_setting.GetValue() && parameter.GetValue()) { + ffi::enable_event_tracing(LoggerCallback::CallbackEvent, ffi::Level::TRACE); + } +} + }; // namespace duckdb diff --git a/src/include/delta_utils.hpp b/src/include/delta_utils.hpp index b34c197..f6d9671 100644 --- a/src/include/delta_utils.hpp +++ b/src/include/delta_utils.hpp @@ -340,10 +340,13 @@ class PredicateVisitor : public ffi::EnginePredicate { // Singleton class to forward logs to DuckDB class LoggerCallback { public: + + //! The Callback for the DuckDB setting to hook up Delta Kernel Logging to the DuckDB logger + static void DuckDBSettingCallBack(ClientContext &context, SetScope scope, Value ¶meter); + //! Singleton GetInstance static LoggerCallback& GetInstance(); static void Initialize(DatabaseInstance &db); - static void CallbackLogLine(ffi::KernelStringSlice log_line); static void CallbackEvent(ffi::Event log_line); diff --git a/test/sql/delta_kernel_rs/logging.test b/test/sql/delta_kernel_rs/logging.test new file mode 100644 index 0000000..1909eba --- /dev/null +++ b/test/sql/delta_kernel_rs/logging.test @@ -0,0 +1,49 @@ +# name: test/sql/delta_kernel_rs/logging.test +# description: test that delta kernel log entries are properly propagated to the DuckDB logger +# group: [delta_kernel_rs] + +require parquet + +require delta + +require-env DELTA_KERNEL_TESTS_PATH + +statement ok +set enable_logging=true; + +statement ok +SELECT * FROM delta_scan('${DELTA_KERNEL_TESTS_PATH}/basic_partitioned') + +# No kernel logging available yet: we need to set delta_kernel_logging=true +query I +SELECT count(*) FROM duckdb_logs WHERE starts_with(type, 'delta.Kernel') +---- +0 + +statement ok +set delta_kernel_logging=true; + +statement ok +set logging_level = 'TRACE'; + +statement ok +SELECT * FROM delta_scan('${DELTA_KERNEL_TESTS_PATH}/basic_partitioned') + +# Now we have log! +query I +SELECT count(*) > 50 FROM duckdb_logs WHERE starts_with(type, 'delta.Kernel') +---- +true + +statement ok +set delta_kernel_logging=true; + +statement error +set delta_kernel_logging=false; +---- +Invalid Input Error: Can not disable 'delta_kernel_logging' after enabling it + +mode output_result + +statement ok +FROM duckdb_logs WHERE starts_with(type, 'delta.Kernel') \ No newline at end of file From e5ee3e1b6677c81cec16aa17720141a19a223a83 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Mon, 13 Jan 2025 13:16:52 +0100 Subject: [PATCH 4/8] format --- src/delta_extension.cpp | 11 ++-- src/delta_utils.cpp | 103 +++++++++++++++++----------------- src/include/delta_utils.hpp | 23 ++++---- src/storage/delta_catalog.cpp | 12 ++-- 4 files changed, 75 insertions(+), 74 deletions(-) diff --git a/src/delta_extension.cpp b/src/delta_extension.cpp index 2d9c03f..dac8888 100644 --- a/src/delta_extension.cpp +++ b/src/delta_extension.cpp @@ -65,12 +65,13 @@ static void LoadInternal(DatabaseInstance &instance) { "delta scan during explain analyze queries.", LogicalType::BOOLEAN, Value(true)); - config.AddExtensionOption("delta_kernel_logging", - "Forwards the internal logging of the Delta Kernel to the duckdb logger. Warning: this may impact " - "performance even with DuckDB logging disabled.", - LogicalType::BOOLEAN, Value(false), LoggerCallback::DuckDBSettingCallBack); + config.AddExtensionOption( + "delta_kernel_logging", + "Forwards the internal logging of the Delta Kernel to the duckdb logger. Warning: this may impact " + "performance even with DuckDB logging disabled.", + LogicalType::BOOLEAN, Value(false), LoggerCallback::DuckDBSettingCallBack); - LoggerCallback::Initialize(instance); + LoggerCallback::Initialize(instance); } void DeltaExtension::Load(DuckDB &db) { diff --git a/src/delta_utils.cpp b/src/delta_utils.cpp index e8ed7c9..f76c6e3 100644 --- a/src/delta_utils.cpp +++ b/src/delta_utils.cpp @@ -636,69 +636,70 @@ uintptr_t PredicateVisitor::VisitFilter(const string &col_name, const TableFilte } void LoggerCallback::Initialize(DatabaseInstance &db_p) { - auto &instance = GetInstance(); - unique_lock lck(instance.lock); - instance.db = db_p.shared_from_this(); + auto &instance = GetInstance(); + unique_lock lck(instance.lock); + instance.db = db_p.shared_from_this(); } void LoggerCallback::CallbackEvent(ffi::Event event) { - auto &instance = GetInstance(); - auto db_locked = instance.db.lock(); - if (db_locked) { - auto transformed_log_level = GetDuckDBLogLevel(event.level); - string constructed_log_message; - Logger::Log("delta.Kernel", *db_locked, transformed_log_level, [&]() { - auto log_type = KernelUtils::FromDeltaString(event.target); - auto message = KernelUtils::FromDeltaString(event.message); - auto file = KernelUtils::FromDeltaString(event.file); - if (!file.empty()) { - constructed_log_message = StringUtil::Format("[%s] %s@%u : %s ", log_type, file, event.line, message); - } else { - constructed_log_message = message; - } - - return constructed_log_message.c_str(); - }); - } + auto &instance = GetInstance(); + auto db_locked = instance.db.lock(); + if (db_locked) { + auto transformed_log_level = GetDuckDBLogLevel(event.level); + string constructed_log_message; + Logger::Log("delta.Kernel", *db_locked, transformed_log_level, [&]() { + auto log_type = KernelUtils::FromDeltaString(event.target); + auto message = KernelUtils::FromDeltaString(event.message); + auto file = KernelUtils::FromDeltaString(event.file); + if (!file.empty()) { + constructed_log_message = StringUtil::Format("[%s] %s@%u : %s ", log_type, file, event.line, message); + } else { + constructed_log_message = message; + } + + return constructed_log_message.c_str(); + }); + } } LogLevel LoggerCallback::GetDuckDBLogLevel(ffi::Level level) { - switch (level) { - case ffi::Level::TRACE: - return LogLevel::LOG_TRACE; - case ffi::Level::DEBUGGING: - return LogLevel::LOG_DEBUG; - case ffi::Level::INFO: - return LogLevel::LOG_INFO; - case ffi::Level::WARN: - return LogLevel::LOG_WARN; - case ffi::Level::ERROR: - return LogLevel::LOG_ERROR; - } + switch (level) { + case ffi::Level::TRACE: + return LogLevel::LOG_TRACE; + case ffi::Level::DEBUGGING: + return LogLevel::LOG_DEBUG; + case ffi::Level::INFO: + return LogLevel::LOG_INFO; + case ffi::Level::WARN: + return LogLevel::LOG_WARN; + case ffi::Level::ERROR: + return LogLevel::LOG_ERROR; + } } LoggerCallback &LoggerCallback::GetInstance() { - static LoggerCallback instance; - return instance; + static LoggerCallback instance; + return instance; } void LoggerCallback::DuckDBSettingCallBack(ClientContext &context, SetScope scope, Value ¶meter) { - Value current_setting; - auto res = context.TryGetCurrentSetting("delta_kernel_logging", current_setting); - - if (res.GetScope() == SettingScope::INVALID) { - throw InternalException("Failed to find setting 'delta_kernel_logging'"); - } - - if (current_setting.GetValue() && !parameter.GetValue()) { - throw InvalidInputException("Can not disable 'delta_kernel_logging' after enabling it. You can disable DuckDB " - "logging with SET enable_logging=false, but there will still be some performance overhead from 'delta_kernel_logging'" - "that can only be mitigated by restarting DuckDB"); - } - - if (!current_setting.GetValue() && parameter.GetValue()) { - ffi::enable_event_tracing(LoggerCallback::CallbackEvent, ffi::Level::TRACE); - } + Value current_setting; + auto res = context.TryGetCurrentSetting("delta_kernel_logging", current_setting); + + if (res.GetScope() == SettingScope::INVALID) { + throw InternalException("Failed to find setting 'delta_kernel_logging'"); + } + + if (current_setting.GetValue() && !parameter.GetValue()) { + throw InvalidInputException("Can not disable 'delta_kernel_logging' after enabling it. You can disable DuckDB " + "logging with SET enable_logging=false, but there will still be some performance " + "overhead from 'delta_kernel_logging'" + "that can only be mitigated by restarting DuckDB"); + } + + if (!current_setting.GetValue() && parameter.GetValue()) { + ffi::enable_event_tracing(LoggerCallback::CallbackEvent, ffi::Level::TRACE); + } } }; // namespace duckdb diff --git a/src/include/delta_utils.hpp b/src/include/delta_utils.hpp index f6d9671..5bf4e1d 100644 --- a/src/include/delta_utils.hpp +++ b/src/include/delta_utils.hpp @@ -340,23 +340,22 @@ class PredicateVisitor : public ffi::EnginePredicate { // Singleton class to forward logs to DuckDB class LoggerCallback { public: + //! The Callback for the DuckDB setting to hook up Delta Kernel Logging to the DuckDB logger + static void DuckDBSettingCallBack(ClientContext &context, SetScope scope, Value ¶meter); - //! The Callback for the DuckDB setting to hook up Delta Kernel Logging to the DuckDB logger - static void DuckDBSettingCallBack(ClientContext &context, SetScope scope, Value ¶meter); + //! Singleton GetInstance + static LoggerCallback &GetInstance(); + static void Initialize(DatabaseInstance &db); + static void CallbackEvent(ffi::Event log_line); - //! Singleton GetInstance - static LoggerCallback& GetInstance(); - static void Initialize(DatabaseInstance &db); - static void CallbackEvent(ffi::Event log_line); - - - static LogLevel GetDuckDBLogLevel(ffi::Level); + static LogLevel GetDuckDBLogLevel(ffi::Level); protected: - LoggerCallback() {} + LoggerCallback() { + } - mutex lock; - weak_ptr db; + mutex lock; + weak_ptr db; }; } // namespace duckdb diff --git a/src/storage/delta_catalog.cpp b/src/storage/delta_catalog.cpp index 5ecba54..fb301e2 100644 --- a/src/storage/delta_catalog.cpp +++ b/src/storage/delta_catalog.cpp @@ -60,22 +60,22 @@ bool DeltaCatalog::UseCachedSnapshot() { optional_idx DeltaCatalog::GetCatalogVersion(ClientContext &context) { auto &delta_transaction = DeltaTransaction::Get(context, *this); - idx_t version = DConstants::INVALID_INDEX; + idx_t version = DConstants::INVALID_INDEX; // Option 1: snapshot is cached table-wide auto cached_snapshot = main_schema->GetCachedTable(); if (cached_snapshot) { - version = cached_snapshot->snapshot->GetVersion(); + version = cached_snapshot->snapshot->GetVersion(); } // Option 2: snapshot is cached in transaction if (delta_transaction.table_entry) { - version = delta_transaction.table_entry->snapshot->GetVersion(); + version = delta_transaction.table_entry->snapshot->GetVersion(); } - if (version != DConstants::INVALID_INDEX) { - return version; - } + if (version != DConstants::INVALID_INDEX) { + return version; + } return optional_idx::Invalid(); } From 07955a1b05dc2c620feff3d390abcec6b2b6a7b1 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Mon, 13 Jan 2025 13:22:53 +0100 Subject: [PATCH 5/8] remove forgotten print statement --- test/sql/delta_kernel_rs/logging.test | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/test/sql/delta_kernel_rs/logging.test b/test/sql/delta_kernel_rs/logging.test index 1909eba..68fd5b1 100644 --- a/test/sql/delta_kernel_rs/logging.test +++ b/test/sql/delta_kernel_rs/logging.test @@ -41,9 +41,4 @@ set delta_kernel_logging=true; statement error set delta_kernel_logging=false; ---- -Invalid Input Error: Can not disable 'delta_kernel_logging' after enabling it - -mode output_result - -statement ok -FROM duckdb_logs WHERE starts_with(type, 'delta.Kernel') \ No newline at end of file +Invalid Input Error: Can not disable 'delta_kernel_logging' after enabling it \ No newline at end of file From 89ebdb7267f9158c34a77e4480027cb283515427 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Mon, 13 Jan 2025 13:28:24 +0100 Subject: [PATCH 6/8] re-enable building test extensions --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 4a361b3..7eb8376 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ test_debug: export DELTA_KERNEL_TESTS_PATH=./build/debug/rust/src/delta_kernel/k test_debug: export DAT_PATH=./build/debug/rust/src/delta_kernel/acceptance/tests/dat # Core extensions that we need for testing -#CORE_EXTENSIONS='tpcds;tpch;aws;azure;httpfs' +CORE_EXTENSIONS='tpcds;tpch;aws;azure;httpfs' # Set this flag during building to enable the benchmark runner ifeq (${BUILD_BENCHMARK}, 1) From 0ece6669c56af62f181b6ee57d12978b27a3634b Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Mon, 13 Jan 2025 13:38:16 +0100 Subject: [PATCH 7/8] fix thread-safety issue for logging singleton --- src/delta_utils.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/delta_utils.cpp b/src/delta_utils.cpp index f76c6e3..9cf8337 100644 --- a/src/delta_utils.cpp +++ b/src/delta_utils.cpp @@ -638,7 +638,9 @@ uintptr_t PredicateVisitor::VisitFilter(const string &col_name, const TableFilte void LoggerCallback::Initialize(DatabaseInstance &db_p) { auto &instance = GetInstance(); unique_lock lck(instance.lock); - instance.db = db_p.shared_from_this(); + if (instance.db.expired()) { + instance.db = db_p.shared_from_this(); + } } void LoggerCallback::CallbackEvent(ffi::Event event) { From 7e99954d72ac1f9d9159c658202b7577949002b2 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Mon, 13 Jan 2025 16:29:13 +0100 Subject: [PATCH 8/8] skip musl --- .github/workflows/MainDistributionPipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index a42cbc0..45d26e7 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -20,7 +20,7 @@ jobs: ci_tools_version: main extension_name: delta enable_rust: true - exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64_mingw' + exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64_mingw;linux_amd64_musl' extra_toolchains: 'python3' duckdb-stable-deploy: @@ -32,4 +32,4 @@ jobs: extension_name: delta duckdb_version: main ci_tools_version: main - exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64_mingw' + exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64_mingw;linux_amd64_musl'