From ef1dd70dee920175a559f69fc3ffa388b778c9ea Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Thu, 13 Jun 2024 11:33:00 +0200 Subject: [PATCH] bump delta to c901665b98b --- CMakeLists.txt | 2 +- Makefile | 5 ----- extension-ci-tools | 2 +- scripts/generate_test_data.py | 8 +++++++- src/delta_utils.cpp | 36 ++++++++++++++++++++++++++++++++--- src/functions/delta_scan.cpp | 2 ++ src/include/delta_utils.hpp | 5 +++++ 7 files changed, 49 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a37370c..28ea1d2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -60,7 +60,7 @@ endif() ExternalProject_Add( ${KERNEL_NAME} GIT_REPOSITORY "https://github.com/delta-incubator/delta-kernel-rs" - GIT_TAG 823367e4dc13b627914412ee2ca7933a1c7b822a + GIT_TAG c901665b98b2fed5ff1c713a9666eba9d16ea281 CONFIGURE_COMMAND "" UPDATE_COMMAND "" BUILD_IN_SOURCE 1 diff --git a/Makefile b/Makefile index 05db957..78144e6 100644 --- a/Makefile +++ b/Makefile @@ -14,11 +14,6 @@ test_debug: export DAT_PATH=./build/debug/rust/src/delta_kernel/acceptance/tests # Include the Makefile from extension-ci-tools include extension-ci-tools/makefiles/duckdb_extension.Makefile -reldebug: - mkdir -p build/reldebug && \ - cmake $(GENERATOR) $(BUILD_FLAGS) $(EXT_RELEASE_FLAGS) -DCMAKE_BUILD_TYPE=RelWithDebInfo -S ./duckdb/ -B build/reldebug && \ - cmake --build build/reldebug --config RelWithDebInfo - # Generate some test data to test with generate-data: python3 -m pip install delta-spark duckdb pandas deltalake pyspark delta diff --git a/extension-ci-tools b/extension-ci-tools index 71b8a60..c0cc931 160000 --- a/extension-ci-tools +++ b/extension-ci-tools @@ -1 +1 @@ -Subproject commit 71b8a603ea24b1ac8a2cff134aca28163576548f +Subproject commit c0cc9319492bfa38344c2f28bd35f2304c74cdde diff --git a/scripts/generate_test_data.py b/scripts/generate_test_data.py index 715e882..cb1d2f7 100644 --- a/scripts/generate_test_data.py +++ b/scripts/generate_test_data.py @@ -136,8 +136,14 @@ def generate_test_data_pyspark(name, current_path, input_path, delete_predicate for table in ["customer","lineitem","nation","orders","part","partsupp","region","supplier"]: generate_test_data_pyspark(f"tpch_sf0_01_{table}", f'tpch_sf0_01/{table}', f'{TMP_PATH}/tpch_sf0_01_export/{table}.parquet') +## TPCH SF1 full dataset +con = duckdb.connect() +con.query(f"call dbgen(sf=1); EXPORT DATABASE '{TMP_PATH}/tpch_sf1_export' (FORMAT parquet)") +for table in ["customer","lineitem","nation","orders","part","partsupp","region","supplier"]: + generate_test_data_pyspark(f"tpch_sf1_{table}", f'tpch_sf1/{table}', f'{TMP_PATH}/tpch_sf1_export/{table}.parquet') + ## TPCDS SF0.01 full dataset con = duckdb.connect() con.query(f"call dsdgen(sf=0.01); EXPORT DATABASE '{TMP_PATH}/tpcds_sf0_01_export' (FORMAT parquet)") for table in ["call_center","catalog_page","catalog_returns","catalog_sales","customer","customer_demographics","customer_address","date_dim","household_demographics","inventory","income_band","item","promotion","reason","ship_mode","store","store_returns","store_sales","time_dim","warehouse","web_page","web_returns","web_sales","web_site"]: - generate_test_data_pyspark(f"tpcds_sf0_01_{table}", f'tpcds_sf0_01/{table}', f'{TMP_PATH}/tpcds_sf0_01_export/{table}.parquet') + generate_test_data_pyspark(f"tpcds_sf0_01_{table}", f'tpcds_sf0_01/{table}', f'{TMP_PATH}/tpcds_sf0_01_export/{table}.parquet') \ No newline at end of file diff --git a/src/delta_utils.cpp b/src/delta_utils.cpp index 7f1d3a7..a805d15 100644 --- a/src/delta_utils.cpp +++ b/src/delta_utils.cpp @@ -3,6 +3,7 @@ #include "duckdb.hpp" #include "duckdb/main/extension_util.hpp" #include +#include namespace duckdb { @@ -257,8 +258,24 @@ uintptr_t PredicateVisitor::VisitConstantFilter(const string &col_name, const Co case LogicalType::BIGINT: right = visit_expression_literal_long(state, BigIntValue::Get(value)); break; - - + // case LogicalType::INTEGER: + // right = visit_expression_literal_int(state, IntegerValue::Get(value)); + // break; + // case LogicalType::SMALLINT: + // right = visit_expression_literal_short(state, SmallIntValue::Get(value)); + // break; + // case LogicalType::TINYINT: + // right = visit_expression_literal_byte(state, TinyIntValue::Get(value)); + // break; + // case LogicalType::FLOAT: + // right = visit_expression_literal_float(state, FloatValue::Get(value)); + // break; + // case LogicalType::DOUBLE: + // right = visit_expression_literal_double(state, DoubleValue::Get(value)); + // break; + // case LogicalType::BOOLEAN: + // right = visit_expression_literal_bool(state, BooleanValue::Get(value)); + // break; case LogicalType::VARCHAR: { // WARNING: C++ lifetime extension rules don't protect calls of the form foo(std::string(...).c_str()) auto str = StringValue::Get(value); @@ -266,7 +283,6 @@ uintptr_t PredicateVisitor::VisitConstantFilter(const string &col_name, const Co right = KernelUtils::UnpackResult(maybe_right, "VisitConstantFilter failed to visit_expression_literal_string"); break; } - default: break; // unsupported type } @@ -305,12 +321,26 @@ uintptr_t PredicateVisitor::VisitAndFilter(const string &col_name, const Conjunc return visit_expression_and(state, &eit); } +uintptr_t PredicateVisitor::VisitIsNull(const string &col_name, ffi::KernelExpressionVisitorState *state) { + auto maybe_left = ffi::visit_expression_column(state, KernelUtils::ToDeltaString(col_name), DuckDBEngineError::AllocateError); + uintptr_t left = KernelUtils::UnpackResult(maybe_left, "VisitIsNull failed to visit_expression_column"); + return ffi::visit_expression_is_null(state, left); +} + +uintptr_t PredicateVisitor::VisitIsNotNull(const string &col_name, ffi::KernelExpressionVisitorState *state) { + return ffi::visit_expression_not(state, VisitIsNull(col_name, state)); +} + uintptr_t PredicateVisitor::VisitFilter(const string &col_name, const TableFilter &filter, ffi::KernelExpressionVisitorState* state) { switch (filter.filter_type) { case TableFilterType::CONSTANT_COMPARISON: return VisitConstantFilter(col_name, static_cast(filter), state); case TableFilterType::CONJUNCTION_AND: return VisitAndFilter(col_name, static_cast(filter), state); + // case TableFilterType::IS_NULL: + // return VisitIsNull(col_name, state); + // case TableFilterType::IS_NOT_NULL: + // return VisitIsNotNull(col_name, state); default: return ~0; } diff --git a/src/functions/delta_scan.cpp b/src/functions/delta_scan.cpp index dd2a027..d4320e5 100644 --- a/src/functions/delta_scan.cpp +++ b/src/functions/delta_scan.cpp @@ -31,6 +31,8 @@ static void visit_callback(ffi::NullableCvoid engine_context, struct ffi::Kernel StringUtil::RTrim(path_string, "/"); path_string += "/" + KernelUtils::FromDeltaString(path); + printf("Got File %s\n", path_string.c_str()); + // First we append the file to our resolved files context->resolved_files.push_back(DeltaSnapshot::ToDuckDBPath(path_string)); context->metadata.emplace_back(make_uniq()); diff --git a/src/include/delta_utils.hpp b/src/include/delta_utils.hpp index bcb5f74..37dc289 100644 --- a/src/include/delta_utils.hpp +++ b/src/include/delta_utils.hpp @@ -5,6 +5,7 @@ #include "duckdb/planner/filter/conjunction_filter.hpp" #include "duckdb/common/enum_util.hpp" #include +#include // TODO: clean up this file as we go @@ -140,6 +141,10 @@ class PredicateVisitor : public ffi::EnginePredicate { uintptr_t VisitConstantFilter(const string &col_name, const ConstantFilter &filter, ffi::KernelExpressionVisitorState* state); uintptr_t VisitAndFilter(const string &col_name, const ConjunctionAndFilter &filter, ffi::KernelExpressionVisitorState* state); + + uintptr_t VisitIsNull(const string &col_name, ffi::KernelExpressionVisitorState* state); + uintptr_t VisitIsNotNull(const string &col_name, ffi::KernelExpressionVisitorState* state); + uintptr_t VisitFilter(const string &col_name, const TableFilter &filter, ffi::KernelExpressionVisitorState* state); };