From 1710ff9908aecaa9ed8f113b5ab293a7a00b388b Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Tue, 14 Jan 2025 02:49:51 -0500 Subject: [PATCH 01/13] Merge is done --- CMakeLists.txt | 20 +- src/iceberg_extension.cpp | 147 ++++++++ src/include/storage/uc_catalog.hpp | 87 +++++ src/include/storage/uc_catalog_set.hpp | 54 +++ src/include/storage/uc_schema_entry.hpp | 53 +++ src/include/storage/uc_schema_set.hpp | 28 ++ src/include/storage/uc_table_entry.hpp | 53 +++ src/include/storage/uc_table_set.hpp | 43 +++ src/include/storage/uc_transaction.hpp | 42 +++ .../storage/uc_transaction_manager.hpp | 33 ++ src/include/uc_api.hpp | 61 ++++ src/include/uc_utils.hpp | 33 ++ src/storage/uc_catalog.cpp | 102 ++++++ src/storage/uc_catalog_set.cpp | 69 ++++ src/storage/uc_clear_cache.cpp | 50 +++ src/storage/uc_schema_entry.cpp | 161 +++++++++ src/storage/uc_schema_set.cpp | 38 +++ src/storage/uc_table_entry.cpp | 147 ++++++++ src/storage/uc_table_set.cpp | 87 +++++ src/storage/uc_transaction.cpp | 61 ++++ src/storage/uc_transaction_manager.cpp | 40 +++ src/uc_api.cpp | 320 ++++++++++++++++++ src/uc_utils.cpp | 193 +++++++++++ vcpkg.json | 1 + 24 files changed, 1920 insertions(+), 3 deletions(-) create mode 100644 src/include/storage/uc_catalog.hpp create mode 100644 src/include/storage/uc_catalog_set.hpp create mode 100644 src/include/storage/uc_schema_entry.hpp create mode 100644 src/include/storage/uc_schema_set.hpp create mode 100644 src/include/storage/uc_table_entry.hpp create mode 100644 src/include/storage/uc_table_set.hpp create mode 100644 src/include/storage/uc_transaction.hpp create mode 100644 src/include/storage/uc_transaction_manager.hpp create mode 100644 src/include/uc_api.hpp create mode 100644 src/include/uc_utils.hpp create mode 100644 src/storage/uc_catalog.cpp create mode 100644 src/storage/uc_catalog_set.cpp create mode 100644 src/storage/uc_clear_cache.cpp create mode 100644 src/storage/uc_schema_entry.cpp create mode 100644 src/storage/uc_schema_set.cpp create mode 100644 src/storage/uc_table_entry.cpp create mode 100644 src/storage/uc_table_set.cpp create mode 100644 src/storage/uc_transaction.cpp create mode 100644 src/storage/uc_transaction_manager.cpp create mode 100644 src/uc_api.cpp create mode 100644 src/uc_utils.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index fe79363..db7837f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,12 +13,24 @@ include_directories(src/include) set(EXTENSION_SOURCES src/iceberg_extension.cpp src/iceberg_functions.cpp + src/uc_api.cpp + src/uc_utils.cpp src/common/utils.cpp src/common/schema.cpp src/common/iceberg.cpp src/iceberg_functions/iceberg_snapshots.cpp src/iceberg_functions/iceberg_scan.cpp - src/iceberg_functions/iceberg_metadata.cpp) + src/iceberg_functions/iceberg_metadata.cpp + src/storage/uc_catalog.cpp + src/storage/uc_catalog_set.cpp + src/storage/uc_clear_cache.cpp + src/storage/uc_schema_entry.cpp + src/storage/uc_schema_set.cpp + src/storage/uc_table_entry.cpp + src/storage/uc_table_set.cpp + src/storage/uc_transaction.cpp + src/storage/uc_transaction_manager.cpp + ) add_library(${EXTENSION_NAME} STATIC ${EXTENSION_SOURCES}) @@ -73,10 +85,12 @@ target_link_libraries( Snappy::snappy ZLIB::ZLIB) +find_package(CURL REQUIRED) + # Link dependencies into extension -target_link_libraries(${EXTENSION_NAME} PUBLIC optimized avro_static_release +target_link_libraries(${EXTENSION_NAME} PUBLIC optimized avro_static_release CURL::libcurl debug avro_static_debug) -target_link_libraries(${TARGET_NAME}_loadable_extension optimized +target_link_libraries(${TARGET_NAME}_loadable_extension optimized CURL::libcurl avro_static_release debug avro_static_debug) install( diff --git a/src/iceberg_extension.cpp b/src/iceberg_extension.cpp index f82c6b4..2d4a3b9 100644 --- a/src/iceberg_extension.cpp +++ b/src/iceberg_extension.cpp @@ -1,20 +1,153 @@ #define DUCKDB_EXTENSION_MAIN #include "iceberg_extension.hpp" +#include "storage/uc_catalog.hpp" +#include "storage/uc_transaction_manager.hpp" #include "duckdb.hpp" +#include "duckdb/main/secret/secret_manager.hpp" #include "duckdb/common/exception.hpp" #include "duckdb/common/string_util.hpp" #include "duckdb/function/scalar_function.hpp" +#include "duckdb/main/extension_util.hpp" #include "duckdb/catalog/catalog_entry/macro_catalog_entry.hpp" #include "duckdb/catalog/default/default_functions.hpp" +#include "duckdb/parser/parsed_data/create_scalar_function_info.hpp" +#include "duckdb/parser/parsed_data/attach_info.hpp" +#include "duckdb/storage/storage_extension.hpp" #include "iceberg_functions.hpp" #include "yyjson.hpp" #include "duckdb/main/extension_util.hpp" #include +#include "uc_api.hpp" namespace duckdb { +static unique_ptr CreatePolarisSecretFunction(ClientContext &, CreateSecretInput &input) { + // apply any overridden settings + vector prefix_paths; + auto result = make_uniq(prefix_paths, "iceberg", "config", input.name); + + for (const auto &named_param : input.options) { + auto lower_name = StringUtil::Lower(named_param.first); + + if (lower_name == "client_id" || + lower_name == "client_secret" || + lower_name == "endpoint" || + lower_name == "aws_region") { + result->secret_map[lower_name] = named_param.second.ToString(); + } else { + throw InternalException("Unknown named parameter passed to CreateUCSecretFunction: " + lower_name); + } + } + + // Get token from catalog + result->secret_map["token"] = UCAPI::GetToken( + result->secret_map["client_id"].ToString(), + result->secret_map["client_secret"].ToString(), + result->secret_map["endpoint"].ToString()); + + //! Set redact keys + result->redact_keys = {"token", "client_id", "client_secret"}; + + return std::move(result); +} + +static void SetPolarisSecretParameters(CreateSecretFunction &function) { + function.named_parameters["client_id"] = LogicalType::VARCHAR; + function.named_parameters["client_secret"] = LogicalType::VARCHAR; + function.named_parameters["endpoint"] = LogicalType::VARCHAR; + function.named_parameters["aws_region"] = LogicalType::VARCHAR; + function.named_parameters["token"] = LogicalType::VARCHAR; +} + +unique_ptr GetSecret(ClientContext &context, const string &secret_name) { + auto &secret_manager = SecretManager::Get(context); + auto transaction = CatalogTransaction::GetSystemCatalogTransaction(context); + // FIXME: this should be adjusted once the `GetSecretByName` API supports this + // use case + auto secret_entry = secret_manager.GetSecretByName(transaction, secret_name, "memory"); + if (secret_entry) { + return secret_entry; + } + secret_entry = secret_manager.GetSecretByName(transaction, secret_name, "local_file"); + if (secret_entry) { + return secret_entry; + } + return nullptr; +} + +static unique_ptr PolarisCatalogAttach(StorageExtensionInfo *storage_info, ClientContext &context, + AttachedDatabase &db, const string &name, AttachInfo &info, + AccessMode access_mode) { + UCCredentials credentials; + + // check if we have a secret provided + string secret_name; + for (auto &entry : info.options) { + auto lower_name = StringUtil::Lower(entry.first); + if (lower_name == "type" || lower_name == "read_only") { + // already handled + } else if (lower_name == "secret") { + secret_name = entry.second.ToString(); + } else { + throw BinderException("Unrecognized option for PC attach: %s", entry.first); + } + } + + // if no secret is specified we default to the unnamed mysql secret, if it + // exists + bool explicit_secret = !secret_name.empty(); + if (!explicit_secret) { + // look up settings from the default unnamed mysql secret if none is + // provided + secret_name = "__default_iceberg"; + } + + string connection_string = info.path; + auto secret_entry = GetSecret(context, secret_name); + if (secret_entry) { + // secret found - read data + const auto &kv_secret = dynamic_cast(*secret_entry->secret); + string new_connection_info; + + Value token_val = kv_secret.TryGetValue("token"); + if (token_val.IsNull()) { + throw std::runtime_error("Token is blank"); + } + credentials.token = token_val.ToString(); + + Value endpoint_val = kv_secret.TryGetValue("endpoint"); + credentials.endpoint = endpoint_val.IsNull() ? "" : endpoint_val.ToString(); + StringUtil::RTrim(credentials.endpoint, "/"); + + Value aws_region_val = kv_secret.TryGetValue("aws_region"); + credentials.aws_region = endpoint_val.IsNull() ? "" : aws_region_val.ToString(); + + } else if (explicit_secret) { + // secret not found and one was explicitly provided - throw an error + throw BinderException("Secret with name \"%s\" not found", secret_name); + } + + // TODO: Check catalog with name actually exists! + + return make_uniq(db, info.path, access_mode, credentials); +} + +static unique_ptr CreateTransactionManager(StorageExtensionInfo *storage_info, AttachedDatabase &db, + Catalog &catalog) { + auto &uc_catalog = catalog.Cast(); + return make_uniq(db, uc_catalog); +} + +class UCCatalogStorageExtension : public StorageExtension { +public: + UCCatalogStorageExtension() { + attach = PolarisCatalogAttach; + create_transaction_manager = CreateTransactionManager; + } +}; + static void LoadInternal(DatabaseInstance &instance) { auto &config = DBConfig::GetConfig(instance); @@ -34,6 +167,20 @@ static void LoadInternal(DatabaseInstance &instance) { for (auto &fun : IcebergFunctions::GetScalarFunctions()) { ExtensionUtil::RegisterFunction(instance, fun); } + + UCAPI::InitializeCurl(); + + SecretType secret_type; + secret_type.name = "iceberg"; + secret_type.deserializer = KeyValueSecret::Deserialize; + secret_type.default_provider = "config"; + + ExtensionUtil::RegisterSecretType(instance, secret_type); + CreateSecretFunction secret_function = {"iceberg", "config", CreatePolarisSecretFunction}; + SetPolarisSecretParameters(secret_function); + ExtensionUtil::RegisterFunction(instance, secret_function); + + config.storage_extensions["iceberg"] = make_uniq(); } void IcebergExtension::Load(DuckDB &db) { diff --git a/src/include/storage/uc_catalog.hpp b/src/include/storage/uc_catalog.hpp new file mode 100644 index 0000000..22df4f0 --- /dev/null +++ b/src/include/storage/uc_catalog.hpp @@ -0,0 +1,87 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// storage/uc_catalog.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/function/table_function.hpp" +#include "duckdb/common/enums/access_mode.hpp" +#include "storage/uc_schema_set.hpp" + +namespace duckdb { +class UCSchemaEntry; + +struct UCCredentials { + string endpoint; + string client_id; + string client_secret; + // required to query s3 tables + string aws_region; + // Catalog generates the token using client id & secret + string token; +}; + +class UCClearCacheFunction : public TableFunction { +public: + UCClearCacheFunction(); + + static void ClearCacheOnSetting(ClientContext &context, SetScope scope, Value ¶meter); +}; + +class UCCatalog : public Catalog { +public: + explicit UCCatalog(AttachedDatabase &db_p, const string &internal_name, AccessMode access_mode, + UCCredentials credentials); + ~UCCatalog(); + + string internal_name; + AccessMode access_mode; + UCCredentials credentials; + +public: + void Initialize(bool load_builtin) override; + string GetCatalogType() override { + return "iceberg"; + } + + optional_ptr CreateSchema(CatalogTransaction transaction, CreateSchemaInfo &info) override; + + void ScanSchemas(ClientContext &context, std::function callback) override; + + optional_ptr GetSchema(CatalogTransaction transaction, const string &schema_name, + OnEntryNotFound if_not_found, + QueryErrorContext error_context = QueryErrorContext()) override; + + unique_ptr PlanInsert(ClientContext &context, LogicalInsert &op, + unique_ptr plan) override; + unique_ptr PlanCreateTableAs(ClientContext &context, LogicalCreateTable &op, + unique_ptr plan) override; + unique_ptr PlanDelete(ClientContext &context, LogicalDelete &op, + unique_ptr plan) override; + unique_ptr PlanUpdate(ClientContext &context, LogicalUpdate &op, + unique_ptr plan) override; + unique_ptr BindCreateIndex(Binder &binder, CreateStatement &stmt, TableCatalogEntry &table, + unique_ptr plan) override; + + DatabaseSize GetDatabaseSize(ClientContext &context) override; + + //! Whether or not this is an in-memory PC database + bool InMemory() override; + string GetDBPath() override; + + void ClearCache(); + +private: + void DropSchema(ClientContext &context, DropInfo &info) override; + +private: + UCSchemaSet schemas; + string default_schema; +}; + +} // namespace duckdb diff --git a/src/include/storage/uc_catalog_set.hpp b/src/include/storage/uc_catalog_set.hpp new file mode 100644 index 0000000..8cd9ba3 --- /dev/null +++ b/src/include/storage/uc_catalog_set.hpp @@ -0,0 +1,54 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// storage/uc_catalog_set.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/transaction/transaction.hpp" +#include "duckdb/common/case_insensitive_map.hpp" +#include "duckdb/common/mutex.hpp" + +namespace duckdb { +struct DropInfo; +class UCSchemaEntry; +class UCTransaction; + +class UCCatalogSet { +public: + UCCatalogSet(Catalog &catalog); + + optional_ptr GetEntry(ClientContext &context, const string &name); + virtual void DropEntry(ClientContext &context, DropInfo &info); + void Scan(ClientContext &context, const std::function &callback); + virtual optional_ptr CreateEntry(unique_ptr entry); + void ClearEntries(); + +protected: + virtual void LoadEntries(ClientContext &context) = 0; + + void EraseEntryInternal(const string &name); + +protected: + Catalog &catalog; + +private: + mutex entry_lock; + case_insensitive_map_t> entries; + bool is_loaded; +}; + +class UCInSchemaSet : public UCCatalogSet { +public: + UCInSchemaSet(UCSchemaEntry &schema); + + optional_ptr CreateEntry(unique_ptr entry) override; + +protected: + UCSchemaEntry &schema; +}; + +} // namespace duckdb diff --git a/src/include/storage/uc_schema_entry.hpp b/src/include/storage/uc_schema_entry.hpp new file mode 100644 index 0000000..b9d11f1 --- /dev/null +++ b/src/include/storage/uc_schema_entry.hpp @@ -0,0 +1,53 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// storage/uc_schema_entry.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "uc_api.hpp" +#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" +#include "storage/uc_table_set.hpp" + +namespace duckdb { +class UCTransaction; + +class UCSchemaEntry : public SchemaCatalogEntry { +public: + UCSchemaEntry(Catalog &catalog, CreateSchemaInfo &info); + ~UCSchemaEntry() override; + + unique_ptr schema_data; + +public: + optional_ptr CreateTable(CatalogTransaction transaction, BoundCreateTableInfo &info) override; + optional_ptr CreateFunction(CatalogTransaction transaction, CreateFunctionInfo &info) override; + optional_ptr CreateIndex(CatalogTransaction transaction, CreateIndexInfo &info, + TableCatalogEntry &table) override; + optional_ptr CreateView(CatalogTransaction transaction, CreateViewInfo &info) override; + optional_ptr CreateSequence(CatalogTransaction transaction, CreateSequenceInfo &info) override; + optional_ptr CreateTableFunction(CatalogTransaction transaction, + CreateTableFunctionInfo &info) override; + optional_ptr CreateCopyFunction(CatalogTransaction transaction, + CreateCopyFunctionInfo &info) override; + optional_ptr CreatePragmaFunction(CatalogTransaction transaction, + CreatePragmaFunctionInfo &info) override; + optional_ptr CreateCollation(CatalogTransaction transaction, CreateCollationInfo &info) override; + optional_ptr CreateType(CatalogTransaction transaction, CreateTypeInfo &info) override; + void Alter(CatalogTransaction transaction, AlterInfo &info) override; + void Scan(ClientContext &context, CatalogType type, const std::function &callback) override; + void Scan(CatalogType type, const std::function &callback) override; + void DropEntry(ClientContext &context, DropInfo &info) override; + optional_ptr GetEntry(CatalogTransaction transaction, CatalogType type, const string &name) override; + +private: + UCCatalogSet &GetCatalogSet(CatalogType type); + +private: + UCTableSet tables; +}; + +} // namespace duckdb diff --git a/src/include/storage/uc_schema_set.hpp b/src/include/storage/uc_schema_set.hpp new file mode 100644 index 0000000..38bfb2f --- /dev/null +++ b/src/include/storage/uc_schema_set.hpp @@ -0,0 +1,28 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// storage/uc_schema_set.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "storage/uc_catalog_set.hpp" +#include "storage/uc_schema_entry.hpp" + +namespace duckdb { +struct CreateSchemaInfo; + +class UCSchemaSet : public UCCatalogSet { +public: + explicit UCSchemaSet(Catalog &catalog); + +public: + optional_ptr CreateSchema(ClientContext &context, CreateSchemaInfo &info); + +protected: + void LoadEntries(ClientContext &context) override; +}; + +} // namespace duckdb diff --git a/src/include/storage/uc_table_entry.hpp b/src/include/storage/uc_table_entry.hpp new file mode 100644 index 0000000..50b2528 --- /dev/null +++ b/src/include/storage/uc_table_entry.hpp @@ -0,0 +1,53 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// storage/uc_table_entry.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "uc_api.hpp" +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" +#include "duckdb/parser/parsed_data/create_table_info.hpp" + +namespace duckdb { + +struct UCTableInfo { + UCTableInfo() { + create_info = make_uniq(); + } + UCTableInfo(const string &schema, const string &table) { + create_info = make_uniq(string(), schema, table); + } + UCTableInfo(const SchemaCatalogEntry &schema, const string &table) { + create_info = make_uniq((SchemaCatalogEntry &)schema, table); + } + + const string &GetTableName() const { + return create_info->table; + } + + unique_ptr create_info; +}; + +class UCTableEntry : public TableCatalogEntry { +public: + UCTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTableInfo &info); + UCTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, UCTableInfo &info); + + unique_ptr table_data; + +public: + unique_ptr GetStatistics(ClientContext &context, column_t column_id) override; + + TableFunction GetScanFunction(ClientContext &context, unique_ptr &bind_data) override; + + TableStorageInfo GetStorageInfo(ClientContext &context) override; + + void BindUpdateConstraints(Binder &binder, LogicalGet &get, LogicalProjection &proj, LogicalUpdate &update, + ClientContext &context) override; +}; + +} // namespace duckdb diff --git a/src/include/storage/uc_table_set.hpp b/src/include/storage/uc_table_set.hpp new file mode 100644 index 0000000..fd8ec0b --- /dev/null +++ b/src/include/storage/uc_table_set.hpp @@ -0,0 +1,43 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// storage/uc_table_set.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "storage/uc_catalog_set.hpp" +#include "storage/uc_table_entry.hpp" + +namespace duckdb { +struct CreateTableInfo; +class UCResult; +class UCSchemaEntry; + +class UCTableSet : public UCInSchemaSet { +public: + explicit UCTableSet(UCSchemaEntry &schema); + +public: + optional_ptr CreateTable(ClientContext &context, BoundCreateTableInfo &info); + + static unique_ptr GetTableInfo(ClientContext &context, UCSchemaEntry &schema, + const string &table_name); + optional_ptr RefreshTable(ClientContext &context, const string &table_name); + + void AlterTable(ClientContext &context, AlterTableInfo &info); + +protected: + void LoadEntries(ClientContext &context) override; + + void AlterTable(ClientContext &context, RenameTableInfo &info); + void AlterTable(ClientContext &context, RenameColumnInfo &info); + void AlterTable(ClientContext &context, AddColumnInfo &info); + void AlterTable(ClientContext &context, RemoveColumnInfo &info); + + static void AddColumn(ClientContext &context, UCResult &result, UCTableInfo &table_info, idx_t column_offset = 0); +}; + +} // namespace duckdb diff --git a/src/include/storage/uc_transaction.hpp b/src/include/storage/uc_transaction.hpp new file mode 100644 index 0000000..86a1423 --- /dev/null +++ b/src/include/storage/uc_transaction.hpp @@ -0,0 +1,42 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// storage/uc_transaction.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/transaction/transaction.hpp" + +namespace duckdb { +class UCCatalog; +class UCSchemaEntry; +class UCTableEntry; + +enum class UCTransactionState { TRANSACTION_NOT_YET_STARTED, TRANSACTION_STARTED, TRANSACTION_FINISHED }; + +class UCTransaction : public Transaction { +public: + UCTransaction(UCCatalog &uc_catalog, TransactionManager &manager, ClientContext &context); + ~UCTransaction() override; + + void Start(); + void Commit(); + void Rollback(); + + // UCConnection &GetConnection(); + // unique_ptr Query(const string &query); + static UCTransaction &Get(ClientContext &context, Catalog &catalog); + AccessMode GetAccessMode() const { + return access_mode; + } + +private: + // UCConnection connection; + UCTransactionState transaction_state; + AccessMode access_mode; +}; + +} // namespace duckdb diff --git a/src/include/storage/uc_transaction_manager.hpp b/src/include/storage/uc_transaction_manager.hpp new file mode 100644 index 0000000..b368f51 --- /dev/null +++ b/src/include/storage/uc_transaction_manager.hpp @@ -0,0 +1,33 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// storage/uc_transaction_manager.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/transaction/transaction_manager.hpp" +#include "storage/uc_catalog.hpp" +#include "storage/uc_transaction.hpp" + +namespace duckdb { + +class UCTransactionManager : public TransactionManager { +public: + UCTransactionManager(AttachedDatabase &db_p, UCCatalog &uc_catalog); + + Transaction &StartTransaction(ClientContext &context) override; + ErrorData CommitTransaction(ClientContext &context, Transaction &transaction) override; + void RollbackTransaction(Transaction &transaction) override; + + void Checkpoint(ClientContext &context, bool force = false) override; + +private: + UCCatalog &uc_catalog; + mutex transaction_lock; + reference_map_t> transactions; +}; + +} // namespace duckdb diff --git a/src/include/uc_api.hpp b/src/include/uc_api.hpp new file mode 100644 index 0000000..a65f208 --- /dev/null +++ b/src/include/uc_api.hpp @@ -0,0 +1,61 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// src/include/uc_api.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/common/types.hpp" + +namespace duckdb { +struct UCCredentials; + +struct UCAPIColumnDefinition { + string name; + string type_text; + idx_t precision; + idx_t scale; + idx_t position; +}; + +struct UCAPITable { + string table_id; + + string name; + string catalog_name; + string schema_name; + string table_type; + string data_source_format; + + string storage_location; + + vector columns; +}; + +struct UCAPISchema { + string schema_name; + string catalog_name; +}; + +struct UCAPITableCredentials { + string key_id; + string secret; + string session_token; +}; + +class UCAPI { +public: + //! WARNING: not thread-safe. To be called once on extension initialization + static void InitializeCurl(); + + static UCAPITableCredentials GetTableCredentials(const string &internal, const string &schema, const string &table, UCCredentials credentials); + static vector GetCatalogs(const string &catalog, UCCredentials credentials); + static vector GetTables(const string &catalog, const string &internal, const string &schema, UCCredentials credentials); + static vector GetSchemas(const string &catalog, const string &internal, UCCredentials credentials); + static vector GetTablesInSchema(const string &catalog, const string &schema, UCCredentials credentials); + static string GetToken(string id, string secret, string endpoint); +}; +} // namespace duckdb diff --git a/src/include/uc_utils.hpp b/src/include/uc_utils.hpp new file mode 100644 index 0000000..8927943 --- /dev/null +++ b/src/include/uc_utils.hpp @@ -0,0 +1,33 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// mysql_utils.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb.hpp" +#include "uc_api.hpp" + +namespace duckdb { +class UCSchemaEntry; +class UCTransaction; + +enum class UCTypeAnnotation { STANDARD, CAST_TO_VARCHAR, NUMERIC_AS_DOUBLE, CTID, JSONB, FIXED_LENGTH_CHAR }; + +struct UCType { + idx_t oid = 0; + UCTypeAnnotation info = UCTypeAnnotation::STANDARD; + vector children; +}; + +class UCUtils { +public: + static LogicalType ToUCType(const LogicalType &input); + static LogicalType TypeToLogicalType(ClientContext &context, const string &columnDefinition); + static string TypeToString(const LogicalType &input); +}; + +} // namespace duckdb diff --git a/src/storage/uc_catalog.cpp b/src/storage/uc_catalog.cpp new file mode 100644 index 0000000..3c23771 --- /dev/null +++ b/src/storage/uc_catalog.cpp @@ -0,0 +1,102 @@ +#include "storage/uc_catalog.hpp" +#include "storage/uc_schema_entry.hpp" +#include "storage/uc_transaction.hpp" +#include "duckdb/storage/database_size.hpp" +#include "duckdb/parser/parsed_data/drop_info.hpp" +#include "duckdb/parser/parsed_data/create_schema_info.hpp" +#include "duckdb/main/attached_database.hpp" + +namespace duckdb { + +UCCatalog::UCCatalog(AttachedDatabase &db_p, const string &internal_name, AccessMode access_mode, + UCCredentials credentials) + : Catalog(db_p), internal_name(internal_name), access_mode(access_mode), credentials(std::move(credentials)), + schemas(*this) { +} + +UCCatalog::~UCCatalog() = default; + +void UCCatalog::Initialize(bool load_builtin) { +} + +optional_ptr UCCatalog::CreateSchema(CatalogTransaction transaction, CreateSchemaInfo &info) { + if (info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) { + DropInfo try_drop; + try_drop.type = CatalogType::SCHEMA_ENTRY; + try_drop.name = info.schema; + try_drop.if_not_found = OnEntryNotFound::RETURN_NULL; + try_drop.cascade = false; + schemas.DropEntry(transaction.GetContext(), try_drop); + } + return schemas.CreateSchema(transaction.GetContext(), info); +} + +void UCCatalog::DropSchema(ClientContext &context, DropInfo &info) { + return schemas.DropEntry(context, info); +} + +void UCCatalog::ScanSchemas(ClientContext &context, std::function callback) { + schemas.Scan(context, [&](CatalogEntry &schema) { callback(schema.Cast()); }); +} + +optional_ptr UCCatalog::GetSchema(CatalogTransaction transaction, const string &schema_name, + OnEntryNotFound if_not_found, QueryErrorContext error_context) { + if (schema_name == DEFAULT_SCHEMA) { + if (default_schema.empty()) { + throw InvalidInputException("Attempting to fetch the default schema - but no database was " + "provided in the connection string"); + } + return GetSchema(transaction, default_schema, if_not_found, error_context); + } + auto entry = schemas.GetEntry(transaction.GetContext(), schema_name); + if (!entry && if_not_found != OnEntryNotFound::RETURN_NULL) { + throw BinderException("Schema with name \"%s\" not found", schema_name); + } + return reinterpret_cast(entry.get()); +} + +bool UCCatalog::InMemory() { + return false; +} + +string UCCatalog::GetDBPath() { + return internal_name; +} + + + +DatabaseSize UCCatalog::GetDatabaseSize(ClientContext &context) { + if (default_schema.empty()) { + throw InvalidInputException("Attempting to fetch the database size - but no database was provided " + "in the connection string"); + } + DatabaseSize size; + return size; +} + +void UCCatalog::ClearCache() { + schemas.ClearEntries(); +} + +unique_ptr UCCatalog::PlanInsert(ClientContext &context, LogicalInsert &op, + unique_ptr plan) { + throw NotImplementedException("UCCatalog PlanInsert"); +} +unique_ptr UCCatalog::PlanCreateTableAs(ClientContext &context, LogicalCreateTable &op, + unique_ptr plan) { + throw NotImplementedException("UCCatalog PlanCreateTableAs"); +} +unique_ptr UCCatalog::PlanDelete(ClientContext &context, LogicalDelete &op, + unique_ptr plan) { + throw NotImplementedException("UCCatalog PlanDelete"); +} +unique_ptr UCCatalog::PlanUpdate(ClientContext &context, LogicalUpdate &op, + unique_ptr plan) { + throw NotImplementedException("UCCatalog PlanUpdate"); +} +unique_ptr UCCatalog::BindCreateIndex(Binder &binder, CreateStatement &stmt, TableCatalogEntry &table, + unique_ptr plan) { + throw NotImplementedException("UCCatalog BindCreateIndex"); +} + +} // namespace duckdb diff --git a/src/storage/uc_catalog_set.cpp b/src/storage/uc_catalog_set.cpp new file mode 100644 index 0000000..6451087 --- /dev/null +++ b/src/storage/uc_catalog_set.cpp @@ -0,0 +1,69 @@ +#include "storage/uc_catalog_set.hpp" +#include "storage/uc_transaction.hpp" +#include "duckdb/parser/parsed_data/drop_info.hpp" +#include "storage/uc_schema_entry.hpp" + +namespace duckdb { + +UCCatalogSet::UCCatalogSet(Catalog &catalog) : catalog(catalog), is_loaded(false) { +} + +optional_ptr UCCatalogSet::GetEntry(ClientContext &context, const string &name) { + if (!is_loaded) { + is_loaded = true; + LoadEntries(context); + } + lock_guard l(entry_lock); + auto entry = entries.find(name); + if (entry == entries.end()) { + return nullptr; + } + return entry->second.get(); +} + +void UCCatalogSet::DropEntry(ClientContext &context, DropInfo &info) { + throw NotImplementedException("UCCatalogSet::DropEntry"); +} + +void UCCatalogSet::EraseEntryInternal(const string &name) { + lock_guard l(entry_lock); + entries.erase(name); +} + +void UCCatalogSet::Scan(ClientContext &context, const std::function &callback) { + if (!is_loaded) { + is_loaded = true; + LoadEntries(context); + } + lock_guard l(entry_lock); + for (auto &entry : entries) { + callback(*entry.second); + } +} + +optional_ptr UCCatalogSet::CreateEntry(unique_ptr entry) { + lock_guard l(entry_lock); + auto result = entry.get(); + if (result->name.empty()) { + throw InternalException("UCCatalogSet::CreateEntry called with empty name"); + } + entries.insert(make_pair(result->name, std::move(entry))); + return result; +} + +void UCCatalogSet::ClearEntries() { + entries.clear(); + is_loaded = false; +} + +UCInSchemaSet::UCInSchemaSet(UCSchemaEntry &schema) : UCCatalogSet(schema.ParentCatalog()), schema(schema) { +} + +optional_ptr UCInSchemaSet::CreateEntry(unique_ptr entry) { + if (!entry->internal) { + entry->internal = schema.internal; + } + return UCCatalogSet::CreateEntry(std::move(entry)); +} + +} // namespace duckdb diff --git a/src/storage/uc_clear_cache.cpp b/src/storage/uc_clear_cache.cpp new file mode 100644 index 0000000..2c51abb --- /dev/null +++ b/src/storage/uc_clear_cache.cpp @@ -0,0 +1,50 @@ +#include "duckdb.hpp" + +#include "duckdb/parser/parsed_data/create_table_function_info.hpp" +#include "duckdb/main/database_manager.hpp" +#include "duckdb/main/attached_database.hpp" +#include "storage/uc_catalog.hpp" + +namespace duckdb { + +struct ClearCacheFunctionData : public TableFunctionData { + bool finished = false; +}; + +static unique_ptr ClearCacheBind(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, vector &names) { + + auto result = make_uniq(); + return_types.push_back(LogicalType::BOOLEAN); + names.emplace_back("Success"); + return std::move(result); +} + +static void ClearUCCaches(ClientContext &context) { + auto databases = DatabaseManager::Get(context).GetDatabases(context); + for (auto &db_ref : databases) { + auto &db = db_ref.get(); + auto &catalog = db.GetCatalog(); + if (catalog.GetCatalogType() != "iceberg") { + continue; + } + catalog.Cast().ClearCache(); + } +} + +static void ClearCacheFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { + auto &data = data_p.bind_data->CastNoConst(); + if (data.finished) { + return; + } + ClearUCCaches(context); + data.finished = true; +} + +void UCClearCacheFunction::ClearCacheOnSetting(ClientContext &context, SetScope scope, Value ¶meter) { + ClearUCCaches(context); +} + +UCClearCacheFunction::UCClearCacheFunction() : TableFunction("pc_clear_cache", {}, ClearCacheFunction, ClearCacheBind) { +} +} // namespace duckdb diff --git a/src/storage/uc_schema_entry.cpp b/src/storage/uc_schema_entry.cpp new file mode 100644 index 0000000..7bc93ce --- /dev/null +++ b/src/storage/uc_schema_entry.cpp @@ -0,0 +1,161 @@ +#include "storage/uc_schema_entry.hpp" +#include "storage/uc_table_entry.hpp" +#include "storage/uc_transaction.hpp" +#include "duckdb/parser/parsed_data/create_view_info.hpp" +#include "duckdb/parser/parsed_data/create_index_info.hpp" +#include "duckdb/planner/parsed_data/bound_create_table_info.hpp" +#include "duckdb/parser/parsed_data/drop_info.hpp" +#include "duckdb/parser/constraints/list.hpp" +#include "duckdb/common/unordered_set.hpp" +#include "duckdb/parser/parsed_data/alter_info.hpp" +#include "duckdb/parser/parsed_data/alter_table_info.hpp" +#include "duckdb/parser/parsed_expression_iterator.hpp" + +namespace duckdb { + +UCSchemaEntry::UCSchemaEntry(Catalog &catalog, CreateSchemaInfo &info) + : SchemaCatalogEntry(catalog, info), tables(*this) { +} + +UCSchemaEntry::~UCSchemaEntry() { +} + +UCTransaction &GetUCTransaction(CatalogTransaction transaction) { + if (!transaction.transaction) { + throw InternalException("No transaction!?"); + } + return transaction.transaction->Cast(); +} + +optional_ptr UCSchemaEntry::CreateTable(CatalogTransaction transaction, BoundCreateTableInfo &info) { + auto &base_info = info.Base(); + auto table_name = base_info.table; + if (base_info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) { + throw NotImplementedException("REPLACE ON CONFLICT in CreateTable"); + } + return tables.CreateTable(transaction.GetContext(), info); +} + +optional_ptr UCSchemaEntry::CreateFunction(CatalogTransaction transaction, CreateFunctionInfo &info) { + throw BinderException("PC databases do not support creating functions"); +} + +void UCUnqualifyColumnRef(ParsedExpression &expr) { + if (expr.type == ExpressionType::COLUMN_REF) { + auto &colref = expr.Cast(); + auto name = std::move(colref.column_names.back()); + colref.column_names = {std::move(name)}; + return; + } + ParsedExpressionIterator::EnumerateChildren(expr, UCUnqualifyColumnRef); +} + +optional_ptr UCSchemaEntry::CreateIndex(CatalogTransaction transaction, CreateIndexInfo &info, + TableCatalogEntry &table) { + throw NotImplementedException("CreateIndex"); +} + +string GetUCCreateView(CreateViewInfo &info) { + throw NotImplementedException("GetCreateView"); +} + +optional_ptr UCSchemaEntry::CreateView(CatalogTransaction transaction, CreateViewInfo &info) { + if (info.sql.empty()) { + throw BinderException("Cannot create view in PC that originated from an " + "empty SQL statement"); + } + if (info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT || + info.on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT) { + auto current_entry = GetEntry(transaction, CatalogType::VIEW_ENTRY, info.view_name); + if (current_entry) { + if (info.on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT) { + return current_entry; + } + throw NotImplementedException("REPLACE ON CONFLICT in CreateView"); + } + } + auto &uc_transaction = GetUCTransaction(transaction); + // uc_transaction.Query(GetUCCreateView(info)); + return tables.RefreshTable(transaction.GetContext(), info.view_name); +} + +optional_ptr UCSchemaEntry::CreateType(CatalogTransaction transaction, CreateTypeInfo &info) { + throw BinderException("PC databases do not support creating types"); +} + +optional_ptr UCSchemaEntry::CreateSequence(CatalogTransaction transaction, CreateSequenceInfo &info) { + throw BinderException("PC databases do not support creating sequences"); +} + +optional_ptr UCSchemaEntry::CreateTableFunction(CatalogTransaction transaction, + CreateTableFunctionInfo &info) { + throw BinderException("PC databases do not support creating table functions"); +} + +optional_ptr UCSchemaEntry::CreateCopyFunction(CatalogTransaction transaction, + CreateCopyFunctionInfo &info) { + throw BinderException("PC databases do not support creating copy functions"); +} + +optional_ptr UCSchemaEntry::CreatePragmaFunction(CatalogTransaction transaction, + CreatePragmaFunctionInfo &info) { + throw BinderException("PC databases do not support creating pragma functions"); +} + +optional_ptr UCSchemaEntry::CreateCollation(CatalogTransaction transaction, CreateCollationInfo &info) { + throw BinderException("PC databases do not support creating collations"); +} + +void UCSchemaEntry::Alter(CatalogTransaction transaction, AlterInfo &info) { + if (info.type != AlterType::ALTER_TABLE) { + throw BinderException("Only altering tables is supported for now"); + } + auto &alter = info.Cast(); + tables.AlterTable(transaction.GetContext(), alter); +} + +bool CatalogTypeIsSupported(CatalogType type) { + switch (type) { + case CatalogType::INDEX_ENTRY: + case CatalogType::TABLE_ENTRY: + case CatalogType::VIEW_ENTRY: + return true; + default: + return false; + } +} + +void UCSchemaEntry::Scan(ClientContext &context, CatalogType type, + const std::function &callback) { + if (!CatalogTypeIsSupported(type)) { + return; + } + GetCatalogSet(type).Scan(context, callback); +} +void UCSchemaEntry::Scan(CatalogType type, const std::function &callback) { + throw NotImplementedException("Scan without context not supported"); +} + +void UCSchemaEntry::DropEntry(ClientContext &context, DropInfo &info) { + GetCatalogSet(info.type).DropEntry(context, info); +} + +optional_ptr UCSchemaEntry::GetEntry(CatalogTransaction transaction, CatalogType type, + const string &name) { + if (!CatalogTypeIsSupported(type)) { + return nullptr; + } + return GetCatalogSet(type).GetEntry(transaction.GetContext(), name); +} + +UCCatalogSet &UCSchemaEntry::GetCatalogSet(CatalogType type) { + switch (type) { + case CatalogType::TABLE_ENTRY: + case CatalogType::VIEW_ENTRY: + return tables; + default: + throw InternalException("Type not supported for GetCatalogSet"); + } +} + +} // namespace duckdb diff --git a/src/storage/uc_schema_set.cpp b/src/storage/uc_schema_set.cpp new file mode 100644 index 0000000..6efb318 --- /dev/null +++ b/src/storage/uc_schema_set.cpp @@ -0,0 +1,38 @@ +#include "storage/uc_schema_set.hpp" +#include "storage/uc_catalog.hpp" +#include "uc_api.hpp" +#include "storage/uc_transaction.hpp" +#include "duckdb/parser/parsed_data/create_schema_info.hpp" +#include "duckdb/catalog/catalog.hpp" + +namespace duckdb { + +UCSchemaSet::UCSchemaSet(Catalog &catalog) : UCCatalogSet(catalog) { +} + +static bool IsInternalTable(const string &catalog, const string &schema) { + if (schema == "information_schema") { + return true; + } + return false; +} +void UCSchemaSet::LoadEntries(ClientContext &context) { + + auto &uc_catalog = catalog.Cast(); + auto tables = UCAPI::GetSchemas(catalog.GetName(), uc_catalog.internal_name, uc_catalog.credentials); + + for (const auto &schema : tables) { + CreateSchemaInfo info; + info.schema = schema.schema_name; + info.internal = IsInternalTable(schema.catalog_name, schema.schema_name); + auto schema_entry = make_uniq(catalog, info); + schema_entry->schema_data = make_uniq(schema); + CreateEntry(std::move(schema_entry)); + } +} + +optional_ptr UCSchemaSet::CreateSchema(ClientContext &context, CreateSchemaInfo &info) { + throw NotImplementedException("Schema creation"); +} + +} // namespace duckdb diff --git a/src/storage/uc_table_entry.cpp b/src/storage/uc_table_entry.cpp new file mode 100644 index 0000000..7691a2f --- /dev/null +++ b/src/storage/uc_table_entry.cpp @@ -0,0 +1,147 @@ +#include "storage/uc_catalog.hpp" +#include "storage/uc_schema_entry.hpp" +#include "storage/uc_table_entry.hpp" +#include "storage/uc_transaction.hpp" +#include "duckdb/storage/statistics/base_statistics.hpp" +#include "duckdb/storage/table_storage_info.hpp" +#include "duckdb/main/extension_util.hpp" +#include "duckdb/main/database.hpp" +#include "duckdb/main/secret/secret_manager.hpp" +#include "duckdb/catalog/catalog_entry/table_function_catalog_entry.hpp" +#include "duckdb/parser/tableref/table_function_ref.hpp" +#include "uc_api.hpp" +#include "../../duckdb/third_party/catch/catch.hpp" +#include "duckdb/planner/binder.hpp" +#include "duckdb/planner/tableref/bound_table_function.hpp" +#include "duckdb/planner/logical_operator.hpp" +#include "duckdb/planner/operator/logical_get.hpp" + + +namespace duckdb { + +UCTableEntry::UCTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTableInfo &info) + : TableCatalogEntry(catalog, schema, info) { + this->internal = false; +} + +UCTableEntry::UCTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, UCTableInfo &info) + : TableCatalogEntry(catalog, schema, *info.create_info) { + this->internal = false; +} + +unique_ptr UCTableEntry::GetStatistics(ClientContext &context, column_t column_id) { + return nullptr; +} + +void UCTableEntry::BindUpdateConstraints(Binder &binder, LogicalGet &, LogicalProjection &, LogicalUpdate &, + ClientContext &) { + throw NotImplementedException("BindUpdateConstraints"); +} + +struct MyIcebergFunctionData : public FunctionData { + std::string path; // store the path or any other relevant info here + + // Optional: implement Copy for caching/pushdown logic + unique_ptr Copy() const override { + auto copy = make_uniq(); + copy->path = path; + return copy; + } + + // Optional: implement Equals for caching + bool Equals(const FunctionData &other_p) const override { + auto &other = (const MyIcebergFunctionData &)other_p; + return path == other.path; + } +}; + +TableFunction UCTableEntry::GetScanFunction(ClientContext &context, unique_ptr &bind_data) { + auto &db = DatabaseInstance::GetDatabase(context); + auto &uc_catalog = catalog.Cast(); + + auto &parquet_function_set = ExtensionUtil::GetTableFunction(db, "parquet_scan"); + auto parquet_scan_function = parquet_function_set.functions.GetFunctionByArguments(context, {LogicalType::VARCHAR}); + + auto &iceberg_function_set = ExtensionUtil::GetTableFunction(db, "iceberg_scan"); + auto iceberg_scan_function = iceberg_function_set.functions.GetFunctionByArguments(context, {LogicalType::VARCHAR}); + + D_ASSERT(table_data); + + if (table_data->data_source_format != "ICEBERG") { + throw NotImplementedException("Table '%s' is of unsupported format '%s', ", table_data->name, + table_data->data_source_format); + } + + if (table_data->storage_location.find("file://") != 0) { + auto &secret_manager = SecretManager::Get(context); + // Get Credentials from UCAPI + auto table_credentials = UCAPI::GetTableCredentials( + uc_catalog.internal_name, table_data->schema_name, table_data->name, uc_catalog.credentials); + + // Inject secret into secret manager scoped to this path + CreateSecretInfo info(OnCreateConflict::REPLACE_ON_CONFLICT, SecretPersistType::TEMPORARY); + info.name = "__internal_uc_" + table_data->table_id; + info.type = "s3"; + info.provider = "config"; + info.storage_type = "memory"; + info.options = { + {"key_id", table_credentials.key_id}, + {"secret", table_credentials.secret}, + {"session_token", table_credentials.session_token}, + {"region", uc_catalog.credentials.aws_region}, + }; + + std::string lc_storage_location; + lc_storage_location.resize(table_data->storage_location.size()); + std::transform(table_data->storage_location.begin(), table_data->storage_location.end(), lc_storage_location.begin(), ::tolower); + size_t metadata_pos = lc_storage_location.find("metadata"); + if (metadata_pos != std::string::npos) { + info.scope = {lc_storage_location.substr(0, metadata_pos)}; + } else { + throw std::runtime_error("Substring not found"); + } + + auto my_secret = secret_manager.CreateSecret(context, info); + } + + named_parameter_map_t param_map; + vector return_types; + vector names; + TableFunctionRef empty_ref; + + // Set the S3 path as input to table function + vector inputs = {table_data->storage_location}; + + TableFunctionBindInput bind_input(inputs, param_map, return_types, names, nullptr, nullptr, + iceberg_scan_function, empty_ref); + + auto table_ref = iceberg_scan_function.bind_replace(context, bind_input); + + // 1) Create a Binder and bind the parser-level TableRef -> BoundTableRef + auto binder = Binder::CreateBinder(context); + auto bound_ref = binder->Bind(*table_ref); + + // 2) Create a logical plan from the bound reference + unique_ptr logical_plan = binder->CreatePlan(*bound_ref); + + // 3) Recursively search the logical plan for a LogicalGet node + // For a single table function, you often have just one operator: LogicalGet + LogicalOperator *op = logical_plan.get(); + if (op->type != LogicalOperatorType::LOGICAL_GET) { + throw std::runtime_error("Expected a LogicalGet, but got something else!"); + } + + // 4) Access the bind_data inside LogicalGet + auto &get = (LogicalGet &)*op; + bind_data = std::move(get.bind_data); + + return parquet_scan_function; +} + +TableStorageInfo UCTableEntry::GetStorageInfo(ClientContext &context) { + TableStorageInfo result; + // TODO fill info + return result; +} + +} // namespace duckdb diff --git a/src/storage/uc_table_set.cpp b/src/storage/uc_table_set.cpp new file mode 100644 index 0000000..494060c --- /dev/null +++ b/src/storage/uc_table_set.cpp @@ -0,0 +1,87 @@ +#include "uc_api.hpp" +#include "uc_utils.hpp" + +#include "storage/uc_catalog.hpp" +#include "storage/uc_table_set.hpp" +#include "storage/uc_transaction.hpp" +#include "duckdb/parser/parsed_data/create_table_info.hpp" +#include "duckdb/parser/constraints/not_null_constraint.hpp" +#include "duckdb/parser/constraints/unique_constraint.hpp" +#include "duckdb/parser/expression/constant_expression.hpp" +#include "duckdb/planner/parsed_data/bound_create_table_info.hpp" +#include "duckdb/catalog/dependency_list.hpp" +#include "duckdb/parser/parsed_data/create_table_info.hpp" +#include "duckdb/parser/constraints/list.hpp" +#include "storage/uc_schema_entry.hpp" +#include "duckdb/parser/parser.hpp" + +namespace duckdb { + +UCTableSet::UCTableSet(UCSchemaEntry &schema) : UCInSchemaSet(schema) { +} + +static ColumnDefinition CreateColumnDefinition(ClientContext &context, UCAPIColumnDefinition &coldef) { + return {coldef.name, UCUtils::TypeToLogicalType(context, coldef.type_text)}; +} + +void UCTableSet::LoadEntries(ClientContext &context) { + auto &transaction = UCTransaction::Get(context, catalog); + + auto &uc_catalog = catalog.Cast(); + + // TODO: handle out-of-order columns using position property + auto tables = UCAPI::GetTables(catalog.GetName(), catalog.GetDBPath(), schema.name, uc_catalog.credentials); + + for (auto &table : tables) { + D_ASSERT(schema.name == table.schema_name); + CreateTableInfo info; + for (auto &col : table.columns) { + info.columns.AddColumn(CreateColumnDefinition(context, col)); + } + + info.table = table.name; + auto table_entry = make_uniq(catalog, schema, info); + table_entry->table_data = make_uniq(table); + + CreateEntry(std::move(table_entry)); + } +} + +optional_ptr UCTableSet::RefreshTable(ClientContext &context, const string &table_name) { + auto table_info = GetTableInfo(context, schema, table_name); + auto table_entry = make_uniq(catalog, schema, *table_info); + auto table_ptr = table_entry.get(); + CreateEntry(std::move(table_entry)); + return table_ptr; +} + +unique_ptr UCTableSet::GetTableInfo(ClientContext &context, UCSchemaEntry &schema, + const string &table_name) { + throw NotImplementedException("UCTableSet::CreateTable"); +} + +optional_ptr UCTableSet::CreateTable(ClientContext &context, BoundCreateTableInfo &info) { + throw NotImplementedException("UCTableSet::CreateTable"); +} + +void UCTableSet::AlterTable(ClientContext &context, RenameTableInfo &info) { + throw NotImplementedException("UCTableSet::AlterTable"); +} + +void UCTableSet::AlterTable(ClientContext &context, RenameColumnInfo &info) { + throw NotImplementedException("UCTableSet::AlterTable"); +} + +void UCTableSet::AlterTable(ClientContext &context, AddColumnInfo &info) { + throw NotImplementedException("UCTableSet::AlterTable"); +} + +void UCTableSet::AlterTable(ClientContext &context, RemoveColumnInfo &info) { + throw NotImplementedException("UCTableSet::AlterTable"); +} + +void UCTableSet::AlterTable(ClientContext &context, AlterTableInfo &alter) { + throw NotImplementedException("UCTableSet::AlterTable"); +} + +} // namespace duckdb diff --git a/src/storage/uc_transaction.cpp b/src/storage/uc_transaction.cpp new file mode 100644 index 0000000..2942458 --- /dev/null +++ b/src/storage/uc_transaction.cpp @@ -0,0 +1,61 @@ +#include "storage/uc_transaction.hpp" +#include "storage/uc_catalog.hpp" +#include "duckdb/parser/parsed_data/create_view_info.hpp" +#include "duckdb/catalog/catalog_entry/index_catalog_entry.hpp" +#include "duckdb/catalog/catalog_entry/view_catalog_entry.hpp" + +namespace duckdb { + +UCTransaction::UCTransaction(UCCatalog &uc_catalog, TransactionManager &manager, ClientContext &context) + : Transaction(manager, context), access_mode(uc_catalog.access_mode) { + // connection = UCConnection::Open(uc_catalog.path); +} + +UCTransaction::~UCTransaction() = default; + +void UCTransaction::Start() { + transaction_state = UCTransactionState::TRANSACTION_NOT_YET_STARTED; +} +void UCTransaction::Commit() { + if (transaction_state == UCTransactionState::TRANSACTION_STARTED) { + transaction_state = UCTransactionState::TRANSACTION_FINISHED; + // connection.Execute("COMMIT"); + } +} +void UCTransaction::Rollback() { + if (transaction_state == UCTransactionState::TRANSACTION_STARTED) { + transaction_state = UCTransactionState::TRANSACTION_FINISHED; + // connection.Execute("ROLLBACK"); + } +} + +// UCConnection &UCTransaction::GetConnection() { +// if (transaction_state == UCTransactionState::TRANSACTION_NOT_YET_STARTED) { +// transaction_state = UCTransactionState::TRANSACTION_STARTED; +// string query = "START TRANSACTION"; +// if (access_mode == AccessMode::READ_ONLY) { +// query += " READ ONLY"; +// } +//// conne/**/ction.Execute(query); +// } +// return connection; +//} + +// unique_ptr UCTransaction::Query(const string &query) { +// if (transaction_state == UCTransactionState::TRANSACTION_NOT_YET_STARTED) { +// transaction_state = UCTransactionState::TRANSACTION_STARTED; +// string transaction_start = "START TRANSACTION"; +// if (access_mode == AccessMode::READ_ONLY) { +// transaction_start += " READ ONLY"; +// } +// connection.Query(transaction_start); +// return connection.Query(query); +// } +// return connection.Query(query); +//} + +UCTransaction &UCTransaction::Get(ClientContext &context, Catalog &catalog) { + return Transaction::Get(context, catalog).Cast(); +} + +} // namespace duckdb diff --git a/src/storage/uc_transaction_manager.cpp b/src/storage/uc_transaction_manager.cpp new file mode 100644 index 0000000..afd7e07 --- /dev/null +++ b/src/storage/uc_transaction_manager.cpp @@ -0,0 +1,40 @@ +#include "storage/uc_transaction_manager.hpp" +#include "duckdb/main/attached_database.hpp" + +namespace duckdb { + +UCTransactionManager::UCTransactionManager(AttachedDatabase &db_p, UCCatalog &uc_catalog) + : TransactionManager(db_p), uc_catalog(uc_catalog) { +} + +Transaction &UCTransactionManager::StartTransaction(ClientContext &context) { + auto transaction = make_uniq(uc_catalog, *this, context); + transaction->Start(); + auto &result = *transaction; + lock_guard l(transaction_lock); + transactions[result] = std::move(transaction); + return result; +} + +ErrorData UCTransactionManager::CommitTransaction(ClientContext &context, Transaction &transaction) { + auto &uc_transaction = transaction.Cast(); + uc_transaction.Commit(); + lock_guard l(transaction_lock); + transactions.erase(transaction); + return ErrorData(); +} + +void UCTransactionManager::RollbackTransaction(Transaction &transaction) { + auto &uc_transaction = transaction.Cast(); + uc_transaction.Rollback(); + lock_guard l(transaction_lock); + transactions.erase(transaction); +} + +void UCTransactionManager::Checkpoint(ClientContext &context, bool force) { + auto &transaction = UCTransaction::Get(context, db.GetCatalog()); + // auto &db = transaction.GetConnection(); + // db.Execute("CHECKPOINT"); +} + +} // namespace duckdb diff --git a/src/uc_api.cpp b/src/uc_api.cpp new file mode 100644 index 0000000..6a6e8d7 --- /dev/null +++ b/src/uc_api.cpp @@ -0,0 +1,320 @@ +#include "uc_api.hpp" +#include "storage/uc_catalog.hpp" +#include "yyjson.hpp" +#include +#include + +#include + +namespace duckdb { + +//! We use a global here to store the path that is selected on the UCAPI::InitializeCurl call +static string SELECTED_CURL_CERT_PATH = ""; + +static size_t GetRequestWriteCallback(void *contents, size_t size, size_t nmemb, void *userp) { + ((std::string *)userp)->append((char *)contents, size * nmemb); + return size * nmemb; +} + +// we statically compile in libcurl, which means the cert file location of the build machine is the +// place curl will look. But not every distro has this file in the same location, so we search a +// number of common locations and use the first one we find. +static string certFileLocations[] = { + // Arch, Debian-based, Gentoo + "/etc/ssl/certs/ca-certificates.crt", + // RedHat 7 based + "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", + // Redhat 6 based + "/etc/pki/tls/certs/ca-bundle.crt", + // OpenSUSE + "/etc/ssl/ca-bundle.pem", + // Alpine + "/etc/ssl/cert.pem" +}; + +// Look through the the above locations and if one of the files exists, set that as the location curl should use. +static bool SelectCurlCertPath() { + for (string& caFile : certFileLocations) { + struct stat buf; + if (stat(caFile.c_str(), &buf) == 0) { + SELECTED_CURL_CERT_PATH = caFile; + } + } + return false; +} + +static bool SetCurlCAFileInfo(CURL* curl) { + if (!SELECTED_CURL_CERT_PATH.empty()) { + curl_easy_setopt(curl, CURLOPT_CAINFO, SELECTED_CURL_CERT_PATH.c_str()); + return true; + } + return false; +} + +// Note: every curl object we use should set this, because without it some linux distro's may not find the CA certificate. +static void InitializeCurlObject(CURL * curl, const string &token) { + if (!token.empty()) { + curl_easy_setopt(curl, CURLOPT_XOAUTH2_BEARER, token.c_str()); + curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_BEARER); + } + SetCurlCAFileInfo(curl); +} + +static string GetRequest(const string &url, const string &token = "", curl_slist *extra_headers = NULL) { + CURL *curl; + CURLcode res; + string readBuffer; + + curl = curl_easy_init(); + if (curl) { + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, GetRequestWriteCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer); + if(extra_headers) { + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, extra_headers); + } + InitializeCurlObject(curl, token); + res = curl_easy_perform(curl); + curl_easy_cleanup(curl); + + if (res != CURLcode::CURLE_OK) { + string error = curl_easy_strerror(res); + throw IOException("Curl Request to '%s' failed with error: '%s'", url, error); + } + return readBuffer; + } + throw InternalException("Failed to initialize curl"); +} + +static string PostRequest(const string &url, const string &post_data, curl_slist *extra_headers = NULL) { + string readBuffer; + CURL *curl = curl_easy_init(); + if (!curl) { + throw InternalException("Failed to initialize curl"); + } + + // Set the URL + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + + // Set POST method + curl_easy_setopt(curl, CURLOPT_POST, 1L); + + // Set POST data + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, post_data.c_str()); + + // Set up response handling + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, GetRequestWriteCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer); + + // Create default headers for content type + struct curl_slist *headers = NULL; + headers = curl_slist_append(headers, "Content-Type: application/x-www-form-urlencoded"); + + // Append any extra headers + if (extra_headers) { + struct curl_slist *temp = extra_headers; + while (temp) { + headers = curl_slist_append(headers, temp->data); + temp = temp->next; + } + } + + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + InitializeCurlObject(curl, ""); + + // Perform the request + CURLcode res = curl_easy_perform(curl); + + // Clean up + curl_slist_free_all(headers); + curl_easy_cleanup(curl); + + if (res != CURLcode::CURLE_OK) { + string error = curl_easy_strerror(res); + throw IOException("Curl Request to '%s' failed with error: '%s'", url, error); + } + return readBuffer; + +} + +static duckdb_yyjson::yyjson_val *GetTableMetadata(const string &internal, const string &schema, const string &table, UCCredentials credentials) { + struct curl_slist *extra_headers = NULL; + extra_headers = curl_slist_append(extra_headers, "X-Iceberg-Access-Delegation: vended-credentials"); + auto api_result = GetRequest( + credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables/" + table, + credentials.token, + extra_headers); + duckdb_yyjson::yyjson_doc *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); + return yyjson_doc_get_root(doc); +} + +template +static TYPE TemplatedTryGetYYJson(duckdb_yyjson::yyjson_val *obj, const string &field, TYPE default_val, + bool fail_on_missing = true) { + auto val = yyjson_obj_get(obj, field.c_str()); + if (val && yyjson_get_type(val) == TYPE_NUM) { + return get_function(val); + } else if (!fail_on_missing) { + return default_val; + } + throw IOException("Invalid field found while parsing field: " + field); +} + +static uint64_t TryGetNumFromObject(duckdb_yyjson::yyjson_val *obj, const string &field, bool fail_on_missing = true, + uint64_t default_val = 0) { + return TemplatedTryGetYYJson(obj, field, default_val, + fail_on_missing); +} +static bool TryGetBoolFromObject(duckdb_yyjson::yyjson_val *obj, const string &field, bool fail_on_missing = false, + bool default_val = false) { + return TemplatedTryGetYYJson(obj, field, default_val, + fail_on_missing); +} +static string TryGetStrFromObject(duckdb_yyjson::yyjson_val *obj, const string &field, bool fail_on_missing = true, + const char *default_val = "") { + return TemplatedTryGetYYJson(obj, field, default_val, + fail_on_missing); +} + +void UCAPI::InitializeCurl() { + SelectCurlCertPath(); +} + +vector UCAPI::GetCatalogs(const string &catalog, UCCredentials credentials) { + throw NotImplementedException("UCAPI::GetCatalogs"); +} + +static UCAPIColumnDefinition ParseColumnDefinition(duckdb_yyjson::yyjson_val *column_def) { + UCAPIColumnDefinition result; + + result.name = TryGetStrFromObject(column_def, "name"); + result.type_text = TryGetStrFromObject(column_def, "type"); + result.precision = -1; //TODO: TryGetNumFromObject(column_def, "type_precision"); + result.scale = -1; //TODO: TryGetNumFromObject(column_def, "type_scale"); + result.position = TryGetNumFromObject(column_def, "id") - 1; + + return result; +} + +UCAPITableCredentials UCAPI::GetTableCredentials(const string &internal, const string &schema, const string &table, UCCredentials credentials) { + UCAPITableCredentials result; + + duckdb_yyjson::yyjson_val *root = GetTableMetadata(internal, schema, table, credentials); + auto *aws_temp_credentials = yyjson_obj_get(root, "config"); + + if (aws_temp_credentials) { + result.key_id = TryGetStrFromObject(aws_temp_credentials, "s3.access-key-id"); + result.secret = TryGetStrFromObject(aws_temp_credentials, "s3.secret-access-key"); + result.session_token = TryGetStrFromObject(aws_temp_credentials, "s3.session-token"); + } + + return result; +} + +string UCAPI::GetToken(string id, string secret, string endpoint) { + string post_data = "grant_type=client_credentials&client_id=" + id + "&client_secret=" + secret + "&scope=PRINCIPAL_ROLE:ALL"; + string api_result = PostRequest(endpoint + "/v1/oauth/tokens", post_data); + + // Read JSON and get root + auto *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); + auto *root = yyjson_doc_get_root(doc); + + auto *error = yyjson_obj_get(root, "error"); + if (error != NULL) { + string err_msg = TryGetStrFromObject(error, "message"); + throw std::runtime_error(err_msg); + } + + return TryGetStrFromObject(root, "access_token"); +} + +vector UCAPI::GetTables(const string &catalog, const string &internal, const string &schema, UCCredentials credentials) { + vector result; + + auto api_result = GetRequest(credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables", credentials.token); + + // Read JSON and get root + auto *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); + auto *root = yyjson_doc_get_root(doc); + + // Get root["hits"], iterate over the array + auto *tables = yyjson_obj_get(root, "identifiers"); + size_t idx, max; + duckdb_yyjson::yyjson_val *table; + yyjson_arr_foreach(tables, idx, max, table) { + UCAPITable table_result; + table_result.catalog_name = catalog; + table_result.schema_name = schema; + table_result.name = TryGetStrFromObject(table, "name"); + + auto *metadata_root = GetTableMetadata(internal, schema, table_result.name, credentials); + table_result.data_source_format = "ICEBERG"; + table_result.storage_location = TryGetStrFromObject(metadata_root, "metadata-location"); + auto *metadata = yyjson_obj_get(metadata_root, "metadata"); + table_result.table_id = TryGetStrFromObject(metadata, "table-uuid"); + + uint64_t current_schema_id = TryGetNumFromObject(metadata, "current-schema-id"); + auto *schemas = yyjson_obj_get(metadata, "schemas"); + duckdb_yyjson::yyjson_val *schema; + size_t schema_idx, schema_max; + bool found = false; + yyjson_arr_foreach(schemas, schema_idx, schema_max, schema) { + uint64_t schema_id = TryGetNumFromObject(schema, "schema-id"); + if (schema_id == current_schema_id) { + found = true; + auto *columns = yyjson_obj_get(schema, "fields"); + duckdb_yyjson::yyjson_val *col; + size_t col_idx, col_max; + yyjson_arr_foreach(columns, col_idx, col_max, col) { + auto column_definition = ParseColumnDefinition(col); + table_result.columns.push_back(column_definition); + } + result.push_back(table_result); + + // This could take a while so display found tables to indicate progress + // TODO: Is there a way to show progress some other way? + std::cout << table_result.schema_name << " | " << table_result.name << std::endl; + } + } + + if (!found) { + throw InternalException("Current schema not found"); + } + } + + return result; +} + +vector UCAPI::GetSchemas(const string &catalog, const string &internal, UCCredentials credentials) { + vector result; + + auto api_result = + GetRequest(credentials.endpoint + "/v1/" + internal + "/namespaces", credentials.token); + + // Read JSON and get root + duckdb_yyjson::yyjson_doc *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); + duckdb_yyjson::yyjson_val *root = yyjson_doc_get_root(doc); + + auto *error = yyjson_obj_get(root, "error"); + if (error != NULL) { + string err_msg = TryGetStrFromObject(error, "message"); + throw std::runtime_error(err_msg); + } + + // Get root["hits"], iterate over the array + auto *schemas = yyjson_obj_get(root, "namespaces"); + size_t idx, max; + duckdb_yyjson::yyjson_val *schema; + yyjson_arr_foreach(schemas, idx, max, schema) { + UCAPISchema schema_result; + + schema_result.catalog_name = catalog; + duckdb_yyjson::yyjson_val *value = yyjson_arr_get(schema, 0); + schema_result.schema_name = yyjson_get_str(value); + result.push_back(schema_result); + } + + return result; +} + +} // namespace duckdb diff --git a/src/uc_utils.cpp b/src/uc_utils.cpp new file mode 100644 index 0000000..5acae60 --- /dev/null +++ b/src/uc_utils.cpp @@ -0,0 +1,193 @@ +#include "uc_utils.hpp" +#include "duckdb/common/operator/cast_operators.hpp" +#include "storage/uc_schema_entry.hpp" +#include "storage/uc_transaction.hpp" + +#include + +namespace duckdb { + +string UCUtils::TypeToString(const LogicalType &input) { + switch (input.id()) { + case LogicalType::VARCHAR: + return "TEXT"; + case LogicalType::UTINYINT: + return "TINYINT UNSIGNED"; + case LogicalType::USMALLINT: + return "SMALLINT UNSIGNED"; + case LogicalType::UINTEGER: + return "INTEGER UNSIGNED"; + case LogicalType::UBIGINT: + return "BIGINT UNSIGNED"; + case LogicalType::TIMESTAMP: + return "DATETIME"; + case LogicalType::TIMESTAMP_TZ: + return "TIMESTAMP"; + default: + return input.ToString(); + } +} + +LogicalType UCUtils::TypeToLogicalType(ClientContext &context, const string &type_text) { + if (type_text == "tinyint") { + return LogicalType::TINYINT; + } else if (type_text == "smallint") { + return LogicalType::SMALLINT; + } else if (type_text == "bigint") { + return LogicalType::BIGINT; + } else if (type_text == "int") { + return LogicalType::INTEGER; + } else if (type_text == "long") { + return LogicalType::BIGINT; + } else if (type_text == "string") { + return LogicalType::VARCHAR; + } else if (type_text == "double") { + return LogicalType::DOUBLE; + } else if (type_text == "float") { + return LogicalType::FLOAT; + } else if (type_text == "boolean") { + return LogicalType::BOOLEAN; + } else if (type_text == "timestamp") { + return LogicalType::TIMESTAMP; + } else if (type_text == "timestamptz") { + return LogicalType::TIMESTAMP_TZ; + } else if (type_text == "binary") { + return LogicalType::BLOB; + } else if (type_text == "date") { + return LogicalType::DATE; + } else if (type_text == "timestamp") { + return LogicalType::TIMESTAMP; // TODO: Is this the right timestamp + } else if (type_text.find("decimal(") == 0) { + size_t spec_end = type_text.find(')'); + if (spec_end != string::npos) { + size_t sep = type_text.find(','); + auto prec_str = type_text.substr(8, sep - 8); + auto scale_str = type_text.substr(sep + 1, spec_end - sep - 1); + uint8_t prec = Cast::Operation(prec_str); + uint8_t scale = Cast::Operation(scale_str); + return LogicalType::DECIMAL(prec, scale); + } + } else if (type_text.find("array<") == 0) { + size_t type_end = type_text.rfind('>'); // find last, to deal with nested + if (type_end != string::npos) { + auto child_type_str = type_text.substr(6, type_end - 6); + auto child_type = UCUtils::TypeToLogicalType(context, child_type_str); + return LogicalType::LIST(child_type); + } + } else if (type_text.find("map<") == 0) { + size_t type_end = type_text.rfind('>'); // find last, to deal with nested + if (type_end != string::npos) { + // TODO: Factor this and struct parsing into an iterator over ',' separated values + vector key_val; + size_t cur = 4; + auto nested_opens = 0; + for (;;) { + size_t next_sep = cur; + // find the location of the next ',' ignoring nested commas + while (type_text[next_sep] != ',' || nested_opens > 0) { + if (type_text[next_sep] == '<') { + nested_opens++; + } else if (type_text[next_sep] == '>') { + nested_opens--; + } + next_sep++; + if (next_sep == type_end) { + break; + } + } + auto child_str = type_text.substr(cur, next_sep - cur); + auto child_type = UCUtils::TypeToLogicalType(context, child_str); + key_val.push_back(child_type); + if (next_sep == type_end) { + break; + } + cur = next_sep + 1; + } + if (key_val.size() != 2) { + throw NotImplementedException("Invalid map specification with %i types", key_val.size()); + } + return LogicalType::MAP(key_val[0], key_val[1]); + } + } else if (type_text.find("struct<") == 0) { + size_t type_end = type_text.rfind('>'); // find last, to deal with nested + if (type_end != string::npos) { + child_list_t children; + size_t cur = 7; + auto nested_opens = 0; + for (;;) { + size_t next_sep = cur; + // find the location of the next ',' ignoring nested commas + while (type_text[next_sep] != ',' || nested_opens > 0) { + if (type_text[next_sep] == '<') { + nested_opens++; + } else if (type_text[next_sep] == '>') { + nested_opens--; + } + next_sep++; + if (next_sep == type_end) { + break; + } + } + auto child_str = type_text.substr(cur, next_sep - cur); + size_t type_sep = child_str.find(':'); + if (type_sep == string::npos) { + throw NotImplementedException("Invalid struct child type specifier: %s", child_str); + } + auto child_name = child_str.substr(0, type_sep); + auto child_type = UCUtils::TypeToLogicalType(context, child_str.substr(type_sep + 1, string::npos)); + children.push_back({child_name, child_type}); + if (next_sep == type_end) { + break; + } + cur = next_sep + 1; + } + return LogicalType::STRUCT(children); + } + } + + throw NotImplementedException("Tried to fallback to unknown type for '%s'", type_text); + // fallback for unknown types + return LogicalType::VARCHAR; +} + +LogicalType UCUtils::ToUCType(const LogicalType &input) { + // todo do we need this mapping? + throw NotImplementedException("ToUCType not yet implemented"); + switch (input.id()) { + case LogicalTypeId::BOOLEAN: + case LogicalTypeId::SMALLINT: + case LogicalTypeId::INTEGER: + case LogicalTypeId::BIGINT: + case LogicalTypeId::TINYINT: + case LogicalTypeId::UTINYINT: + case LogicalTypeId::USMALLINT: + case LogicalTypeId::UINTEGER: + case LogicalTypeId::UBIGINT: + case LogicalTypeId::FLOAT: + case LogicalTypeId::DOUBLE: + case LogicalTypeId::BLOB: + case LogicalTypeId::DATE: + case LogicalTypeId::DECIMAL: + case LogicalTypeId::TIMESTAMP: + case LogicalTypeId::TIMESTAMP_TZ: + case LogicalTypeId::VARCHAR: + return input; + case LogicalTypeId::LIST: + throw NotImplementedException("PC does not support arrays - unsupported type \"%s\"", input.ToString()); + case LogicalTypeId::STRUCT: + case LogicalTypeId::MAP: + case LogicalTypeId::UNION: + throw NotImplementedException("PC does not support composite types - unsupported type \"%s\"", + input.ToString()); + case LogicalTypeId::TIMESTAMP_SEC: + case LogicalTypeId::TIMESTAMP_MS: + case LogicalTypeId::TIMESTAMP_NS: + return LogicalType::TIMESTAMP; + case LogicalTypeId::HUGEINT: + return LogicalType::DOUBLE; + default: + return LogicalType::VARCHAR; + } +} + +} // namespace duckdb diff --git a/vcpkg.json b/vcpkg.json index 94ab7f7..17661df 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -4,6 +4,7 @@ "name": "avro-cpp", "features": ["snappy"] }, + "curl", "openssl" ], "vcpkg-configuration": { From f3d5d8785704b21a6beda9661d369f22889ed5e3 Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Tue, 14 Jan 2025 22:40:33 -0500 Subject: [PATCH 02/13] Rename files uc -> ic --- CMakeLists.txt | 22 +++++++++---------- src/{uc_api.cpp => catalog_api.cpp} | 4 ++-- src/{uc_utils.cpp => catalog_utils.cpp} | 6 ++--- src/iceberg_extension.cpp | 10 ++++----- src/include/{uc_api.hpp => catalog_api.hpp} | 7 ------ .../{uc_utils.hpp => catalog_utils.hpp} | 9 +------- .../{uc_catalog.hpp => ic_catalog.hpp} | 9 +------- ...{uc_catalog_set.hpp => ic_catalog_set.hpp} | 7 ------ ...c_schema_entry.hpp => ic_schema_entry.hpp} | 11 ++-------- .../{uc_schema_set.hpp => ic_schema_set.hpp} | 11 ++-------- ...{uc_table_entry.hpp => ic_table_entry.hpp} | 9 +------- .../{uc_table_set.hpp => ic_table_set.hpp} | 11 ++-------- ...{uc_transaction.hpp => ic_transaction.hpp} | 9 +------- ...manager.hpp => ic_transaction_manager.hpp} | 15 ++++--------- .../{uc_catalog.cpp => ic_catalog.cpp} | 6 ++--- ...{uc_catalog_set.cpp => ic_catalog_set.cpp} | 6 ++--- ...{uc_clear_cache.cpp => ic_clear_cache.cpp} | 2 +- ...c_schema_entry.cpp => ic_schema_entry.cpp} | 10 ++++----- .../{uc_schema_set.cpp => ic_schema_set.cpp} | 12 +++++----- ...{uc_table_entry.cpp => ic_table_entry.cpp} | 18 +++++++-------- .../{uc_table_set.cpp => ic_table_set.cpp} | 16 +++++++------- ...{uc_transaction.cpp => ic_transaction.cpp} | 10 ++++----- ...manager.cpp => ic_transaction_manager.cpp} | 16 +++++++------- 23 files changed, 83 insertions(+), 153 deletions(-) rename src/{uc_api.cpp => catalog_api.cpp} (99%) rename src/{uc_utils.cpp => catalog_utils.cpp} (98%) rename src/include/{uc_api.hpp => catalog_api.hpp} (85%) rename src/include/{uc_utils.hpp => catalog_utils.hpp} (70%) rename src/include/storage/{uc_catalog.hpp => ic_catalog.hpp} (91%) rename src/include/storage/{uc_catalog_set.hpp => ic_catalog_set.hpp} (82%) rename src/include/storage/{uc_schema_entry.hpp => ic_schema_entry.hpp} (88%) rename src/include/storage/{uc_schema_set.hpp => ic_schema_set.hpp} (52%) rename src/include/storage/{uc_table_entry.hpp => ic_table_entry.hpp} (84%) rename src/include/storage/{uc_table_set.hpp => ic_table_set.hpp} (77%) rename src/include/storage/{uc_transaction.hpp => ic_transaction.hpp} (71%) rename src/include/storage/{uc_transaction_manager.hpp => ic_transaction_manager.hpp} (59%) rename src/storage/{uc_catalog.cpp => ic_catalog.cpp} (97%) rename src/storage/{uc_catalog_set.cpp => ic_catalog_set.cpp} (93%) rename src/storage/{uc_clear_cache.cpp => ic_clear_cache.cpp} (97%) rename src/storage/{uc_schema_entry.cpp => ic_schema_entry.cpp} (96%) rename src/storage/{uc_schema_set.cpp => ic_schema_set.cpp} (76%) rename src/storage/{uc_table_entry.cpp => ic_table_entry.cpp} (92%) rename src/storage/{uc_table_set.cpp => ic_table_set.cpp} (90%) rename src/storage/{uc_transaction.cpp => ic_transaction.cpp} (87%) rename src/storage/{uc_transaction_manager.cpp => ic_transaction_manager.cpp} (72%) diff --git a/CMakeLists.txt b/CMakeLists.txt index db7837f..0f57eb2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,23 +13,23 @@ include_directories(src/include) set(EXTENSION_SOURCES src/iceberg_extension.cpp src/iceberg_functions.cpp - src/uc_api.cpp - src/uc_utils.cpp + src/catalog_api.cpp + src/catalog_utils.cpp src/common/utils.cpp src/common/schema.cpp src/common/iceberg.cpp src/iceberg_functions/iceberg_snapshots.cpp src/iceberg_functions/iceberg_scan.cpp src/iceberg_functions/iceberg_metadata.cpp - src/storage/uc_catalog.cpp - src/storage/uc_catalog_set.cpp - src/storage/uc_clear_cache.cpp - src/storage/uc_schema_entry.cpp - src/storage/uc_schema_set.cpp - src/storage/uc_table_entry.cpp - src/storage/uc_table_set.cpp - src/storage/uc_transaction.cpp - src/storage/uc_transaction_manager.cpp + src/storage/ic_catalog.cpp + src/storage/ic_catalog_set.cpp + src/storage/ic_clear_cache.cpp + src/storage/ic_schema_entry.cpp + src/storage/ic_schema_set.cpp + src/storage/ic_table_entry.cpp + src/storage/ic_table_set.cpp + src/storage/ic_transaction.cpp + src/storage/ic_transaction_manager.cpp ) add_library(${EXTENSION_NAME} STATIC ${EXTENSION_SOURCES}) diff --git a/src/uc_api.cpp b/src/catalog_api.cpp similarity index 99% rename from src/uc_api.cpp rename to src/catalog_api.cpp index 6a6e8d7..78bc9ea 100644 --- a/src/uc_api.cpp +++ b/src/catalog_api.cpp @@ -1,5 +1,5 @@ -#include "uc_api.hpp" -#include "storage/uc_catalog.hpp" +#include "catalog_api.hpp" +#include "storage/ic_catalog.hpp" #include "yyjson.hpp" #include #include diff --git a/src/uc_utils.cpp b/src/catalog_utils.cpp similarity index 98% rename from src/uc_utils.cpp rename to src/catalog_utils.cpp index 5acae60..591f6ab 100644 --- a/src/uc_utils.cpp +++ b/src/catalog_utils.cpp @@ -1,7 +1,7 @@ -#include "uc_utils.hpp" +#include "catalog_utils.hpp" #include "duckdb/common/operator/cast_operators.hpp" -#include "storage/uc_schema_entry.hpp" -#include "storage/uc_transaction.hpp" +#include "storage/ic_schema_entry.hpp" +#include "storage/ic_transaction.hpp" #include diff --git a/src/iceberg_extension.cpp b/src/iceberg_extension.cpp index 2d4a3b9..10937ff 100644 --- a/src/iceberg_extension.cpp +++ b/src/iceberg_extension.cpp @@ -1,8 +1,8 @@ #define DUCKDB_EXTENSION_MAIN #include "iceberg_extension.hpp" -#include "storage/uc_catalog.hpp" -#include "storage/uc_transaction_manager.hpp" +#include "storage/ic_catalog.hpp" +#include "storage/ic_transaction_manager.hpp" #include "duckdb.hpp" #include "duckdb/main/secret/secret_manager.hpp" @@ -19,7 +19,7 @@ #include "yyjson.hpp" #include "duckdb/main/extension_util.hpp" #include -#include "uc_api.hpp" +#include "catalog_api.hpp" namespace duckdb { @@ -136,8 +136,8 @@ static unique_ptr PolarisCatalogAttach(StorageExtensionInfo *storage_in static unique_ptr CreateTransactionManager(StorageExtensionInfo *storage_info, AttachedDatabase &db, Catalog &catalog) { - auto &uc_catalog = catalog.Cast(); - return make_uniq(db, uc_catalog); + auto &ic_catalog = catalog.Cast(); + return make_uniq(db, ic_catalog); } class UCCatalogStorageExtension : public StorageExtension { diff --git a/src/include/uc_api.hpp b/src/include/catalog_api.hpp similarity index 85% rename from src/include/uc_api.hpp rename to src/include/catalog_api.hpp index a65f208..ff25711 100644 --- a/src/include/uc_api.hpp +++ b/src/include/catalog_api.hpp @@ -1,10 +1,3 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// src/include/uc_api.hpp -// -// -//===----------------------------------------------------------------------===// #pragma once diff --git a/src/include/uc_utils.hpp b/src/include/catalog_utils.hpp similarity index 70% rename from src/include/uc_utils.hpp rename to src/include/catalog_utils.hpp index 8927943..e7db342 100644 --- a/src/include/uc_utils.hpp +++ b/src/include/catalog_utils.hpp @@ -1,15 +1,8 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// mysql_utils.hpp -// -// -//===----------------------------------------------------------------------===// #pragma once #include "duckdb.hpp" -#include "uc_api.hpp" +#include "catalog_api.hpp" namespace duckdb { class UCSchemaEntry; diff --git a/src/include/storage/uc_catalog.hpp b/src/include/storage/ic_catalog.hpp similarity index 91% rename from src/include/storage/uc_catalog.hpp rename to src/include/storage/ic_catalog.hpp index 22df4f0..42da0b4 100644 --- a/src/include/storage/uc_catalog.hpp +++ b/src/include/storage/ic_catalog.hpp @@ -1,17 +1,10 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// storage/uc_catalog.hpp -// -// -//===----------------------------------------------------------------------===// #pragma once #include "duckdb/catalog/catalog.hpp" #include "duckdb/function/table_function.hpp" #include "duckdb/common/enums/access_mode.hpp" -#include "storage/uc_schema_set.hpp" +#include "storage/ic_schema_set.hpp" namespace duckdb { class UCSchemaEntry; diff --git a/src/include/storage/uc_catalog_set.hpp b/src/include/storage/ic_catalog_set.hpp similarity index 82% rename from src/include/storage/uc_catalog_set.hpp rename to src/include/storage/ic_catalog_set.hpp index 8cd9ba3..d21af26 100644 --- a/src/include/storage/uc_catalog_set.hpp +++ b/src/include/storage/ic_catalog_set.hpp @@ -1,10 +1,3 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// storage/uc_catalog_set.hpp -// -// -//===----------------------------------------------------------------------===// #pragma once diff --git a/src/include/storage/uc_schema_entry.hpp b/src/include/storage/ic_schema_entry.hpp similarity index 88% rename from src/include/storage/uc_schema_entry.hpp rename to src/include/storage/ic_schema_entry.hpp index b9d11f1..9cde3da 100644 --- a/src/include/storage/uc_schema_entry.hpp +++ b/src/include/storage/ic_schema_entry.hpp @@ -1,16 +1,9 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// storage/uc_schema_entry.hpp -// -// -//===----------------------------------------------------------------------===// #pragma once -#include "uc_api.hpp" +#include "catalog_api.hpp" #include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" -#include "storage/uc_table_set.hpp" +#include "storage/ic_table_set.hpp" namespace duckdb { class UCTransaction; diff --git a/src/include/storage/uc_schema_set.hpp b/src/include/storage/ic_schema_set.hpp similarity index 52% rename from src/include/storage/uc_schema_set.hpp rename to src/include/storage/ic_schema_set.hpp index 38bfb2f..c019f6e 100644 --- a/src/include/storage/uc_schema_set.hpp +++ b/src/include/storage/ic_schema_set.hpp @@ -1,15 +1,8 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// storage/uc_schema_set.hpp -// -// -//===----------------------------------------------------------------------===// #pragma once -#include "storage/uc_catalog_set.hpp" -#include "storage/uc_schema_entry.hpp" +#include "storage/ic_catalog_set.hpp" +#include "storage/ic_schema_entry.hpp" namespace duckdb { struct CreateSchemaInfo; diff --git a/src/include/storage/uc_table_entry.hpp b/src/include/storage/ic_table_entry.hpp similarity index 84% rename from src/include/storage/uc_table_entry.hpp rename to src/include/storage/ic_table_entry.hpp index 50b2528..5b25e82 100644 --- a/src/include/storage/uc_table_entry.hpp +++ b/src/include/storage/ic_table_entry.hpp @@ -1,14 +1,7 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// storage/uc_table_entry.hpp -// -// -//===----------------------------------------------------------------------===// #pragma once -#include "uc_api.hpp" +#include "catalog_api.hpp" #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" #include "duckdb/parser/parsed_data/create_table_info.hpp" diff --git a/src/include/storage/uc_table_set.hpp b/src/include/storage/ic_table_set.hpp similarity index 77% rename from src/include/storage/uc_table_set.hpp rename to src/include/storage/ic_table_set.hpp index fd8ec0b..c73b1bf 100644 --- a/src/include/storage/uc_table_set.hpp +++ b/src/include/storage/ic_table_set.hpp @@ -1,15 +1,8 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// storage/uc_table_set.hpp -// -// -//===----------------------------------------------------------------------===// #pragma once -#include "storage/uc_catalog_set.hpp" -#include "storage/uc_table_entry.hpp" +#include "storage/ic_catalog_set.hpp" +#include "storage/ic_table_entry.hpp" namespace duckdb { struct CreateTableInfo; diff --git a/src/include/storage/uc_transaction.hpp b/src/include/storage/ic_transaction.hpp similarity index 71% rename from src/include/storage/uc_transaction.hpp rename to src/include/storage/ic_transaction.hpp index 86a1423..16d6da8 100644 --- a/src/include/storage/uc_transaction.hpp +++ b/src/include/storage/ic_transaction.hpp @@ -1,10 +1,3 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// storage/uc_transaction.hpp -// -// -//===----------------------------------------------------------------------===// #pragma once @@ -19,7 +12,7 @@ enum class UCTransactionState { TRANSACTION_NOT_YET_STARTED, TRANSACTION_STARTED class UCTransaction : public Transaction { public: - UCTransaction(UCCatalog &uc_catalog, TransactionManager &manager, ClientContext &context); + UCTransaction(UCCatalog &ic_catalog, TransactionManager &manager, ClientContext &context); ~UCTransaction() override; void Start(); diff --git a/src/include/storage/uc_transaction_manager.hpp b/src/include/storage/ic_transaction_manager.hpp similarity index 59% rename from src/include/storage/uc_transaction_manager.hpp rename to src/include/storage/ic_transaction_manager.hpp index b368f51..7f9fb80 100644 --- a/src/include/storage/uc_transaction_manager.hpp +++ b/src/include/storage/ic_transaction_manager.hpp @@ -1,22 +1,15 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// storage/uc_transaction_manager.hpp -// -// -//===----------------------------------------------------------------------===// #pragma once #include "duckdb/transaction/transaction_manager.hpp" -#include "storage/uc_catalog.hpp" -#include "storage/uc_transaction.hpp" +#include "storage/ic_catalog.hpp" +#include "storage/ic_transaction.hpp" namespace duckdb { class UCTransactionManager : public TransactionManager { public: - UCTransactionManager(AttachedDatabase &db_p, UCCatalog &uc_catalog); + UCTransactionManager(AttachedDatabase &db_p, UCCatalog &ic_catalog); Transaction &StartTransaction(ClientContext &context) override; ErrorData CommitTransaction(ClientContext &context, Transaction &transaction) override; @@ -25,7 +18,7 @@ class UCTransactionManager : public TransactionManager { void Checkpoint(ClientContext &context, bool force = false) override; private: - UCCatalog &uc_catalog; + UCCatalog &ic_catalog; mutex transaction_lock; reference_map_t> transactions; }; diff --git a/src/storage/uc_catalog.cpp b/src/storage/ic_catalog.cpp similarity index 97% rename from src/storage/uc_catalog.cpp rename to src/storage/ic_catalog.cpp index 3c23771..f01387c 100644 --- a/src/storage/uc_catalog.cpp +++ b/src/storage/ic_catalog.cpp @@ -1,6 +1,6 @@ -#include "storage/uc_catalog.hpp" -#include "storage/uc_schema_entry.hpp" -#include "storage/uc_transaction.hpp" +#include "storage/ic_catalog.hpp" +#include "storage/ic_schema_entry.hpp" +#include "storage/ic_transaction.hpp" #include "duckdb/storage/database_size.hpp" #include "duckdb/parser/parsed_data/drop_info.hpp" #include "duckdb/parser/parsed_data/create_schema_info.hpp" diff --git a/src/storage/uc_catalog_set.cpp b/src/storage/ic_catalog_set.cpp similarity index 93% rename from src/storage/uc_catalog_set.cpp rename to src/storage/ic_catalog_set.cpp index 6451087..7f7a26c 100644 --- a/src/storage/uc_catalog_set.cpp +++ b/src/storage/ic_catalog_set.cpp @@ -1,7 +1,7 @@ -#include "storage/uc_catalog_set.hpp" -#include "storage/uc_transaction.hpp" +#include "storage/ic_catalog_set.hpp" +#include "storage/ic_transaction.hpp" #include "duckdb/parser/parsed_data/drop_info.hpp" -#include "storage/uc_schema_entry.hpp" +#include "storage/ic_schema_entry.hpp" namespace duckdb { diff --git a/src/storage/uc_clear_cache.cpp b/src/storage/ic_clear_cache.cpp similarity index 97% rename from src/storage/uc_clear_cache.cpp rename to src/storage/ic_clear_cache.cpp index 2c51abb..d4346b8 100644 --- a/src/storage/uc_clear_cache.cpp +++ b/src/storage/ic_clear_cache.cpp @@ -3,7 +3,7 @@ #include "duckdb/parser/parsed_data/create_table_function_info.hpp" #include "duckdb/main/database_manager.hpp" #include "duckdb/main/attached_database.hpp" -#include "storage/uc_catalog.hpp" +#include "storage/ic_catalog.hpp" namespace duckdb { diff --git a/src/storage/uc_schema_entry.cpp b/src/storage/ic_schema_entry.cpp similarity index 96% rename from src/storage/uc_schema_entry.cpp rename to src/storage/ic_schema_entry.cpp index 7bc93ce..57cd7d2 100644 --- a/src/storage/uc_schema_entry.cpp +++ b/src/storage/ic_schema_entry.cpp @@ -1,6 +1,6 @@ -#include "storage/uc_schema_entry.hpp" -#include "storage/uc_table_entry.hpp" -#include "storage/uc_transaction.hpp" +#include "storage/ic_schema_entry.hpp" +#include "storage/ic_table_entry.hpp" +#include "storage/ic_transaction.hpp" #include "duckdb/parser/parsed_data/create_view_info.hpp" #include "duckdb/parser/parsed_data/create_index_info.hpp" #include "duckdb/planner/parsed_data/bound_create_table_info.hpp" @@ -74,8 +74,8 @@ optional_ptr UCSchemaEntry::CreateView(CatalogTransaction transact throw NotImplementedException("REPLACE ON CONFLICT in CreateView"); } } - auto &uc_transaction = GetUCTransaction(transaction); - // uc_transaction.Query(GetUCCreateView(info)); + auto &ic_transaction = GetUCTransaction(transaction); + // ic_transaction.Query(GetUCCreateView(info)); return tables.RefreshTable(transaction.GetContext(), info.view_name); } diff --git a/src/storage/uc_schema_set.cpp b/src/storage/ic_schema_set.cpp similarity index 76% rename from src/storage/uc_schema_set.cpp rename to src/storage/ic_schema_set.cpp index 6efb318..3f85a09 100644 --- a/src/storage/uc_schema_set.cpp +++ b/src/storage/ic_schema_set.cpp @@ -1,7 +1,7 @@ -#include "storage/uc_schema_set.hpp" -#include "storage/uc_catalog.hpp" -#include "uc_api.hpp" -#include "storage/uc_transaction.hpp" +#include "storage/ic_schema_set.hpp" +#include "storage/ic_catalog.hpp" +#include "catalog_api.hpp" +#include "storage/ic_transaction.hpp" #include "duckdb/parser/parsed_data/create_schema_info.hpp" #include "duckdb/catalog/catalog.hpp" @@ -18,8 +18,8 @@ static bool IsInternalTable(const string &catalog, const string &schema) { } void UCSchemaSet::LoadEntries(ClientContext &context) { - auto &uc_catalog = catalog.Cast(); - auto tables = UCAPI::GetSchemas(catalog.GetName(), uc_catalog.internal_name, uc_catalog.credentials); + auto &ic_catalog = catalog.Cast(); + auto tables = UCAPI::GetSchemas(catalog.GetName(), ic_catalog.internal_name, ic_catalog.credentials); for (const auto &schema : tables) { CreateSchemaInfo info; diff --git a/src/storage/uc_table_entry.cpp b/src/storage/ic_table_entry.cpp similarity index 92% rename from src/storage/uc_table_entry.cpp rename to src/storage/ic_table_entry.cpp index 7691a2f..2928870 100644 --- a/src/storage/uc_table_entry.cpp +++ b/src/storage/ic_table_entry.cpp @@ -1,7 +1,7 @@ -#include "storage/uc_catalog.hpp" -#include "storage/uc_schema_entry.hpp" -#include "storage/uc_table_entry.hpp" -#include "storage/uc_transaction.hpp" +#include "storage/ic_catalog.hpp" +#include "storage/ic_schema_entry.hpp" +#include "storage/ic_table_entry.hpp" +#include "storage/ic_transaction.hpp" #include "duckdb/storage/statistics/base_statistics.hpp" #include "duckdb/storage/table_storage_info.hpp" #include "duckdb/main/extension_util.hpp" @@ -9,7 +9,7 @@ #include "duckdb/main/secret/secret_manager.hpp" #include "duckdb/catalog/catalog_entry/table_function_catalog_entry.hpp" #include "duckdb/parser/tableref/table_function_ref.hpp" -#include "uc_api.hpp" +#include "catalog_api.hpp" #include "../../duckdb/third_party/catch/catch.hpp" #include "duckdb/planner/binder.hpp" #include "duckdb/planner/tableref/bound_table_function.hpp" @@ -57,7 +57,7 @@ struct MyIcebergFunctionData : public FunctionData { TableFunction UCTableEntry::GetScanFunction(ClientContext &context, unique_ptr &bind_data) { auto &db = DatabaseInstance::GetDatabase(context); - auto &uc_catalog = catalog.Cast(); + auto &ic_catalog = catalog.Cast(); auto &parquet_function_set = ExtensionUtil::GetTableFunction(db, "parquet_scan"); auto parquet_scan_function = parquet_function_set.functions.GetFunctionByArguments(context, {LogicalType::VARCHAR}); @@ -76,11 +76,11 @@ TableFunction UCTableEntry::GetScanFunction(ClientContext &context, unique_ptrschema_name, table_data->name, uc_catalog.credentials); + ic_catalog.internal_name, table_data->schema_name, table_data->name, ic_catalog.credentials); // Inject secret into secret manager scoped to this path CreateSecretInfo info(OnCreateConflict::REPLACE_ON_CONFLICT, SecretPersistType::TEMPORARY); - info.name = "__internal_uc_" + table_data->table_id; + info.name = "__internal_ic_" + table_data->table_id; info.type = "s3"; info.provider = "config"; info.storage_type = "memory"; @@ -88,7 +88,7 @@ TableFunction UCTableEntry::GetScanFunction(ClientContext &context, unique_ptr(); + auto &ic_catalog = catalog.Cast(); // TODO: handle out-of-order columns using position property - auto tables = UCAPI::GetTables(catalog.GetName(), catalog.GetDBPath(), schema.name, uc_catalog.credentials); + auto tables = UCAPI::GetTables(catalog.GetName(), catalog.GetDBPath(), schema.name, ic_catalog.credentials); for (auto &table : tables) { D_ASSERT(schema.name == table.schema_name); diff --git a/src/storage/uc_transaction.cpp b/src/storage/ic_transaction.cpp similarity index 87% rename from src/storage/uc_transaction.cpp rename to src/storage/ic_transaction.cpp index 2942458..fbff966 100644 --- a/src/storage/uc_transaction.cpp +++ b/src/storage/ic_transaction.cpp @@ -1,14 +1,14 @@ -#include "storage/uc_transaction.hpp" -#include "storage/uc_catalog.hpp" +#include "storage/ic_transaction.hpp" +#include "storage/ic_catalog.hpp" #include "duckdb/parser/parsed_data/create_view_info.hpp" #include "duckdb/catalog/catalog_entry/index_catalog_entry.hpp" #include "duckdb/catalog/catalog_entry/view_catalog_entry.hpp" namespace duckdb { -UCTransaction::UCTransaction(UCCatalog &uc_catalog, TransactionManager &manager, ClientContext &context) - : Transaction(manager, context), access_mode(uc_catalog.access_mode) { - // connection = UCConnection::Open(uc_catalog.path); +UCTransaction::UCTransaction(UCCatalog &ic_catalog, TransactionManager &manager, ClientContext &context) + : Transaction(manager, context), access_mode(ic_catalog.access_mode) { + // connection = UCConnection::Open(ic_catalog.path); } UCTransaction::~UCTransaction() = default; diff --git a/src/storage/uc_transaction_manager.cpp b/src/storage/ic_transaction_manager.cpp similarity index 72% rename from src/storage/uc_transaction_manager.cpp rename to src/storage/ic_transaction_manager.cpp index afd7e07..bf0f319 100644 --- a/src/storage/uc_transaction_manager.cpp +++ b/src/storage/ic_transaction_manager.cpp @@ -1,14 +1,14 @@ -#include "storage/uc_transaction_manager.hpp" +#include "storage/ic_transaction_manager.hpp" #include "duckdb/main/attached_database.hpp" namespace duckdb { -UCTransactionManager::UCTransactionManager(AttachedDatabase &db_p, UCCatalog &uc_catalog) - : TransactionManager(db_p), uc_catalog(uc_catalog) { +UCTransactionManager::UCTransactionManager(AttachedDatabase &db_p, UCCatalog &ic_catalog) + : TransactionManager(db_p), ic_catalog(ic_catalog) { } Transaction &UCTransactionManager::StartTransaction(ClientContext &context) { - auto transaction = make_uniq(uc_catalog, *this, context); + auto transaction = make_uniq(ic_catalog, *this, context); transaction->Start(); auto &result = *transaction; lock_guard l(transaction_lock); @@ -17,16 +17,16 @@ Transaction &UCTransactionManager::StartTransaction(ClientContext &context) { } ErrorData UCTransactionManager::CommitTransaction(ClientContext &context, Transaction &transaction) { - auto &uc_transaction = transaction.Cast(); - uc_transaction.Commit(); + auto &ic_transaction = transaction.Cast(); + ic_transaction.Commit(); lock_guard l(transaction_lock); transactions.erase(transaction); return ErrorData(); } void UCTransactionManager::RollbackTransaction(Transaction &transaction) { - auto &uc_transaction = transaction.Cast(); - uc_transaction.Rollback(); + auto &ic_transaction = transaction.Cast(); + ic_transaction.Rollback(); lock_guard l(transaction_lock); transactions.erase(transaction); } From 1be2fa33cae9fc657b4625f77cfadae4dd8b37bc Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Tue, 14 Jan 2025 22:50:52 -0500 Subject: [PATCH 03/13] Started renaming classes --- src/catalog_api.cpp | 32 +++++++-------- src/catalog_utils.cpp | 12 +++--- src/iceberg_extension.cpp | 18 ++++----- src/include/catalog_api.hpp | 24 +++++------ src/include/catalog_utils.hpp | 10 ++--- src/include/storage/ic_catalog.hpp | 8 ++-- src/include/storage/ic_catalog_set.hpp | 14 +++---- src/include/storage/ic_schema_entry.hpp | 12 +++--- src/include/storage/ic_schema_set.hpp | 2 +- src/include/storage/ic_table_entry.hpp | 2 +- src/include/storage/ic_table_set.hpp | 6 +-- src/include/storage/ic_transaction.hpp | 10 ++--- .../storage/ic_transaction_manager.hpp | 2 +- src/storage/ic_catalog.cpp | 4 +- src/storage/ic_catalog_set.cpp | 22 +++++----- src/storage/ic_schema_entry.cpp | 40 +++++++++---------- src/storage/ic_schema_set.cpp | 8 ++-- src/storage/ic_table_entry.cpp | 4 +- src/storage/ic_table_set.cpp | 14 +++---- src/storage/ic_transaction.cpp | 18 ++++----- src/storage/ic_transaction_manager.cpp | 8 ++-- 21 files changed, 135 insertions(+), 135 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index 78bc9ea..dd40f7b 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -8,7 +8,7 @@ namespace duckdb { -//! We use a global here to store the path that is selected on the UCAPI::InitializeCurl call +//! We use a global here to store the path that is selected on the IBAPI::InitializeCurl call static string SELECTED_CURL_CERT_PATH = ""; static size_t GetRequestWriteCallback(void *contents, size_t size, size_t nmemb, void *userp) { @@ -137,7 +137,7 @@ static string PostRequest(const string &url, const string &post_data, curl_slist } -static duckdb_yyjson::yyjson_val *GetTableMetadata(const string &internal, const string &schema, const string &table, UCCredentials credentials) { +static duckdb_yyjson::yyjson_val *GetTableMetadata(const string &internal, const string &schema, const string &table, IBCredentials credentials) { struct curl_slist *extra_headers = NULL; extra_headers = curl_slist_append(extra_headers, "X-Iceberg-Access-Delegation: vended-credentials"); auto api_result = GetRequest( @@ -176,16 +176,16 @@ static string TryGetStrFromObject(duckdb_yyjson::yyjson_val *obj, const string & fail_on_missing); } -void UCAPI::InitializeCurl() { +void IBAPI::InitializeCurl() { SelectCurlCertPath(); } -vector UCAPI::GetCatalogs(const string &catalog, UCCredentials credentials) { - throw NotImplementedException("UCAPI::GetCatalogs"); +vector IBAPI::GetCatalogs(const string &catalog, IBCredentials credentials) { + throw NotImplementedException("IBAPI::GetCatalogs"); } -static UCAPIColumnDefinition ParseColumnDefinition(duckdb_yyjson::yyjson_val *column_def) { - UCAPIColumnDefinition result; +static IBAPIColumnDefinition ParseColumnDefinition(duckdb_yyjson::yyjson_val *column_def) { + IBAPIColumnDefinition result; result.name = TryGetStrFromObject(column_def, "name"); result.type_text = TryGetStrFromObject(column_def, "type"); @@ -196,8 +196,8 @@ static UCAPIColumnDefinition ParseColumnDefinition(duckdb_yyjson::yyjson_val *co return result; } -UCAPITableCredentials UCAPI::GetTableCredentials(const string &internal, const string &schema, const string &table, UCCredentials credentials) { - UCAPITableCredentials result; +IBAPITableCredentials IBAPI::GetTableCredentials(const string &internal, const string &schema, const string &table, IBCredentials credentials) { + IBAPITableCredentials result; duckdb_yyjson::yyjson_val *root = GetTableMetadata(internal, schema, table, credentials); auto *aws_temp_credentials = yyjson_obj_get(root, "config"); @@ -211,7 +211,7 @@ UCAPITableCredentials UCAPI::GetTableCredentials(const string &internal, const s return result; } -string UCAPI::GetToken(string id, string secret, string endpoint) { +string IBAPI::GetToken(string id, string secret, string endpoint) { string post_data = "grant_type=client_credentials&client_id=" + id + "&client_secret=" + secret + "&scope=PRINCIPAL_ROLE:ALL"; string api_result = PostRequest(endpoint + "/v1/oauth/tokens", post_data); @@ -228,8 +228,8 @@ string UCAPI::GetToken(string id, string secret, string endpoint) { return TryGetStrFromObject(root, "access_token"); } -vector UCAPI::GetTables(const string &catalog, const string &internal, const string &schema, UCCredentials credentials) { - vector result; +vector IBAPI::GetTables(const string &catalog, const string &internal, const string &schema, IBCredentials credentials) { + vector result; auto api_result = GetRequest(credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables", credentials.token); @@ -242,7 +242,7 @@ vector UCAPI::GetTables(const string &catalog, const string &interna size_t idx, max; duckdb_yyjson::yyjson_val *table; yyjson_arr_foreach(tables, idx, max, table) { - UCAPITable table_result; + IBAPITable table_result; table_result.catalog_name = catalog; table_result.schema_name = schema; table_result.name = TryGetStrFromObject(table, "name"); @@ -285,8 +285,8 @@ vector UCAPI::GetTables(const string &catalog, const string &interna return result; } -vector UCAPI::GetSchemas(const string &catalog, const string &internal, UCCredentials credentials) { - vector result; +vector IBAPI::GetSchemas(const string &catalog, const string &internal, IBCredentials credentials) { + vector result; auto api_result = GetRequest(credentials.endpoint + "/v1/" + internal + "/namespaces", credentials.token); @@ -306,7 +306,7 @@ vector UCAPI::GetSchemas(const string &catalog, const string &inter size_t idx, max; duckdb_yyjson::yyjson_val *schema; yyjson_arr_foreach(schemas, idx, max, schema) { - UCAPISchema schema_result; + IBAPISchema schema_result; schema_result.catalog_name = catalog; duckdb_yyjson::yyjson_val *value = yyjson_arr_get(schema, 0); diff --git a/src/catalog_utils.cpp b/src/catalog_utils.cpp index 591f6ab..e6f3e52 100644 --- a/src/catalog_utils.cpp +++ b/src/catalog_utils.cpp @@ -7,7 +7,7 @@ namespace duckdb { -string UCUtils::TypeToString(const LogicalType &input) { +string IBUtils::TypeToString(const LogicalType &input) { switch (input.id()) { case LogicalType::VARCHAR: return "TEXT"; @@ -28,7 +28,7 @@ string UCUtils::TypeToString(const LogicalType &input) { } } -LogicalType UCUtils::TypeToLogicalType(ClientContext &context, const string &type_text) { +LogicalType IBUtils::TypeToLogicalType(ClientContext &context, const string &type_text) { if (type_text == "tinyint") { return LogicalType::TINYINT; } else if (type_text == "smallint") { @@ -71,7 +71,7 @@ LogicalType UCUtils::TypeToLogicalType(ClientContext &context, const string &typ size_t type_end = type_text.rfind('>'); // find last, to deal with nested if (type_end != string::npos) { auto child_type_str = type_text.substr(6, type_end - 6); - auto child_type = UCUtils::TypeToLogicalType(context, child_type_str); + auto child_type = IBUtils::TypeToLogicalType(context, child_type_str); return LogicalType::LIST(child_type); } } else if (type_text.find("map<") == 0) { @@ -96,7 +96,7 @@ LogicalType UCUtils::TypeToLogicalType(ClientContext &context, const string &typ } } auto child_str = type_text.substr(cur, next_sep - cur); - auto child_type = UCUtils::TypeToLogicalType(context, child_str); + auto child_type = IBUtils::TypeToLogicalType(context, child_str); key_val.push_back(child_type); if (next_sep == type_end) { break; @@ -134,7 +134,7 @@ LogicalType UCUtils::TypeToLogicalType(ClientContext &context, const string &typ throw NotImplementedException("Invalid struct child type specifier: %s", child_str); } auto child_name = child_str.substr(0, type_sep); - auto child_type = UCUtils::TypeToLogicalType(context, child_str.substr(type_sep + 1, string::npos)); + auto child_type = IBUtils::TypeToLogicalType(context, child_str.substr(type_sep + 1, string::npos)); children.push_back({child_name, child_type}); if (next_sep == type_end) { break; @@ -150,7 +150,7 @@ LogicalType UCUtils::TypeToLogicalType(ClientContext &context, const string &typ return LogicalType::VARCHAR; } -LogicalType UCUtils::ToUCType(const LogicalType &input) { +LogicalType IBUtils::ToUCType(const LogicalType &input) { // todo do we need this mapping? throw NotImplementedException("ToUCType not yet implemented"); switch (input.id()) { diff --git a/src/iceberg_extension.cpp b/src/iceberg_extension.cpp index 10937ff..5b5d9c5 100644 --- a/src/iceberg_extension.cpp +++ b/src/iceberg_extension.cpp @@ -23,7 +23,7 @@ namespace duckdb { -static unique_ptr CreatePolarisSecretFunction(ClientContext &, CreateSecretInput &input) { +static unique_ptr CreateCatalogSecretFunction(ClientContext &, CreateSecretInput &input) { // apply any overridden settings vector prefix_paths; auto result = make_uniq(prefix_paths, "iceberg", "config", input.name); @@ -42,7 +42,7 @@ static unique_ptr CreatePolarisSecretFunction(ClientContext &, Creat } // Get token from catalog - result->secret_map["token"] = UCAPI::GetToken( + result->secret_map["token"] = IBAPI::GetToken( result->secret_map["client_id"].ToString(), result->secret_map["client_secret"].ToString(), result->secret_map["endpoint"].ToString()); @@ -53,7 +53,7 @@ static unique_ptr CreatePolarisSecretFunction(ClientContext &, Creat return std::move(result); } -static void SetPolarisSecretParameters(CreateSecretFunction &function) { +static void SetCatalogSecretParameters(CreateSecretFunction &function) { function.named_parameters["client_id"] = LogicalType::VARCHAR; function.named_parameters["client_secret"] = LogicalType::VARCHAR; function.named_parameters["endpoint"] = LogicalType::VARCHAR; @@ -77,10 +77,10 @@ unique_ptr GetSecret(ClientContext &context, const string &secret_n return nullptr; } -static unique_ptr PolarisCatalogAttach(StorageExtensionInfo *storage_info, ClientContext &context, +static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_info, ClientContext &context, AttachedDatabase &db, const string &name, AttachInfo &info, AccessMode access_mode) { - UCCredentials credentials; + IBCredentials credentials; // check if we have a secret provided string secret_name; @@ -143,7 +143,7 @@ static unique_ptr CreateTransactionManager(StorageExtensionI class UCCatalogStorageExtension : public StorageExtension { public: UCCatalogStorageExtension() { - attach = PolarisCatalogAttach; + attach = IcebergCatalogAttach; create_transaction_manager = CreateTransactionManager; } }; @@ -168,7 +168,7 @@ static void LoadInternal(DatabaseInstance &instance) { ExtensionUtil::RegisterFunction(instance, fun); } - UCAPI::InitializeCurl(); + IBAPI::InitializeCurl(); SecretType secret_type; secret_type.name = "iceberg"; @@ -176,8 +176,8 @@ static void LoadInternal(DatabaseInstance &instance) { secret_type.default_provider = "config"; ExtensionUtil::RegisterSecretType(instance, secret_type); - CreateSecretFunction secret_function = {"iceberg", "config", CreatePolarisSecretFunction}; - SetPolarisSecretParameters(secret_function); + CreateSecretFunction secret_function = {"iceberg", "config", CreateCatalogSecretFunction}; + SetCatalogSecretParameters(secret_function); ExtensionUtil::RegisterFunction(instance, secret_function); config.storage_extensions["iceberg"] = make_uniq(); diff --git a/src/include/catalog_api.hpp b/src/include/catalog_api.hpp index ff25711..e8ef25b 100644 --- a/src/include/catalog_api.hpp +++ b/src/include/catalog_api.hpp @@ -4,9 +4,9 @@ #include "duckdb/common/types.hpp" namespace duckdb { -struct UCCredentials; +struct IBCredentials; -struct UCAPIColumnDefinition { +struct IBAPIColumnDefinition { string name; string type_text; idx_t precision; @@ -14,7 +14,7 @@ struct UCAPIColumnDefinition { idx_t position; }; -struct UCAPITable { +struct IBAPITable { string table_id; string name; @@ -25,30 +25,30 @@ struct UCAPITable { string storage_location; - vector columns; + vector columns; }; -struct UCAPISchema { +struct IBAPISchema { string schema_name; string catalog_name; }; -struct UCAPITableCredentials { +struct IBAPITableCredentials { string key_id; string secret; string session_token; }; -class UCAPI { +class IBAPI { public: //! WARNING: not thread-safe. To be called once on extension initialization static void InitializeCurl(); - static UCAPITableCredentials GetTableCredentials(const string &internal, const string &schema, const string &table, UCCredentials credentials); - static vector GetCatalogs(const string &catalog, UCCredentials credentials); - static vector GetTables(const string &catalog, const string &internal, const string &schema, UCCredentials credentials); - static vector GetSchemas(const string &catalog, const string &internal, UCCredentials credentials); - static vector GetTablesInSchema(const string &catalog, const string &schema, UCCredentials credentials); + static IBAPITableCredentials GetTableCredentials(const string &internal, const string &schema, const string &table, IBCredentials credentials); + static vector GetCatalogs(const string &catalog, IBCredentials credentials); + static vector GetTables(const string &catalog, const string &internal, const string &schema, IBCredentials credentials); + static vector GetSchemas(const string &catalog, const string &internal, IBCredentials credentials); + static vector GetTablesInSchema(const string &catalog, const string &schema, IBCredentials credentials); static string GetToken(string id, string secret, string endpoint); }; } // namespace duckdb diff --git a/src/include/catalog_utils.hpp b/src/include/catalog_utils.hpp index e7db342..f294299 100644 --- a/src/include/catalog_utils.hpp +++ b/src/include/catalog_utils.hpp @@ -5,18 +5,18 @@ #include "catalog_api.hpp" namespace duckdb { -class UCSchemaEntry; -class UCTransaction; +class IBSchemaEntry; +class IBTransaction; -enum class UCTypeAnnotation { STANDARD, CAST_TO_VARCHAR, NUMERIC_AS_DOUBLE, CTID, JSONB, FIXED_LENGTH_CHAR }; +enum class IBTypeAnnotation { STANDARD, CAST_TO_VARCHAR, NUMERIC_AS_DOUBLE, CTID, JSONB, FIXED_LENGTH_CHAR }; struct UCType { idx_t oid = 0; - UCTypeAnnotation info = UCTypeAnnotation::STANDARD; + IBTypeAnnotation info = IBTypeAnnotation::STANDARD; vector children; }; -class UCUtils { +class IBUtils { public: static LogicalType ToUCType(const LogicalType &input); static LogicalType TypeToLogicalType(ClientContext &context, const string &columnDefinition); diff --git a/src/include/storage/ic_catalog.hpp b/src/include/storage/ic_catalog.hpp index 42da0b4..901ee05 100644 --- a/src/include/storage/ic_catalog.hpp +++ b/src/include/storage/ic_catalog.hpp @@ -7,9 +7,9 @@ #include "storage/ic_schema_set.hpp" namespace duckdb { -class UCSchemaEntry; +class IBSchemaEntry; -struct UCCredentials { +struct IBCredentials { string endpoint; string client_id; string client_secret; @@ -29,12 +29,12 @@ class UCClearCacheFunction : public TableFunction { class UCCatalog : public Catalog { public: explicit UCCatalog(AttachedDatabase &db_p, const string &internal_name, AccessMode access_mode, - UCCredentials credentials); + IBCredentials credentials); ~UCCatalog(); string internal_name; AccessMode access_mode; - UCCredentials credentials; + IBCredentials credentials; public: void Initialize(bool load_builtin) override; diff --git a/src/include/storage/ic_catalog_set.hpp b/src/include/storage/ic_catalog_set.hpp index d21af26..d2a0099 100644 --- a/src/include/storage/ic_catalog_set.hpp +++ b/src/include/storage/ic_catalog_set.hpp @@ -7,12 +7,12 @@ namespace duckdb { struct DropInfo; -class UCSchemaEntry; -class UCTransaction; +class IBSchemaEntry; +class IBTransaction; -class UCCatalogSet { +class IBCatalogSet { public: - UCCatalogSet(Catalog &catalog); + IBCatalogSet(Catalog &catalog); optional_ptr GetEntry(ClientContext &context, const string &name); virtual void DropEntry(ClientContext &context, DropInfo &info); @@ -34,14 +34,14 @@ class UCCatalogSet { bool is_loaded; }; -class UCInSchemaSet : public UCCatalogSet { +class UCInSchemaSet : public IBCatalogSet { public: - UCInSchemaSet(UCSchemaEntry &schema); + UCInSchemaSet(IBSchemaEntry &schema); optional_ptr CreateEntry(unique_ptr entry) override; protected: - UCSchemaEntry &schema; + IBSchemaEntry &schema; }; } // namespace duckdb diff --git a/src/include/storage/ic_schema_entry.hpp b/src/include/storage/ic_schema_entry.hpp index 9cde3da..3579f59 100644 --- a/src/include/storage/ic_schema_entry.hpp +++ b/src/include/storage/ic_schema_entry.hpp @@ -6,14 +6,14 @@ #include "storage/ic_table_set.hpp" namespace duckdb { -class UCTransaction; +class IBTransaction; -class UCSchemaEntry : public SchemaCatalogEntry { +class IBSchemaEntry : public SchemaCatalogEntry { public: - UCSchemaEntry(Catalog &catalog, CreateSchemaInfo &info); - ~UCSchemaEntry() override; + IBSchemaEntry(Catalog &catalog, CreateSchemaInfo &info); + ~IBSchemaEntry() override; - unique_ptr schema_data; + unique_ptr schema_data; public: optional_ptr CreateTable(CatalogTransaction transaction, BoundCreateTableInfo &info) override; @@ -37,7 +37,7 @@ class UCSchemaEntry : public SchemaCatalogEntry { optional_ptr GetEntry(CatalogTransaction transaction, CatalogType type, const string &name) override; private: - UCCatalogSet &GetCatalogSet(CatalogType type); + IBCatalogSet &GetCatalogSet(CatalogType type); private: UCTableSet tables; diff --git a/src/include/storage/ic_schema_set.hpp b/src/include/storage/ic_schema_set.hpp index c019f6e..365bf0a 100644 --- a/src/include/storage/ic_schema_set.hpp +++ b/src/include/storage/ic_schema_set.hpp @@ -7,7 +7,7 @@ namespace duckdb { struct CreateSchemaInfo; -class UCSchemaSet : public UCCatalogSet { +class UCSchemaSet : public IBCatalogSet { public: explicit UCSchemaSet(Catalog &catalog); diff --git a/src/include/storage/ic_table_entry.hpp b/src/include/storage/ic_table_entry.hpp index 5b25e82..7480ac5 100644 --- a/src/include/storage/ic_table_entry.hpp +++ b/src/include/storage/ic_table_entry.hpp @@ -30,7 +30,7 @@ class UCTableEntry : public TableCatalogEntry { UCTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTableInfo &info); UCTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, UCTableInfo &info); - unique_ptr table_data; + unique_ptr table_data; public: unique_ptr GetStatistics(ClientContext &context, column_t column_id) override; diff --git a/src/include/storage/ic_table_set.hpp b/src/include/storage/ic_table_set.hpp index c73b1bf..e20381d 100644 --- a/src/include/storage/ic_table_set.hpp +++ b/src/include/storage/ic_table_set.hpp @@ -7,16 +7,16 @@ namespace duckdb { struct CreateTableInfo; class UCResult; -class UCSchemaEntry; +class IBSchemaEntry; class UCTableSet : public UCInSchemaSet { public: - explicit UCTableSet(UCSchemaEntry &schema); + explicit UCTableSet(IBSchemaEntry &schema); public: optional_ptr CreateTable(ClientContext &context, BoundCreateTableInfo &info); - static unique_ptr GetTableInfo(ClientContext &context, UCSchemaEntry &schema, + static unique_ptr GetTableInfo(ClientContext &context, IBSchemaEntry &schema, const string &table_name); optional_ptr RefreshTable(ClientContext &context, const string &table_name); diff --git a/src/include/storage/ic_transaction.hpp b/src/include/storage/ic_transaction.hpp index 16d6da8..3043741 100644 --- a/src/include/storage/ic_transaction.hpp +++ b/src/include/storage/ic_transaction.hpp @@ -5,15 +5,15 @@ namespace duckdb { class UCCatalog; -class UCSchemaEntry; +class IBSchemaEntry; class UCTableEntry; enum class UCTransactionState { TRANSACTION_NOT_YET_STARTED, TRANSACTION_STARTED, TRANSACTION_FINISHED }; -class UCTransaction : public Transaction { +class IBTransaction : public Transaction { public: - UCTransaction(UCCatalog &ic_catalog, TransactionManager &manager, ClientContext &context); - ~UCTransaction() override; + IBTransaction(UCCatalog &ic_catalog, TransactionManager &manager, ClientContext &context); + ~IBTransaction() override; void Start(); void Commit(); @@ -21,7 +21,7 @@ class UCTransaction : public Transaction { // UCConnection &GetConnection(); // unique_ptr Query(const string &query); - static UCTransaction &Get(ClientContext &context, Catalog &catalog); + static IBTransaction &Get(ClientContext &context, Catalog &catalog); AccessMode GetAccessMode() const { return access_mode; } diff --git a/src/include/storage/ic_transaction_manager.hpp b/src/include/storage/ic_transaction_manager.hpp index 7f9fb80..255f65f 100644 --- a/src/include/storage/ic_transaction_manager.hpp +++ b/src/include/storage/ic_transaction_manager.hpp @@ -20,7 +20,7 @@ class UCTransactionManager : public TransactionManager { private: UCCatalog &ic_catalog; mutex transaction_lock; - reference_map_t> transactions; + reference_map_t> transactions; }; } // namespace duckdb diff --git a/src/storage/ic_catalog.cpp b/src/storage/ic_catalog.cpp index f01387c..efb4e3e 100644 --- a/src/storage/ic_catalog.cpp +++ b/src/storage/ic_catalog.cpp @@ -9,7 +9,7 @@ namespace duckdb { UCCatalog::UCCatalog(AttachedDatabase &db_p, const string &internal_name, AccessMode access_mode, - UCCredentials credentials) + IBCredentials credentials) : Catalog(db_p), internal_name(internal_name), access_mode(access_mode), credentials(std::move(credentials)), schemas(*this) { } @@ -36,7 +36,7 @@ void UCCatalog::DropSchema(ClientContext &context, DropInfo &info) { } void UCCatalog::ScanSchemas(ClientContext &context, std::function callback) { - schemas.Scan(context, [&](CatalogEntry &schema) { callback(schema.Cast()); }); + schemas.Scan(context, [&](CatalogEntry &schema) { callback(schema.Cast()); }); } optional_ptr UCCatalog::GetSchema(CatalogTransaction transaction, const string &schema_name, diff --git a/src/storage/ic_catalog_set.cpp b/src/storage/ic_catalog_set.cpp index 7f7a26c..e8db5c6 100644 --- a/src/storage/ic_catalog_set.cpp +++ b/src/storage/ic_catalog_set.cpp @@ -5,10 +5,10 @@ namespace duckdb { -UCCatalogSet::UCCatalogSet(Catalog &catalog) : catalog(catalog), is_loaded(false) { +IBCatalogSet::IBCatalogSet(Catalog &catalog) : catalog(catalog), is_loaded(false) { } -optional_ptr UCCatalogSet::GetEntry(ClientContext &context, const string &name) { +optional_ptr IBCatalogSet::GetEntry(ClientContext &context, const string &name) { if (!is_loaded) { is_loaded = true; LoadEntries(context); @@ -21,16 +21,16 @@ optional_ptr UCCatalogSet::GetEntry(ClientContext &context, const return entry->second.get(); } -void UCCatalogSet::DropEntry(ClientContext &context, DropInfo &info) { - throw NotImplementedException("UCCatalogSet::DropEntry"); +void IBCatalogSet::DropEntry(ClientContext &context, DropInfo &info) { + throw NotImplementedException("IBCatalogSet::DropEntry"); } -void UCCatalogSet::EraseEntryInternal(const string &name) { +void IBCatalogSet::EraseEntryInternal(const string &name) { lock_guard l(entry_lock); entries.erase(name); } -void UCCatalogSet::Scan(ClientContext &context, const std::function &callback) { +void IBCatalogSet::Scan(ClientContext &context, const std::function &callback) { if (!is_loaded) { is_loaded = true; LoadEntries(context); @@ -41,29 +41,29 @@ void UCCatalogSet::Scan(ClientContext &context, const std::function UCCatalogSet::CreateEntry(unique_ptr entry) { +optional_ptr IBCatalogSet::CreateEntry(unique_ptr entry) { lock_guard l(entry_lock); auto result = entry.get(); if (result->name.empty()) { - throw InternalException("UCCatalogSet::CreateEntry called with empty name"); + throw InternalException("IBCatalogSet::CreateEntry called with empty name"); } entries.insert(make_pair(result->name, std::move(entry))); return result; } -void UCCatalogSet::ClearEntries() { +void IBCatalogSet::ClearEntries() { entries.clear(); is_loaded = false; } -UCInSchemaSet::UCInSchemaSet(UCSchemaEntry &schema) : UCCatalogSet(schema.ParentCatalog()), schema(schema) { +UCInSchemaSet::UCInSchemaSet(IBSchemaEntry &schema) : IBCatalogSet(schema.ParentCatalog()), schema(schema) { } optional_ptr UCInSchemaSet::CreateEntry(unique_ptr entry) { if (!entry->internal) { entry->internal = schema.internal; } - return UCCatalogSet::CreateEntry(std::move(entry)); + return IBCatalogSet::CreateEntry(std::move(entry)); } } // namespace duckdb diff --git a/src/storage/ic_schema_entry.cpp b/src/storage/ic_schema_entry.cpp index 57cd7d2..5fdc4c6 100644 --- a/src/storage/ic_schema_entry.cpp +++ b/src/storage/ic_schema_entry.cpp @@ -13,21 +13,21 @@ namespace duckdb { -UCSchemaEntry::UCSchemaEntry(Catalog &catalog, CreateSchemaInfo &info) +IBSchemaEntry::IBSchemaEntry(Catalog &catalog, CreateSchemaInfo &info) : SchemaCatalogEntry(catalog, info), tables(*this) { } -UCSchemaEntry::~UCSchemaEntry() { +IBSchemaEntry::~IBSchemaEntry() { } -UCTransaction &GetUCTransaction(CatalogTransaction transaction) { +IBTransaction &GetUCTransaction(CatalogTransaction transaction) { if (!transaction.transaction) { throw InternalException("No transaction!?"); } - return transaction.transaction->Cast(); + return transaction.transaction->Cast(); } -optional_ptr UCSchemaEntry::CreateTable(CatalogTransaction transaction, BoundCreateTableInfo &info) { +optional_ptr IBSchemaEntry::CreateTable(CatalogTransaction transaction, BoundCreateTableInfo &info) { auto &base_info = info.Base(); auto table_name = base_info.table; if (base_info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) { @@ -36,7 +36,7 @@ optional_ptr UCSchemaEntry::CreateTable(CatalogTransaction transac return tables.CreateTable(transaction.GetContext(), info); } -optional_ptr UCSchemaEntry::CreateFunction(CatalogTransaction transaction, CreateFunctionInfo &info) { +optional_ptr IBSchemaEntry::CreateFunction(CatalogTransaction transaction, CreateFunctionInfo &info) { throw BinderException("PC databases do not support creating functions"); } @@ -50,7 +50,7 @@ void UCUnqualifyColumnRef(ParsedExpression &expr) { ParsedExpressionIterator::EnumerateChildren(expr, UCUnqualifyColumnRef); } -optional_ptr UCSchemaEntry::CreateIndex(CatalogTransaction transaction, CreateIndexInfo &info, +optional_ptr IBSchemaEntry::CreateIndex(CatalogTransaction transaction, CreateIndexInfo &info, TableCatalogEntry &table) { throw NotImplementedException("CreateIndex"); } @@ -59,7 +59,7 @@ string GetUCCreateView(CreateViewInfo &info) { throw NotImplementedException("GetCreateView"); } -optional_ptr UCSchemaEntry::CreateView(CatalogTransaction transaction, CreateViewInfo &info) { +optional_ptr IBSchemaEntry::CreateView(CatalogTransaction transaction, CreateViewInfo &info) { if (info.sql.empty()) { throw BinderException("Cannot create view in PC that originated from an " "empty SQL statement"); @@ -79,34 +79,34 @@ optional_ptr UCSchemaEntry::CreateView(CatalogTransaction transact return tables.RefreshTable(transaction.GetContext(), info.view_name); } -optional_ptr UCSchemaEntry::CreateType(CatalogTransaction transaction, CreateTypeInfo &info) { +optional_ptr IBSchemaEntry::CreateType(CatalogTransaction transaction, CreateTypeInfo &info) { throw BinderException("PC databases do not support creating types"); } -optional_ptr UCSchemaEntry::CreateSequence(CatalogTransaction transaction, CreateSequenceInfo &info) { +optional_ptr IBSchemaEntry::CreateSequence(CatalogTransaction transaction, CreateSequenceInfo &info) { throw BinderException("PC databases do not support creating sequences"); } -optional_ptr UCSchemaEntry::CreateTableFunction(CatalogTransaction transaction, +optional_ptr IBSchemaEntry::CreateTableFunction(CatalogTransaction transaction, CreateTableFunctionInfo &info) { throw BinderException("PC databases do not support creating table functions"); } -optional_ptr UCSchemaEntry::CreateCopyFunction(CatalogTransaction transaction, +optional_ptr IBSchemaEntry::CreateCopyFunction(CatalogTransaction transaction, CreateCopyFunctionInfo &info) { throw BinderException("PC databases do not support creating copy functions"); } -optional_ptr UCSchemaEntry::CreatePragmaFunction(CatalogTransaction transaction, +optional_ptr IBSchemaEntry::CreatePragmaFunction(CatalogTransaction transaction, CreatePragmaFunctionInfo &info) { throw BinderException("PC databases do not support creating pragma functions"); } -optional_ptr UCSchemaEntry::CreateCollation(CatalogTransaction transaction, CreateCollationInfo &info) { +optional_ptr IBSchemaEntry::CreateCollation(CatalogTransaction transaction, CreateCollationInfo &info) { throw BinderException("PC databases do not support creating collations"); } -void UCSchemaEntry::Alter(CatalogTransaction transaction, AlterInfo &info) { +void IBSchemaEntry::Alter(CatalogTransaction transaction, AlterInfo &info) { if (info.type != AlterType::ALTER_TABLE) { throw BinderException("Only altering tables is supported for now"); } @@ -125,22 +125,22 @@ bool CatalogTypeIsSupported(CatalogType type) { } } -void UCSchemaEntry::Scan(ClientContext &context, CatalogType type, +void IBSchemaEntry::Scan(ClientContext &context, CatalogType type, const std::function &callback) { if (!CatalogTypeIsSupported(type)) { return; } GetCatalogSet(type).Scan(context, callback); } -void UCSchemaEntry::Scan(CatalogType type, const std::function &callback) { +void IBSchemaEntry::Scan(CatalogType type, const std::function &callback) { throw NotImplementedException("Scan without context not supported"); } -void UCSchemaEntry::DropEntry(ClientContext &context, DropInfo &info) { +void IBSchemaEntry::DropEntry(ClientContext &context, DropInfo &info) { GetCatalogSet(info.type).DropEntry(context, info); } -optional_ptr UCSchemaEntry::GetEntry(CatalogTransaction transaction, CatalogType type, +optional_ptr IBSchemaEntry::GetEntry(CatalogTransaction transaction, CatalogType type, const string &name) { if (!CatalogTypeIsSupported(type)) { return nullptr; @@ -148,7 +148,7 @@ optional_ptr UCSchemaEntry::GetEntry(CatalogTransaction transactio return GetCatalogSet(type).GetEntry(transaction.GetContext(), name); } -UCCatalogSet &UCSchemaEntry::GetCatalogSet(CatalogType type) { +IBCatalogSet &IBSchemaEntry::GetCatalogSet(CatalogType type) { switch (type) { case CatalogType::TABLE_ENTRY: case CatalogType::VIEW_ENTRY: diff --git a/src/storage/ic_schema_set.cpp b/src/storage/ic_schema_set.cpp index 3f85a09..77db3e8 100644 --- a/src/storage/ic_schema_set.cpp +++ b/src/storage/ic_schema_set.cpp @@ -7,7 +7,7 @@ namespace duckdb { -UCSchemaSet::UCSchemaSet(Catalog &catalog) : UCCatalogSet(catalog) { +UCSchemaSet::UCSchemaSet(Catalog &catalog) : IBCatalogSet(catalog) { } static bool IsInternalTable(const string &catalog, const string &schema) { @@ -19,14 +19,14 @@ static bool IsInternalTable(const string &catalog, const string &schema) { void UCSchemaSet::LoadEntries(ClientContext &context) { auto &ic_catalog = catalog.Cast(); - auto tables = UCAPI::GetSchemas(catalog.GetName(), ic_catalog.internal_name, ic_catalog.credentials); + auto tables = IBAPI::GetSchemas(catalog.GetName(), ic_catalog.internal_name, ic_catalog.credentials); for (const auto &schema : tables) { CreateSchemaInfo info; info.schema = schema.schema_name; info.internal = IsInternalTable(schema.catalog_name, schema.schema_name); - auto schema_entry = make_uniq(catalog, info); - schema_entry->schema_data = make_uniq(schema); + auto schema_entry = make_uniq(catalog, info); + schema_entry->schema_data = make_uniq(schema); CreateEntry(std::move(schema_entry)); } } diff --git a/src/storage/ic_table_entry.cpp b/src/storage/ic_table_entry.cpp index 2928870..d85f3bc 100644 --- a/src/storage/ic_table_entry.cpp +++ b/src/storage/ic_table_entry.cpp @@ -74,8 +74,8 @@ TableFunction UCTableEntry::GetScanFunction(ClientContext &context, unique_ptrstorage_location.find("file://") != 0) { auto &secret_manager = SecretManager::Get(context); - // Get Credentials from UCAPI - auto table_credentials = UCAPI::GetTableCredentials( + // Get Credentials from IBAPI + auto table_credentials = IBAPI::GetTableCredentials( ic_catalog.internal_name, table_data->schema_name, table_data->name, ic_catalog.credentials); // Inject secret into secret manager scoped to this path diff --git a/src/storage/ic_table_set.cpp b/src/storage/ic_table_set.cpp index 6ea097d..62b834c 100644 --- a/src/storage/ic_table_set.cpp +++ b/src/storage/ic_table_set.cpp @@ -17,20 +17,20 @@ namespace duckdb { -UCTableSet::UCTableSet(UCSchemaEntry &schema) : UCInSchemaSet(schema) { +UCTableSet::UCTableSet(IBSchemaEntry &schema) : UCInSchemaSet(schema) { } -static ColumnDefinition CreateColumnDefinition(ClientContext &context, UCAPIColumnDefinition &coldef) { - return {coldef.name, UCUtils::TypeToLogicalType(context, coldef.type_text)}; +static ColumnDefinition CreateColumnDefinition(ClientContext &context, IBAPIColumnDefinition &coldef) { + return {coldef.name, IBUtils::TypeToLogicalType(context, coldef.type_text)}; } void UCTableSet::LoadEntries(ClientContext &context) { - auto &transaction = UCTransaction::Get(context, catalog); + auto &transaction = IBTransaction::Get(context, catalog); auto &ic_catalog = catalog.Cast(); // TODO: handle out-of-order columns using position property - auto tables = UCAPI::GetTables(catalog.GetName(), catalog.GetDBPath(), schema.name, ic_catalog.credentials); + auto tables = IBAPI::GetTables(catalog.GetName(), catalog.GetDBPath(), schema.name, ic_catalog.credentials); for (auto &table : tables) { D_ASSERT(schema.name == table.schema_name); @@ -41,7 +41,7 @@ void UCTableSet::LoadEntries(ClientContext &context) { info.table = table.name; auto table_entry = make_uniq(catalog, schema, info); - table_entry->table_data = make_uniq(table); + table_entry->table_data = make_uniq(table); CreateEntry(std::move(table_entry)); } @@ -55,7 +55,7 @@ optional_ptr UCTableSet::RefreshTable(ClientContext &context, cons return table_ptr; } -unique_ptr UCTableSet::GetTableInfo(ClientContext &context, UCSchemaEntry &schema, +unique_ptr UCTableSet::GetTableInfo(ClientContext &context, IBSchemaEntry &schema, const string &table_name) { throw NotImplementedException("UCTableSet::CreateTable"); } diff --git a/src/storage/ic_transaction.cpp b/src/storage/ic_transaction.cpp index fbff966..1a971fb 100644 --- a/src/storage/ic_transaction.cpp +++ b/src/storage/ic_transaction.cpp @@ -6,30 +6,30 @@ namespace duckdb { -UCTransaction::UCTransaction(UCCatalog &ic_catalog, TransactionManager &manager, ClientContext &context) +IBTransaction::IBTransaction(UCCatalog &ic_catalog, TransactionManager &manager, ClientContext &context) : Transaction(manager, context), access_mode(ic_catalog.access_mode) { // connection = UCConnection::Open(ic_catalog.path); } -UCTransaction::~UCTransaction() = default; +IBTransaction::~IBTransaction() = default; -void UCTransaction::Start() { +void IBTransaction::Start() { transaction_state = UCTransactionState::TRANSACTION_NOT_YET_STARTED; } -void UCTransaction::Commit() { +void IBTransaction::Commit() { if (transaction_state == UCTransactionState::TRANSACTION_STARTED) { transaction_state = UCTransactionState::TRANSACTION_FINISHED; // connection.Execute("COMMIT"); } } -void UCTransaction::Rollback() { +void IBTransaction::Rollback() { if (transaction_state == UCTransactionState::TRANSACTION_STARTED) { transaction_state = UCTransactionState::TRANSACTION_FINISHED; // connection.Execute("ROLLBACK"); } } -// UCConnection &UCTransaction::GetConnection() { +// UCConnection &IBTransaction::GetConnection() { // if (transaction_state == UCTransactionState::TRANSACTION_NOT_YET_STARTED) { // transaction_state = UCTransactionState::TRANSACTION_STARTED; // string query = "START TRANSACTION"; @@ -41,7 +41,7 @@ void UCTransaction::Rollback() { // return connection; //} -// unique_ptr UCTransaction::Query(const string &query) { +// unique_ptr IBTransaction::Query(const string &query) { // if (transaction_state == UCTransactionState::TRANSACTION_NOT_YET_STARTED) { // transaction_state = UCTransactionState::TRANSACTION_STARTED; // string transaction_start = "START TRANSACTION"; @@ -54,8 +54,8 @@ void UCTransaction::Rollback() { // return connection.Query(query); //} -UCTransaction &UCTransaction::Get(ClientContext &context, Catalog &catalog) { - return Transaction::Get(context, catalog).Cast(); +IBTransaction &IBTransaction::Get(ClientContext &context, Catalog &catalog) { + return Transaction::Get(context, catalog).Cast(); } } // namespace duckdb diff --git a/src/storage/ic_transaction_manager.cpp b/src/storage/ic_transaction_manager.cpp index bf0f319..5e9f15f 100644 --- a/src/storage/ic_transaction_manager.cpp +++ b/src/storage/ic_transaction_manager.cpp @@ -8,7 +8,7 @@ UCTransactionManager::UCTransactionManager(AttachedDatabase &db_p, UCCatalog &ic } Transaction &UCTransactionManager::StartTransaction(ClientContext &context) { - auto transaction = make_uniq(ic_catalog, *this, context); + auto transaction = make_uniq(ic_catalog, *this, context); transaction->Start(); auto &result = *transaction; lock_guard l(transaction_lock); @@ -17,7 +17,7 @@ Transaction &UCTransactionManager::StartTransaction(ClientContext &context) { } ErrorData UCTransactionManager::CommitTransaction(ClientContext &context, Transaction &transaction) { - auto &ic_transaction = transaction.Cast(); + auto &ic_transaction = transaction.Cast(); ic_transaction.Commit(); lock_guard l(transaction_lock); transactions.erase(transaction); @@ -25,14 +25,14 @@ ErrorData UCTransactionManager::CommitTransaction(ClientContext &context, Transa } void UCTransactionManager::RollbackTransaction(Transaction &transaction) { - auto &ic_transaction = transaction.Cast(); + auto &ic_transaction = transaction.Cast(); ic_transaction.Rollback(); lock_guard l(transaction_lock); transactions.erase(transaction); } void UCTransactionManager::Checkpoint(ClientContext &context, bool force) { - auto &transaction = UCTransaction::Get(context, db.GetCatalog()); + auto &transaction = IBTransaction::Get(context, db.GetCatalog()); // auto &db = transaction.GetConnection(); // db.Execute("CHECKPOINT"); } From 63ef6e00baa277171d424a2173dbfb31d70b2c1e Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Tue, 14 Jan 2025 23:07:36 -0500 Subject: [PATCH 04/13] FInished renaming --- src/iceberg_extension.cpp | 12 +++--- src/include/catalog_utils.hpp | 4 +- src/include/storage/ic_catalog.hpp | 12 +++--- src/include/storage/ic_catalog_set.hpp | 4 +- src/include/storage/ic_schema_entry.hpp | 2 +- src/include/storage/ic_schema_set.hpp | 4 +- src/include/storage/ic_table_entry.hpp | 14 +++---- src/include/storage/ic_table_set.hpp | 10 ++--- src/include/storage/ic_transaction.hpp | 13 +++--- .../storage/ic_transaction_manager.hpp | 6 +-- src/storage/ic_catalog.cpp | 42 +++++++++---------- src/storage/ic_catalog_set.cpp | 4 +- src/storage/ic_clear_cache.cpp | 6 +-- src/storage/ic_schema_entry.cpp | 4 +- src/storage/ic_schema_set.cpp | 8 ++-- src/storage/ic_table_entry.cpp | 14 +++---- src/storage/ic_table_set.cpp | 40 +++++++++--------- src/storage/ic_transaction.cpp | 26 ++++++------ src/storage/ic_transaction_manager.cpp | 10 ++--- 19 files changed, 116 insertions(+), 119 deletions(-) diff --git a/src/iceberg_extension.cpp b/src/iceberg_extension.cpp index 5b5d9c5..ca5971c 100644 --- a/src/iceberg_extension.cpp +++ b/src/iceberg_extension.cpp @@ -131,18 +131,18 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in // TODO: Check catalog with name actually exists! - return make_uniq(db, info.path, access_mode, credentials); + return make_uniq(db, info.path, access_mode, credentials); } static unique_ptr CreateTransactionManager(StorageExtensionInfo *storage_info, AttachedDatabase &db, Catalog &catalog) { - auto &ic_catalog = catalog.Cast(); - return make_uniq(db, ic_catalog); + auto &ic_catalog = catalog.Cast(); + return make_uniq(db, ic_catalog); } -class UCCatalogStorageExtension : public StorageExtension { +class IBCatalogStorageExtension : public StorageExtension { public: - UCCatalogStorageExtension() { + IBCatalogStorageExtension() { attach = IcebergCatalogAttach; create_transaction_manager = CreateTransactionManager; } @@ -180,7 +180,7 @@ static void LoadInternal(DatabaseInstance &instance) { SetCatalogSecretParameters(secret_function); ExtensionUtil::RegisterFunction(instance, secret_function); - config.storage_extensions["iceberg"] = make_uniq(); + config.storage_extensions["iceberg"] = make_uniq(); } void IcebergExtension::Load(DuckDB &db) { diff --git a/src/include/catalog_utils.hpp b/src/include/catalog_utils.hpp index f294299..ee11eb8 100644 --- a/src/include/catalog_utils.hpp +++ b/src/include/catalog_utils.hpp @@ -10,10 +10,10 @@ class IBTransaction; enum class IBTypeAnnotation { STANDARD, CAST_TO_VARCHAR, NUMERIC_AS_DOUBLE, CTID, JSONB, FIXED_LENGTH_CHAR }; -struct UCType { +struct IBType { idx_t oid = 0; IBTypeAnnotation info = IBTypeAnnotation::STANDARD; - vector children; + vector children; }; class IBUtils { diff --git a/src/include/storage/ic_catalog.hpp b/src/include/storage/ic_catalog.hpp index 901ee05..0b97be4 100644 --- a/src/include/storage/ic_catalog.hpp +++ b/src/include/storage/ic_catalog.hpp @@ -19,18 +19,18 @@ struct IBCredentials { string token; }; -class UCClearCacheFunction : public TableFunction { +class IBClearCacheFunction : public TableFunction { public: - UCClearCacheFunction(); + IBClearCacheFunction(); static void ClearCacheOnSetting(ClientContext &context, SetScope scope, Value ¶meter); }; -class UCCatalog : public Catalog { +class IBCatalog : public Catalog { public: - explicit UCCatalog(AttachedDatabase &db_p, const string &internal_name, AccessMode access_mode, + explicit IBCatalog(AttachedDatabase &db_p, const string &internal_name, AccessMode access_mode, IBCredentials credentials); - ~UCCatalog(); + ~IBCatalog(); string internal_name; AccessMode access_mode; @@ -73,7 +73,7 @@ class UCCatalog : public Catalog { void DropSchema(ClientContext &context, DropInfo &info) override; private: - UCSchemaSet schemas; + IBSchemaSet schemas; string default_schema; }; diff --git a/src/include/storage/ic_catalog_set.hpp b/src/include/storage/ic_catalog_set.hpp index d2a0099..b4cae9b 100644 --- a/src/include/storage/ic_catalog_set.hpp +++ b/src/include/storage/ic_catalog_set.hpp @@ -34,9 +34,9 @@ class IBCatalogSet { bool is_loaded; }; -class UCInSchemaSet : public IBCatalogSet { +class IBInSchemaSet : public IBCatalogSet { public: - UCInSchemaSet(IBSchemaEntry &schema); + IBInSchemaSet(IBSchemaEntry &schema); optional_ptr CreateEntry(unique_ptr entry) override; diff --git a/src/include/storage/ic_schema_entry.hpp b/src/include/storage/ic_schema_entry.hpp index 3579f59..c8d5e1c 100644 --- a/src/include/storage/ic_schema_entry.hpp +++ b/src/include/storage/ic_schema_entry.hpp @@ -40,7 +40,7 @@ class IBSchemaEntry : public SchemaCatalogEntry { IBCatalogSet &GetCatalogSet(CatalogType type); private: - UCTableSet tables; + IBTableSet tables; }; } // namespace duckdb diff --git a/src/include/storage/ic_schema_set.hpp b/src/include/storage/ic_schema_set.hpp index 365bf0a..de9974a 100644 --- a/src/include/storage/ic_schema_set.hpp +++ b/src/include/storage/ic_schema_set.hpp @@ -7,9 +7,9 @@ namespace duckdb { struct CreateSchemaInfo; -class UCSchemaSet : public IBCatalogSet { +class IBSchemaSet : public IBCatalogSet { public: - explicit UCSchemaSet(Catalog &catalog); + explicit IBSchemaSet(Catalog &catalog); public: optional_ptr CreateSchema(ClientContext &context, CreateSchemaInfo &info); diff --git a/src/include/storage/ic_table_entry.hpp b/src/include/storage/ic_table_entry.hpp index 7480ac5..f648237 100644 --- a/src/include/storage/ic_table_entry.hpp +++ b/src/include/storage/ic_table_entry.hpp @@ -7,14 +7,14 @@ namespace duckdb { -struct UCTableInfo { - UCTableInfo() { +struct IBTableInfo { + IBTableInfo() { create_info = make_uniq(); } - UCTableInfo(const string &schema, const string &table) { + IBTableInfo(const string &schema, const string &table) { create_info = make_uniq(string(), schema, table); } - UCTableInfo(const SchemaCatalogEntry &schema, const string &table) { + IBTableInfo(const SchemaCatalogEntry &schema, const string &table) { create_info = make_uniq((SchemaCatalogEntry &)schema, table); } @@ -25,10 +25,10 @@ struct UCTableInfo { unique_ptr create_info; }; -class UCTableEntry : public TableCatalogEntry { +class IBTableEntry : public TableCatalogEntry { public: - UCTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTableInfo &info); - UCTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, UCTableInfo &info); + IBTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTableInfo &info); + IBTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, IBTableInfo &info); unique_ptr table_data; diff --git a/src/include/storage/ic_table_set.hpp b/src/include/storage/ic_table_set.hpp index e20381d..5bc9940 100644 --- a/src/include/storage/ic_table_set.hpp +++ b/src/include/storage/ic_table_set.hpp @@ -6,17 +6,17 @@ namespace duckdb { struct CreateTableInfo; -class UCResult; +class IBResult; class IBSchemaEntry; -class UCTableSet : public UCInSchemaSet { +class IBTableSet : public IBInSchemaSet { public: - explicit UCTableSet(IBSchemaEntry &schema); + explicit IBTableSet(IBSchemaEntry &schema); public: optional_ptr CreateTable(ClientContext &context, BoundCreateTableInfo &info); - static unique_ptr GetTableInfo(ClientContext &context, IBSchemaEntry &schema, + static unique_ptr GetTableInfo(ClientContext &context, IBSchemaEntry &schema, const string &table_name); optional_ptr RefreshTable(ClientContext &context, const string &table_name); @@ -30,7 +30,7 @@ class UCTableSet : public UCInSchemaSet { void AlterTable(ClientContext &context, AddColumnInfo &info); void AlterTable(ClientContext &context, RemoveColumnInfo &info); - static void AddColumn(ClientContext &context, UCResult &result, UCTableInfo &table_info, idx_t column_offset = 0); + static void AddColumn(ClientContext &context, IBResult &result, IBTableInfo &table_info, idx_t column_offset = 0); }; } // namespace duckdb diff --git a/src/include/storage/ic_transaction.hpp b/src/include/storage/ic_transaction.hpp index 3043741..7dba882 100644 --- a/src/include/storage/ic_transaction.hpp +++ b/src/include/storage/ic_transaction.hpp @@ -4,31 +4,28 @@ #include "duckdb/transaction/transaction.hpp" namespace duckdb { -class UCCatalog; +class IBCatalog; class IBSchemaEntry; -class UCTableEntry; +class IBTableEntry; -enum class UCTransactionState { TRANSACTION_NOT_YET_STARTED, TRANSACTION_STARTED, TRANSACTION_FINISHED }; +enum class IBTransactionState { TRANSACTION_NOT_YET_STARTED, TRANSACTION_STARTED, TRANSACTION_FINISHED }; class IBTransaction : public Transaction { public: - IBTransaction(UCCatalog &ic_catalog, TransactionManager &manager, ClientContext &context); + IBTransaction(IBCatalog &ic_catalog, TransactionManager &manager, ClientContext &context); ~IBTransaction() override; void Start(); void Commit(); void Rollback(); - // UCConnection &GetConnection(); - // unique_ptr Query(const string &query); static IBTransaction &Get(ClientContext &context, Catalog &catalog); AccessMode GetAccessMode() const { return access_mode; } private: - // UCConnection connection; - UCTransactionState transaction_state; + IBTransactionState transaction_state; AccessMode access_mode; }; diff --git a/src/include/storage/ic_transaction_manager.hpp b/src/include/storage/ic_transaction_manager.hpp index 255f65f..4d02af7 100644 --- a/src/include/storage/ic_transaction_manager.hpp +++ b/src/include/storage/ic_transaction_manager.hpp @@ -7,9 +7,9 @@ namespace duckdb { -class UCTransactionManager : public TransactionManager { +class IBTransactionManager : public TransactionManager { public: - UCTransactionManager(AttachedDatabase &db_p, UCCatalog &ic_catalog); + IBTransactionManager(AttachedDatabase &db_p, IBCatalog &ic_catalog); Transaction &StartTransaction(ClientContext &context) override; ErrorData CommitTransaction(ClientContext &context, Transaction &transaction) override; @@ -18,7 +18,7 @@ class UCTransactionManager : public TransactionManager { void Checkpoint(ClientContext &context, bool force = false) override; private: - UCCatalog &ic_catalog; + IBCatalog &ic_catalog; mutex transaction_lock; reference_map_t> transactions; }; diff --git a/src/storage/ic_catalog.cpp b/src/storage/ic_catalog.cpp index efb4e3e..9b1a447 100644 --- a/src/storage/ic_catalog.cpp +++ b/src/storage/ic_catalog.cpp @@ -8,18 +8,18 @@ namespace duckdb { -UCCatalog::UCCatalog(AttachedDatabase &db_p, const string &internal_name, AccessMode access_mode, +IBCatalog::IBCatalog(AttachedDatabase &db_p, const string &internal_name, AccessMode access_mode, IBCredentials credentials) : Catalog(db_p), internal_name(internal_name), access_mode(access_mode), credentials(std::move(credentials)), schemas(*this) { } -UCCatalog::~UCCatalog() = default; +IBCatalog::~IBCatalog() = default; -void UCCatalog::Initialize(bool load_builtin) { +void IBCatalog::Initialize(bool load_builtin) { } -optional_ptr UCCatalog::CreateSchema(CatalogTransaction transaction, CreateSchemaInfo &info) { +optional_ptr IBCatalog::CreateSchema(CatalogTransaction transaction, CreateSchemaInfo &info) { if (info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) { DropInfo try_drop; try_drop.type = CatalogType::SCHEMA_ENTRY; @@ -31,15 +31,15 @@ optional_ptr UCCatalog::CreateSchema(CatalogTransaction transactio return schemas.CreateSchema(transaction.GetContext(), info); } -void UCCatalog::DropSchema(ClientContext &context, DropInfo &info) { +void IBCatalog::DropSchema(ClientContext &context, DropInfo &info) { return schemas.DropEntry(context, info); } -void UCCatalog::ScanSchemas(ClientContext &context, std::function callback) { +void IBCatalog::ScanSchemas(ClientContext &context, std::function callback) { schemas.Scan(context, [&](CatalogEntry &schema) { callback(schema.Cast()); }); } -optional_ptr UCCatalog::GetSchema(CatalogTransaction transaction, const string &schema_name, +optional_ptr IBCatalog::GetSchema(CatalogTransaction transaction, const string &schema_name, OnEntryNotFound if_not_found, QueryErrorContext error_context) { if (schema_name == DEFAULT_SCHEMA) { if (default_schema.empty()) { @@ -55,17 +55,17 @@ optional_ptr UCCatalog::GetSchema(CatalogTransaction transac return reinterpret_cast(entry.get()); } -bool UCCatalog::InMemory() { +bool IBCatalog::InMemory() { return false; } -string UCCatalog::GetDBPath() { +string IBCatalog::GetDBPath() { return internal_name; } -DatabaseSize UCCatalog::GetDatabaseSize(ClientContext &context) { +DatabaseSize IBCatalog::GetDatabaseSize(ClientContext &context) { if (default_schema.empty()) { throw InvalidInputException("Attempting to fetch the database size - but no database was provided " "in the connection string"); @@ -74,29 +74,29 @@ DatabaseSize UCCatalog::GetDatabaseSize(ClientContext &context) { return size; } -void UCCatalog::ClearCache() { +void IBCatalog::ClearCache() { schemas.ClearEntries(); } -unique_ptr UCCatalog::PlanInsert(ClientContext &context, LogicalInsert &op, +unique_ptr IBCatalog::PlanInsert(ClientContext &context, LogicalInsert &op, unique_ptr plan) { - throw NotImplementedException("UCCatalog PlanInsert"); + throw NotImplementedException("IBCatalog PlanInsert"); } -unique_ptr UCCatalog::PlanCreateTableAs(ClientContext &context, LogicalCreateTable &op, +unique_ptr IBCatalog::PlanCreateTableAs(ClientContext &context, LogicalCreateTable &op, unique_ptr plan) { - throw NotImplementedException("UCCatalog PlanCreateTableAs"); + throw NotImplementedException("IBCatalog PlanCreateTableAs"); } -unique_ptr UCCatalog::PlanDelete(ClientContext &context, LogicalDelete &op, +unique_ptr IBCatalog::PlanDelete(ClientContext &context, LogicalDelete &op, unique_ptr plan) { - throw NotImplementedException("UCCatalog PlanDelete"); + throw NotImplementedException("IBCatalog PlanDelete"); } -unique_ptr UCCatalog::PlanUpdate(ClientContext &context, LogicalUpdate &op, +unique_ptr IBCatalog::PlanUpdate(ClientContext &context, LogicalUpdate &op, unique_ptr plan) { - throw NotImplementedException("UCCatalog PlanUpdate"); + throw NotImplementedException("IBCatalog PlanUpdate"); } -unique_ptr UCCatalog::BindCreateIndex(Binder &binder, CreateStatement &stmt, TableCatalogEntry &table, +unique_ptr IBCatalog::BindCreateIndex(Binder &binder, CreateStatement &stmt, TableCatalogEntry &table, unique_ptr plan) { - throw NotImplementedException("UCCatalog BindCreateIndex"); + throw NotImplementedException("IBCatalog BindCreateIndex"); } } // namespace duckdb diff --git a/src/storage/ic_catalog_set.cpp b/src/storage/ic_catalog_set.cpp index e8db5c6..4c0e1e0 100644 --- a/src/storage/ic_catalog_set.cpp +++ b/src/storage/ic_catalog_set.cpp @@ -56,10 +56,10 @@ void IBCatalogSet::ClearEntries() { is_loaded = false; } -UCInSchemaSet::UCInSchemaSet(IBSchemaEntry &schema) : IBCatalogSet(schema.ParentCatalog()), schema(schema) { +IBInSchemaSet::IBInSchemaSet(IBSchemaEntry &schema) : IBCatalogSet(schema.ParentCatalog()), schema(schema) { } -optional_ptr UCInSchemaSet::CreateEntry(unique_ptr entry) { +optional_ptr IBInSchemaSet::CreateEntry(unique_ptr entry) { if (!entry->internal) { entry->internal = schema.internal; } diff --git a/src/storage/ic_clear_cache.cpp b/src/storage/ic_clear_cache.cpp index d4346b8..99db62a 100644 --- a/src/storage/ic_clear_cache.cpp +++ b/src/storage/ic_clear_cache.cpp @@ -28,7 +28,7 @@ static void ClearUCCaches(ClientContext &context) { if (catalog.GetCatalogType() != "iceberg") { continue; } - catalog.Cast().ClearCache(); + catalog.Cast().ClearCache(); } } @@ -41,10 +41,10 @@ static void ClearCacheFunction(ClientContext &context, TableFunctionInput &data_ data.finished = true; } -void UCClearCacheFunction::ClearCacheOnSetting(ClientContext &context, SetScope scope, Value ¶meter) { +void IBClearCacheFunction::ClearCacheOnSetting(ClientContext &context, SetScope scope, Value ¶meter) { ClearUCCaches(context); } -UCClearCacheFunction::UCClearCacheFunction() : TableFunction("pc_clear_cache", {}, ClearCacheFunction, ClearCacheBind) { +IBClearCacheFunction::IBClearCacheFunction() : TableFunction("pc_clear_cache", {}, ClearCacheFunction, ClearCacheBind) { } } // namespace duckdb diff --git a/src/storage/ic_schema_entry.cpp b/src/storage/ic_schema_entry.cpp index 5fdc4c6..41cecaf 100644 --- a/src/storage/ic_schema_entry.cpp +++ b/src/storage/ic_schema_entry.cpp @@ -40,14 +40,14 @@ optional_ptr IBSchemaEntry::CreateFunction(CatalogTransaction tran throw BinderException("PC databases do not support creating functions"); } -void UCUnqualifyColumnRef(ParsedExpression &expr) { +void IBUnqualifyColumnRef(ParsedExpression &expr) { if (expr.type == ExpressionType::COLUMN_REF) { auto &colref = expr.Cast(); auto name = std::move(colref.column_names.back()); colref.column_names = {std::move(name)}; return; } - ParsedExpressionIterator::EnumerateChildren(expr, UCUnqualifyColumnRef); + ParsedExpressionIterator::EnumerateChildren(expr, IBUnqualifyColumnRef); } optional_ptr IBSchemaEntry::CreateIndex(CatalogTransaction transaction, CreateIndexInfo &info, diff --git a/src/storage/ic_schema_set.cpp b/src/storage/ic_schema_set.cpp index 77db3e8..e117633 100644 --- a/src/storage/ic_schema_set.cpp +++ b/src/storage/ic_schema_set.cpp @@ -7,7 +7,7 @@ namespace duckdb { -UCSchemaSet::UCSchemaSet(Catalog &catalog) : IBCatalogSet(catalog) { +IBSchemaSet::IBSchemaSet(Catalog &catalog) : IBCatalogSet(catalog) { } static bool IsInternalTable(const string &catalog, const string &schema) { @@ -16,9 +16,9 @@ static bool IsInternalTable(const string &catalog, const string &schema) { } return false; } -void UCSchemaSet::LoadEntries(ClientContext &context) { +void IBSchemaSet::LoadEntries(ClientContext &context) { - auto &ic_catalog = catalog.Cast(); + auto &ic_catalog = catalog.Cast(); auto tables = IBAPI::GetSchemas(catalog.GetName(), ic_catalog.internal_name, ic_catalog.credentials); for (const auto &schema : tables) { @@ -31,7 +31,7 @@ void UCSchemaSet::LoadEntries(ClientContext &context) { } } -optional_ptr UCSchemaSet::CreateSchema(ClientContext &context, CreateSchemaInfo &info) { +optional_ptr IBSchemaSet::CreateSchema(ClientContext &context, CreateSchemaInfo &info) { throw NotImplementedException("Schema creation"); } diff --git a/src/storage/ic_table_entry.cpp b/src/storage/ic_table_entry.cpp index d85f3bc..e625ee8 100644 --- a/src/storage/ic_table_entry.cpp +++ b/src/storage/ic_table_entry.cpp @@ -19,21 +19,21 @@ namespace duckdb { -UCTableEntry::UCTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTableInfo &info) +IBTableEntry::IBTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTableInfo &info) : TableCatalogEntry(catalog, schema, info) { this->internal = false; } -UCTableEntry::UCTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, UCTableInfo &info) +IBTableEntry::IBTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, IBTableInfo &info) : TableCatalogEntry(catalog, schema, *info.create_info) { this->internal = false; } -unique_ptr UCTableEntry::GetStatistics(ClientContext &context, column_t column_id) { +unique_ptr IBTableEntry::GetStatistics(ClientContext &context, column_t column_id) { return nullptr; } -void UCTableEntry::BindUpdateConstraints(Binder &binder, LogicalGet &, LogicalProjection &, LogicalUpdate &, +void IBTableEntry::BindUpdateConstraints(Binder &binder, LogicalGet &, LogicalProjection &, LogicalUpdate &, ClientContext &) { throw NotImplementedException("BindUpdateConstraints"); } @@ -55,9 +55,9 @@ struct MyIcebergFunctionData : public FunctionData { } }; -TableFunction UCTableEntry::GetScanFunction(ClientContext &context, unique_ptr &bind_data) { +TableFunction IBTableEntry::GetScanFunction(ClientContext &context, unique_ptr &bind_data) { auto &db = DatabaseInstance::GetDatabase(context); - auto &ic_catalog = catalog.Cast(); + auto &ic_catalog = catalog.Cast(); auto &parquet_function_set = ExtensionUtil::GetTableFunction(db, "parquet_scan"); auto parquet_scan_function = parquet_function_set.functions.GetFunctionByArguments(context, {LogicalType::VARCHAR}); @@ -138,7 +138,7 @@ TableFunction UCTableEntry::GetScanFunction(ClientContext &context, unique_ptr(); + auto &ic_catalog = catalog.Cast(); // TODO: handle out-of-order columns using position property auto tables = IBAPI::GetTables(catalog.GetName(), catalog.GetDBPath(), schema.name, ic_catalog.credentials); @@ -40,48 +40,48 @@ void UCTableSet::LoadEntries(ClientContext &context) { } info.table = table.name; - auto table_entry = make_uniq(catalog, schema, info); + auto table_entry = make_uniq(catalog, schema, info); table_entry->table_data = make_uniq(table); CreateEntry(std::move(table_entry)); } } -optional_ptr UCTableSet::RefreshTable(ClientContext &context, const string &table_name) { +optional_ptr IBTableSet::RefreshTable(ClientContext &context, const string &table_name) { auto table_info = GetTableInfo(context, schema, table_name); - auto table_entry = make_uniq(catalog, schema, *table_info); + auto table_entry = make_uniq(catalog, schema, *table_info); auto table_ptr = table_entry.get(); CreateEntry(std::move(table_entry)); return table_ptr; } -unique_ptr UCTableSet::GetTableInfo(ClientContext &context, IBSchemaEntry &schema, +unique_ptr IBTableSet::GetTableInfo(ClientContext &context, IBSchemaEntry &schema, const string &table_name) { - throw NotImplementedException("UCTableSet::CreateTable"); + throw NotImplementedException("IBTableSet::CreateTable"); } -optional_ptr UCTableSet::CreateTable(ClientContext &context, BoundCreateTableInfo &info) { - throw NotImplementedException("UCTableSet::CreateTable"); +optional_ptr IBTableSet::CreateTable(ClientContext &context, BoundCreateTableInfo &info) { + throw NotImplementedException("IBTableSet::CreateTable"); } -void UCTableSet::AlterTable(ClientContext &context, RenameTableInfo &info) { - throw NotImplementedException("UCTableSet::AlterTable"); +void IBTableSet::AlterTable(ClientContext &context, RenameTableInfo &info) { + throw NotImplementedException("IBTableSet::AlterTable"); } -void UCTableSet::AlterTable(ClientContext &context, RenameColumnInfo &info) { - throw NotImplementedException("UCTableSet::AlterTable"); +void IBTableSet::AlterTable(ClientContext &context, RenameColumnInfo &info) { + throw NotImplementedException("IBTableSet::AlterTable"); } -void UCTableSet::AlterTable(ClientContext &context, AddColumnInfo &info) { - throw NotImplementedException("UCTableSet::AlterTable"); +void IBTableSet::AlterTable(ClientContext &context, AddColumnInfo &info) { + throw NotImplementedException("IBTableSet::AlterTable"); } -void UCTableSet::AlterTable(ClientContext &context, RemoveColumnInfo &info) { - throw NotImplementedException("UCTableSet::AlterTable"); +void IBTableSet::AlterTable(ClientContext &context, RemoveColumnInfo &info) { + throw NotImplementedException("IBTableSet::AlterTable"); } -void UCTableSet::AlterTable(ClientContext &context, AlterTableInfo &alter) { - throw NotImplementedException("UCTableSet::AlterTable"); +void IBTableSet::AlterTable(ClientContext &context, AlterTableInfo &alter) { + throw NotImplementedException("IBTableSet::AlterTable"); } } // namespace duckdb diff --git a/src/storage/ic_transaction.cpp b/src/storage/ic_transaction.cpp index 1a971fb..57c2d2b 100644 --- a/src/storage/ic_transaction.cpp +++ b/src/storage/ic_transaction.cpp @@ -6,32 +6,32 @@ namespace duckdb { -IBTransaction::IBTransaction(UCCatalog &ic_catalog, TransactionManager &manager, ClientContext &context) +IBTransaction::IBTransaction(IBCatalog &ic_catalog, TransactionManager &manager, ClientContext &context) : Transaction(manager, context), access_mode(ic_catalog.access_mode) { - // connection = UCConnection::Open(ic_catalog.path); + // connection = IBConnection::Open(ic_catalog.path); } IBTransaction::~IBTransaction() = default; void IBTransaction::Start() { - transaction_state = UCTransactionState::TRANSACTION_NOT_YET_STARTED; + transaction_state = IBTransactionState::TRANSACTION_NOT_YET_STARTED; } void IBTransaction::Commit() { - if (transaction_state == UCTransactionState::TRANSACTION_STARTED) { - transaction_state = UCTransactionState::TRANSACTION_FINISHED; + if (transaction_state == IBTransactionState::TRANSACTION_STARTED) { + transaction_state = IBTransactionState::TRANSACTION_FINISHED; // connection.Execute("COMMIT"); } } void IBTransaction::Rollback() { - if (transaction_state == UCTransactionState::TRANSACTION_STARTED) { - transaction_state = UCTransactionState::TRANSACTION_FINISHED; + if (transaction_state == IBTransactionState::TRANSACTION_STARTED) { + transaction_state = IBTransactionState::TRANSACTION_FINISHED; // connection.Execute("ROLLBACK"); } } -// UCConnection &IBTransaction::GetConnection() { -// if (transaction_state == UCTransactionState::TRANSACTION_NOT_YET_STARTED) { -// transaction_state = UCTransactionState::TRANSACTION_STARTED; +// IBConnection &IBTransaction::GetConnection() { +// if (transaction_state == IBTransactionState::TRANSACTION_NOT_YET_STARTED) { +// transaction_state = IBTransactionState::TRANSACTION_STARTED; // string query = "START TRANSACTION"; // if (access_mode == AccessMode::READ_ONLY) { // query += " READ ONLY"; @@ -41,9 +41,9 @@ void IBTransaction::Rollback() { // return connection; //} -// unique_ptr IBTransaction::Query(const string &query) { -// if (transaction_state == UCTransactionState::TRANSACTION_NOT_YET_STARTED) { -// transaction_state = UCTransactionState::TRANSACTION_STARTED; +// unique_ptr IBTransaction::Query(const string &query) { +// if (transaction_state == IBTransactionState::TRANSACTION_NOT_YET_STARTED) { +// transaction_state = IBTransactionState::TRANSACTION_STARTED; // string transaction_start = "START TRANSACTION"; // if (access_mode == AccessMode::READ_ONLY) { // transaction_start += " READ ONLY"; diff --git a/src/storage/ic_transaction_manager.cpp b/src/storage/ic_transaction_manager.cpp index 5e9f15f..6b2d3b9 100644 --- a/src/storage/ic_transaction_manager.cpp +++ b/src/storage/ic_transaction_manager.cpp @@ -3,11 +3,11 @@ namespace duckdb { -UCTransactionManager::UCTransactionManager(AttachedDatabase &db_p, UCCatalog &ic_catalog) +IBTransactionManager::IBTransactionManager(AttachedDatabase &db_p, IBCatalog &ic_catalog) : TransactionManager(db_p), ic_catalog(ic_catalog) { } -Transaction &UCTransactionManager::StartTransaction(ClientContext &context) { +Transaction &IBTransactionManager::StartTransaction(ClientContext &context) { auto transaction = make_uniq(ic_catalog, *this, context); transaction->Start(); auto &result = *transaction; @@ -16,7 +16,7 @@ Transaction &UCTransactionManager::StartTransaction(ClientContext &context) { return result; } -ErrorData UCTransactionManager::CommitTransaction(ClientContext &context, Transaction &transaction) { +ErrorData IBTransactionManager::CommitTransaction(ClientContext &context, Transaction &transaction) { auto &ic_transaction = transaction.Cast(); ic_transaction.Commit(); lock_guard l(transaction_lock); @@ -24,14 +24,14 @@ ErrorData UCTransactionManager::CommitTransaction(ClientContext &context, Transa return ErrorData(); } -void UCTransactionManager::RollbackTransaction(Transaction &transaction) { +void IBTransactionManager::RollbackTransaction(Transaction &transaction) { auto &ic_transaction = transaction.Cast(); ic_transaction.Rollback(); lock_guard l(transaction_lock); transactions.erase(transaction); } -void UCTransactionManager::Checkpoint(ClientContext &context, bool force) { +void IBTransactionManager::Checkpoint(ClientContext &context, bool force) { auto &transaction = IBTransaction::Get(context, db.GetCatalog()); // auto &db = transaction.GetConnection(); // db.Execute("CHECKPOINT"); From e6f671ad366c071162dee7bba008928a350e77a8 Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Fri, 17 Jan 2025 22:08:34 -0500 Subject: [PATCH 05/13] Delete duplicate --- src/catalog_utils.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/catalog_utils.cpp b/src/catalog_utils.cpp index e6f3e52..87777df 100644 --- a/src/catalog_utils.cpp +++ b/src/catalog_utils.cpp @@ -55,8 +55,6 @@ LogicalType IBUtils::TypeToLogicalType(ClientContext &context, const string &typ return LogicalType::BLOB; } else if (type_text == "date") { return LogicalType::DATE; - } else if (type_text == "timestamp") { - return LogicalType::TIMESTAMP; // TODO: Is this the right timestamp } else if (type_text.find("decimal(") == 0) { size_t spec_end = type_text.find(')'); if (spec_end != string::npos) { From 46a994699e847644e58ffb365ec4de3346581e38 Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Tue, 21 Jan 2025 12:19:13 -0500 Subject: [PATCH 06/13] Lazy load metadata for tables --- CMakeLists.txt | 2 +- src/catalog_api.cpp | 135 ++++++++++++++----------- src/include/catalog_api.hpp | 1 + src/include/storage/ic_catalog_set.hpp | 2 +- src/include/storage/ic_schema_set.hpp | 5 + src/include/storage/ic_table_set.hpp | 6 ++ src/storage/ic_catalog.cpp | 4 +- src/storage/ic_catalog_set.cpp | 18 ++-- src/storage/ic_schema_set.cpp | 29 +++++- src/storage/ic_table_set.cpp | 49 ++++++--- 10 files changed, 161 insertions(+), 90 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0f57eb2..cd7bd25 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,7 +4,7 @@ cmake_minimum_required(VERSION 2.8.12) set(TARGET_NAME iceberg) project(${TARGET_NAME}) -set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED True) set(EXTENSION_NAME ${TARGET_NAME}_extension) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index dd40f7b..d41660f 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -5,6 +5,7 @@ #include #include +#include namespace duckdb { @@ -60,6 +61,34 @@ static void InitializeCurlObject(CURL * curl, const string &token) { SetCurlCAFileInfo(curl); } +template +static TYPE TemplatedTryGetYYJson(duckdb_yyjson::yyjson_val *obj, const string &field, TYPE default_val, + bool fail_on_missing = true) { + auto val = yyjson_obj_get(obj, field.c_str()); + if (val && yyjson_get_type(val) == TYPE_NUM) { + return get_function(val); + } else if (!fail_on_missing) { + return default_val; + } + throw IOException("Invalid field found while parsing field: " + field); +} + +static uint64_t TryGetNumFromObject(duckdb_yyjson::yyjson_val *obj, const string &field, bool fail_on_missing = true, + uint64_t default_val = 0) { + return TemplatedTryGetYYJson(obj, field, default_val, + fail_on_missing); +} +static bool TryGetBoolFromObject(duckdb_yyjson::yyjson_val *obj, const string &field, bool fail_on_missing = false, + bool default_val = false) { + return TemplatedTryGetYYJson(obj, field, default_val, + fail_on_missing); +} +static string TryGetStrFromObject(duckdb_yyjson::yyjson_val *obj, const string &field, bool fail_on_missing = true, + const char *default_val = "") { + return TemplatedTryGetYYJson(obj, field, default_val, + fail_on_missing); +} + static string GetRequest(const string &url, const string &token = "", curl_slist *extra_headers = NULL) { CURL *curl; CURLcode res; @@ -145,35 +174,15 @@ static duckdb_yyjson::yyjson_val *GetTableMetadata(const string &internal, const credentials.token, extra_headers); duckdb_yyjson::yyjson_doc *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); - return yyjson_doc_get_root(doc); -} - -template -static TYPE TemplatedTryGetYYJson(duckdb_yyjson::yyjson_val *obj, const string &field, TYPE default_val, - bool fail_on_missing = true) { - auto val = yyjson_obj_get(obj, field.c_str()); - if (val && yyjson_get_type(val) == TYPE_NUM) { - return get_function(val); - } else if (!fail_on_missing) { - return default_val; + + auto *root = yyjson_doc_get_root(doc); + auto *error = yyjson_obj_get(root, "error"); + if (error != NULL) { + string err_msg = TryGetStrFromObject(error, "message"); + throw std::runtime_error(err_msg); } - throw IOException("Invalid field found while parsing field: " + field); -} -static uint64_t TryGetNumFromObject(duckdb_yyjson::yyjson_val *obj, const string &field, bool fail_on_missing = true, - uint64_t default_val = 0) { - return TemplatedTryGetYYJson(obj, field, default_val, - fail_on_missing); -} -static bool TryGetBoolFromObject(duckdb_yyjson::yyjson_val *obj, const string &field, bool fail_on_missing = false, - bool default_val = false) { - return TemplatedTryGetYYJson(obj, field, default_val, - fail_on_missing); -} -static string TryGetStrFromObject(duckdb_yyjson::yyjson_val *obj, const string &field, bool fail_on_missing = true, - const char *default_val = "") { - return TemplatedTryGetYYJson(obj, field, default_val, - fail_on_missing); + return yyjson_doc_get_root(doc); } void IBAPI::InitializeCurl() { @@ -228,30 +237,22 @@ string IBAPI::GetToken(string id, string secret, string endpoint) { return TryGetStrFromObject(root, "access_token"); } -vector IBAPI::GetTables(const string &catalog, const string &internal, const string &schema, IBCredentials credentials) { - vector result; - - auto api_result = GetRequest(credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables", credentials.token); - - // Read JSON and get root - auto *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); - auto *root = yyjson_doc_get_root(doc); - - // Get root["hits"], iterate over the array - auto *tables = yyjson_obj_get(root, "identifiers"); - size_t idx, max; - duckdb_yyjson::yyjson_val *table; - yyjson_arr_foreach(tables, idx, max, table) { - IBAPITable table_result; - table_result.catalog_name = catalog; - table_result.schema_name = schema; - table_result.name = TryGetStrFromObject(table, "name"); - - auto *metadata_root = GetTableMetadata(internal, schema, table_result.name, credentials); - table_result.data_source_format = "ICEBERG"; +IBAPITable IBAPI::GetTable( + const string &catalog, const string &internal, const string &schema, const string &table, std::optional credentials) { + + IBAPITable table_result; + table_result.catalog_name = catalog; + table_result.schema_name = schema; + table_result.name = table; + table_result.data_source_format = "ICEBERG"; + table_result.table_id = "uuid-" + schema + "-" + "table"; + std::replace(table_result.table_id.begin(), table_result.table_id.end(), '_', '-'); + + if (credentials) { + auto *metadata_root = GetTableMetadata(internal, schema, table_result.name, *credentials); table_result.storage_location = TryGetStrFromObject(metadata_root, "metadata-location"); auto *metadata = yyjson_obj_get(metadata_root, "metadata"); - table_result.table_id = TryGetStrFromObject(metadata, "table-uuid"); + //table_result.table_id = TryGetStrFromObject(metadata, "table-uuid"); uint64_t current_schema_id = TryGetNumFromObject(metadata, "current-schema-id"); auto *schemas = yyjson_obj_get(metadata, "schemas"); @@ -269,17 +270,39 @@ vector IBAPI::GetTables(const string &catalog, const string &interna auto column_definition = ParseColumnDefinition(col); table_result.columns.push_back(column_definition); } - result.push_back(table_result); - - // This could take a while so display found tables to indicate progress - // TODO: Is there a way to show progress some other way? - std::cout << table_result.schema_name << " | " << table_result.name << std::endl; } } if (!found) { throw InternalException("Current schema not found"); } + } else { + // Skip fetching metadata, we'll do it later when we access the table + IBAPIColumnDefinition col; + col.name = "__"; + col.type_text = "int"; + col.precision = -1; //TODO: TryGetNumFromObject(column_def, "type_precision"); + col.scale = -1; //TODO: TryGetNumFromObject(column_def, "type_scale"); + col.position = 0; + table_result.columns.push_back(col); + } + + return table_result; +} + +// TODO: handle out-of-order columns using position property +vector IBAPI::GetTables(const string &catalog, const string &internal, const string &schema, IBCredentials credentials) { + vector result; + + auto api_result = GetRequest(credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables", credentials.token); + auto *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); + auto *root = yyjson_doc_get_root(doc); + auto *tables = yyjson_obj_get(root, "identifiers"); + size_t idx, max; + duckdb_yyjson::yyjson_val *table; + yyjson_arr_foreach(tables, idx, max, table) { + auto table_result = GetTable(catalog, internal, schema, TryGetStrFromObject(table, "name"), std::nullopt); + result.push_back(table_result); } return result; @@ -291,23 +314,19 @@ vector IBAPI::GetSchemas(const string &catalog, const string &inter auto api_result = GetRequest(credentials.endpoint + "/v1/" + internal + "/namespaces", credentials.token); - // Read JSON and get root duckdb_yyjson::yyjson_doc *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); duckdb_yyjson::yyjson_val *root = yyjson_doc_get_root(doc); - auto *error = yyjson_obj_get(root, "error"); if (error != NULL) { string err_msg = TryGetStrFromObject(error, "message"); throw std::runtime_error(err_msg); } - // Get root["hits"], iterate over the array auto *schemas = yyjson_obj_get(root, "namespaces"); size_t idx, max; duckdb_yyjson::yyjson_val *schema; yyjson_arr_foreach(schemas, idx, max, schema) { IBAPISchema schema_result; - schema_result.catalog_name = catalog; duckdb_yyjson::yyjson_val *value = yyjson_arr_get(schema, 0); schema_result.schema_name = yyjson_get_str(value); diff --git a/src/include/catalog_api.hpp b/src/include/catalog_api.hpp index e8ef25b..89de9a2 100644 --- a/src/include/catalog_api.hpp +++ b/src/include/catalog_api.hpp @@ -47,6 +47,7 @@ class IBAPI { static IBAPITableCredentials GetTableCredentials(const string &internal, const string &schema, const string &table, IBCredentials credentials); static vector GetCatalogs(const string &catalog, IBCredentials credentials); static vector GetTables(const string &catalog, const string &internal, const string &schema, IBCredentials credentials); + static IBAPITable GetTable(const string &catalog, const string &internal, const string &schema, const string &table, std::optional credentials); static vector GetSchemas(const string &catalog, const string &internal, IBCredentials credentials); static vector GetTablesInSchema(const string &catalog, const string &schema, IBCredentials credentials); static string GetToken(string id, string secret, string endpoint); diff --git a/src/include/storage/ic_catalog_set.hpp b/src/include/storage/ic_catalog_set.hpp index b4cae9b..d31578f 100644 --- a/src/include/storage/ic_catalog_set.hpp +++ b/src/include/storage/ic_catalog_set.hpp @@ -22,6 +22,7 @@ class IBCatalogSet { protected: virtual void LoadEntries(ClientContext &context) = 0; + virtual void FillEntry(ClientContext &context, unique_ptr &entry) = 0; void EraseEntryInternal(const string &name); @@ -31,7 +32,6 @@ class IBCatalogSet { private: mutex entry_lock; case_insensitive_map_t> entries; - bool is_loaded; }; class IBInSchemaSet : public IBCatalogSet { diff --git a/src/include/storage/ic_schema_set.hpp b/src/include/storage/ic_schema_set.hpp index de9974a..80aca8e 100644 --- a/src/include/storage/ic_schema_set.hpp +++ b/src/include/storage/ic_schema_set.hpp @@ -13,9 +13,14 @@ class IBSchemaSet : public IBCatalogSet { public: optional_ptr CreateSchema(ClientContext &context, CreateSchemaInfo &info); + void DropSchema(ClientContext &context, DropInfo &info); protected: void LoadEntries(ClientContext &context) override; + void FillEntry(ClientContext &context, unique_ptr &entry) override; + +private: + bool is_loaded; }; } // namespace duckdb diff --git a/src/include/storage/ic_table_set.hpp b/src/include/storage/ic_table_set.hpp index 5bc9940..4207bdd 100644 --- a/src/include/storage/ic_table_set.hpp +++ b/src/include/storage/ic_table_set.hpp @@ -24,6 +24,7 @@ class IBTableSet : public IBInSchemaSet { protected: void LoadEntries(ClientContext &context) override; + void FillEntry(ClientContext &context, unique_ptr &entry) override; void AlterTable(ClientContext &context, RenameTableInfo &info); void AlterTable(ClientContext &context, RenameColumnInfo &info); @@ -31,6 +32,11 @@ class IBTableSet : public IBInSchemaSet { void AlterTable(ClientContext &context, RemoveColumnInfo &info); static void AddColumn(ClientContext &context, IBResult &result, IBTableInfo &table_info, idx_t column_offset = 0); + +private: + unique_ptr _CreateCatalogEntry(ClientContext &context, IBAPITable table); + + bool is_loaded; }; } // namespace duckdb diff --git a/src/storage/ic_catalog.cpp b/src/storage/ic_catalog.cpp index 9b1a447..be7276a 100644 --- a/src/storage/ic_catalog.cpp +++ b/src/storage/ic_catalog.cpp @@ -26,13 +26,13 @@ optional_ptr IBCatalog::CreateSchema(CatalogTransaction transactio try_drop.name = info.schema; try_drop.if_not_found = OnEntryNotFound::RETURN_NULL; try_drop.cascade = false; - schemas.DropEntry(transaction.GetContext(), try_drop); + schemas.DropSchema(transaction.GetContext(), try_drop); } return schemas.CreateSchema(transaction.GetContext(), info); } void IBCatalog::DropSchema(ClientContext &context, DropInfo &info) { - return schemas.DropEntry(context, info); + return schemas.DropSchema(context, info); } void IBCatalog::ScanSchemas(ClientContext &context, std::function callback) { diff --git a/src/storage/ic_catalog_set.cpp b/src/storage/ic_catalog_set.cpp index 4c0e1e0..903f357 100644 --- a/src/storage/ic_catalog_set.cpp +++ b/src/storage/ic_catalog_set.cpp @@ -5,24 +5,22 @@ namespace duckdb { -IBCatalogSet::IBCatalogSet(Catalog &catalog) : catalog(catalog), is_loaded(false) { +IBCatalogSet::IBCatalogSet(Catalog &catalog) : catalog(catalog) { } optional_ptr IBCatalogSet::GetEntry(ClientContext &context, const string &name) { - if (!is_loaded) { - is_loaded = true; - LoadEntries(context); - } + LoadEntries(context); lock_guard l(entry_lock); auto entry = entries.find(name); if (entry == entries.end()) { return nullptr; } + FillEntry(context, entry->second); return entry->second.get(); } void IBCatalogSet::DropEntry(ClientContext &context, DropInfo &info) { - throw NotImplementedException("IBCatalogSet::DropEntry"); + EraseEntryInternal(info.name); } void IBCatalogSet::EraseEntryInternal(const string &name) { @@ -31,10 +29,8 @@ void IBCatalogSet::EraseEntryInternal(const string &name) { } void IBCatalogSet::Scan(ClientContext &context, const std::function &callback) { - if (!is_loaded) { - is_loaded = true; - LoadEntries(context); - } + LoadEntries(context); + lock_guard l(entry_lock); for (auto &entry : entries) { callback(*entry.second); @@ -53,7 +49,7 @@ optional_ptr IBCatalogSet::CreateEntry(unique_ptr en void IBCatalogSet::ClearEntries() { entries.clear(); - is_loaded = false; + // TODO: is_loaded = false; } IBInSchemaSet::IBInSchemaSet(IBSchemaEntry &schema) : IBCatalogSet(schema.ParentCatalog()), schema(schema) { diff --git a/src/storage/ic_schema_set.cpp b/src/storage/ic_schema_set.cpp index e117633..948774b 100644 --- a/src/storage/ic_schema_set.cpp +++ b/src/storage/ic_schema_set.cpp @@ -5,9 +5,11 @@ #include "duckdb/parser/parsed_data/create_schema_info.hpp" #include "duckdb/catalog/catalog.hpp" +#include + namespace duckdb { -IBSchemaSet::IBSchemaSet(Catalog &catalog) : IBCatalogSet(catalog) { +IBSchemaSet::IBSchemaSet(Catalog &catalog) : IBCatalogSet(catalog), is_loaded(false) { } static bool IsInternalTable(const string &catalog, const string &schema) { @@ -16,12 +18,17 @@ static bool IsInternalTable(const string &catalog, const string &schema) { } return false; } + void IBSchemaSet::LoadEntries(ClientContext &context) { + if (is_loaded) { + return; + } + is_loaded = true; auto &ic_catalog = catalog.Cast(); - auto tables = IBAPI::GetSchemas(catalog.GetName(), ic_catalog.internal_name, ic_catalog.credentials); + auto schemas = IBAPI::GetSchemas(catalog.GetName(), ic_catalog.internal_name, ic_catalog.credentials); - for (const auto &schema : tables) { + for (const auto &schema : schemas) { CreateSchemaInfo info; info.schema = schema.schema_name; info.internal = IsInternalTable(schema.catalog_name, schema.schema_name); @@ -31,8 +38,22 @@ void IBSchemaSet::LoadEntries(ClientContext &context) { } } +void IBSchemaSet::FillEntry(ClientContext &context, unique_ptr &entry) { + // Nothing to do +} + optional_ptr IBSchemaSet::CreateSchema(ClientContext &context, CreateSchemaInfo &info) { - throw NotImplementedException("Schema creation"); + // TODO: throw NotImplementedException("Schema creation"); + std::cout << " >> Create schema" << std::endl; + // CreateEntry(...) } +void IBSchemaSet::DropSchema(ClientContext &context, DropInfo &info) { + // TODO + std::cout << " >> Drop schema" << std::endl; + + DropEntry(context, info); +} + + } // namespace duckdb diff --git a/src/storage/ic_table_set.cpp b/src/storage/ic_table_set.cpp index ec88263..67d1cb2 100644 --- a/src/storage/ic_table_set.cpp +++ b/src/storage/ic_table_set.cpp @@ -15,35 +15,58 @@ #include "storage/ic_schema_entry.hpp" #include "duckdb/parser/parser.hpp" +#include + namespace duckdb { -IBTableSet::IBTableSet(IBSchemaEntry &schema) : IBInSchemaSet(schema) { +IBTableSet::IBTableSet(IBSchemaEntry &schema) : IBInSchemaSet(schema), is_loaded(false) { } static ColumnDefinition CreateColumnDefinition(ClientContext &context, IBAPIColumnDefinition &coldef) { return {coldef.name, IBUtils::TypeToLogicalType(context, coldef.type_text)}; } +unique_ptr IBTableSet::_CreateCatalogEntry(ClientContext &context, IBAPITable table) { + D_ASSERT(schema.name == table.schema_name); + CreateTableInfo info; + info.table = table.name; + + for (auto &col : table.columns) { + info.columns.AddColumn(CreateColumnDefinition(context, col)); + } + + auto table_entry = make_uniq(catalog, schema, info); + table_entry->table_data = make_uniq(table); + return table_entry; +} + +void IBTableSet::FillEntry(ClientContext &context, unique_ptr &entry) { + auto* derived = static_cast(entry.get()); + if (!derived->table_data->storage_location.empty()) { + std::cout << "HERE: " << derived->table_data->storage_location << std::endl; + return; + } + + auto &ic_catalog = catalog.Cast(); + auto table = IBAPI::GetTable(catalog.GetName(), catalog.GetDBPath(), schema.name, entry->name, ic_catalog.credentials); + entry = _CreateCatalogEntry(context, table); +} + void IBTableSet::LoadEntries(ClientContext &context) { - auto &transaction = IBTransaction::Get(context, catalog); + if (is_loaded) { + return; + } + is_loaded = true; + auto &transaction = IBTransaction::Get(context, catalog); auto &ic_catalog = catalog.Cast(); // TODO: handle out-of-order columns using position property auto tables = IBAPI::GetTables(catalog.GetName(), catalog.GetDBPath(), schema.name, ic_catalog.credentials); for (auto &table : tables) { - D_ASSERT(schema.name == table.schema_name); - CreateTableInfo info; - for (auto &col : table.columns) { - info.columns.AddColumn(CreateColumnDefinition(context, col)); - } - - info.table = table.name; - auto table_entry = make_uniq(catalog, schema, info); - table_entry->table_data = make_uniq(table); - - CreateEntry(std::move(table_entry)); + auto entry = _CreateCatalogEntry(context, table); + CreateEntry(std::move(entry)); } } From 26d9f1bf4e5acc4df45634c3819ec35df0ae91c0 Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Tue, 21 Jan 2025 12:27:28 -0500 Subject: [PATCH 07/13] Extract precision and scale --- src/catalog_api.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index d41660f..bcdf7dc 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -195,13 +195,11 @@ vector IBAPI::GetCatalogs(const string &catalog, IBCredentials credentia static IBAPIColumnDefinition ParseColumnDefinition(duckdb_yyjson::yyjson_val *column_def) { IBAPIColumnDefinition result; - result.name = TryGetStrFromObject(column_def, "name"); result.type_text = TryGetStrFromObject(column_def, "type"); - result.precision = -1; //TODO: TryGetNumFromObject(column_def, "type_precision"); - result.scale = -1; //TODO: TryGetNumFromObject(column_def, "type_scale"); + result.precision = (result.type_text == "decimal") ? TryGetNumFromObject(column_def, "type_precision") : -1; + result.scale = (result.type_text == "decimal") ? TryGetNumFromObject(column_def, "type_scale") : -1; result.position = TryGetNumFromObject(column_def, "id") - 1; - return result; } @@ -281,8 +279,8 @@ IBAPITable IBAPI::GetTable( IBAPIColumnDefinition col; col.name = "__"; col.type_text = "int"; - col.precision = -1; //TODO: TryGetNumFromObject(column_def, "type_precision"); - col.scale = -1; //TODO: TryGetNumFromObject(column_def, "type_scale"); + col.precision = -1; + col.scale = -1; col.position = 0; table_result.columns.push_back(col); } From 645e83c6488424b58852aa72232434e59e63cf13 Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Tue, 21 Jan 2025 13:28:52 -0500 Subject: [PATCH 08/13] Get rid of is_loaded --- src/include/storage/ic_catalog_set.hpp | 2 +- src/include/storage/ic_schema_set.hpp | 3 --- src/include/storage/ic_table_set.hpp | 2 -- src/storage/ic_catalog_set.cpp | 1 - src/storage/ic_schema_set.cpp | 6 ++---- src/storage/ic_table_set.cpp | 8 ++------ 6 files changed, 5 insertions(+), 17 deletions(-) diff --git a/src/include/storage/ic_catalog_set.hpp b/src/include/storage/ic_catalog_set.hpp index d31578f..f486842 100644 --- a/src/include/storage/ic_catalog_set.hpp +++ b/src/include/storage/ic_catalog_set.hpp @@ -28,10 +28,10 @@ class IBCatalogSet { protected: Catalog &catalog; + case_insensitive_map_t> entries; private: mutex entry_lock; - case_insensitive_map_t> entries; }; class IBInSchemaSet : public IBCatalogSet { diff --git a/src/include/storage/ic_schema_set.hpp b/src/include/storage/ic_schema_set.hpp index 80aca8e..6e8332f 100644 --- a/src/include/storage/ic_schema_set.hpp +++ b/src/include/storage/ic_schema_set.hpp @@ -18,9 +18,6 @@ class IBSchemaSet : public IBCatalogSet { protected: void LoadEntries(ClientContext &context) override; void FillEntry(ClientContext &context, unique_ptr &entry) override; - -private: - bool is_loaded; }; } // namespace duckdb diff --git a/src/include/storage/ic_table_set.hpp b/src/include/storage/ic_table_set.hpp index 4207bdd..5803086 100644 --- a/src/include/storage/ic_table_set.hpp +++ b/src/include/storage/ic_table_set.hpp @@ -35,8 +35,6 @@ class IBTableSet : public IBInSchemaSet { private: unique_ptr _CreateCatalogEntry(ClientContext &context, IBAPITable table); - - bool is_loaded; }; } // namespace duckdb diff --git a/src/storage/ic_catalog_set.cpp b/src/storage/ic_catalog_set.cpp index 903f357..cc00d54 100644 --- a/src/storage/ic_catalog_set.cpp +++ b/src/storage/ic_catalog_set.cpp @@ -49,7 +49,6 @@ optional_ptr IBCatalogSet::CreateEntry(unique_ptr en void IBCatalogSet::ClearEntries() { entries.clear(); - // TODO: is_loaded = false; } IBInSchemaSet::IBInSchemaSet(IBSchemaEntry &schema) : IBCatalogSet(schema.ParentCatalog()), schema(schema) { diff --git a/src/storage/ic_schema_set.cpp b/src/storage/ic_schema_set.cpp index 948774b..2b3fd1f 100644 --- a/src/storage/ic_schema_set.cpp +++ b/src/storage/ic_schema_set.cpp @@ -9,7 +9,7 @@ namespace duckdb { -IBSchemaSet::IBSchemaSet(Catalog &catalog) : IBCatalogSet(catalog), is_loaded(false) { +IBSchemaSet::IBSchemaSet(Catalog &catalog) : IBCatalogSet(catalog) { } static bool IsInternalTable(const string &catalog, const string &schema) { @@ -20,14 +20,12 @@ static bool IsInternalTable(const string &catalog, const string &schema) { } void IBSchemaSet::LoadEntries(ClientContext &context) { - if (is_loaded) { + if (!entries.empty()) { return; } - is_loaded = true; auto &ic_catalog = catalog.Cast(); auto schemas = IBAPI::GetSchemas(catalog.GetName(), ic_catalog.internal_name, ic_catalog.credentials); - for (const auto &schema : schemas) { CreateSchemaInfo info; info.schema = schema.schema_name; diff --git a/src/storage/ic_table_set.cpp b/src/storage/ic_table_set.cpp index 67d1cb2..5c211cb 100644 --- a/src/storage/ic_table_set.cpp +++ b/src/storage/ic_table_set.cpp @@ -15,11 +15,9 @@ #include "storage/ic_schema_entry.hpp" #include "duckdb/parser/parser.hpp" -#include - namespace duckdb { -IBTableSet::IBTableSet(IBSchemaEntry &schema) : IBInSchemaSet(schema), is_loaded(false) { +IBTableSet::IBTableSet(IBSchemaEntry &schema) : IBInSchemaSet(schema) { } static ColumnDefinition CreateColumnDefinition(ClientContext &context, IBAPIColumnDefinition &coldef) { @@ -43,7 +41,6 @@ unique_ptr IBTableSet::_CreateCatalogEntry(ClientContext &context, void IBTableSet::FillEntry(ClientContext &context, unique_ptr &entry) { auto* derived = static_cast(entry.get()); if (!derived->table_data->storage_location.empty()) { - std::cout << "HERE: " << derived->table_data->storage_location << std::endl; return; } @@ -53,11 +50,10 @@ void IBTableSet::FillEntry(ClientContext &context, unique_ptr &ent } void IBTableSet::LoadEntries(ClientContext &context) { - if (is_loaded) { + if (!entries.empty()) { return; } - is_loaded = true; auto &transaction = IBTransaction::Get(context, catalog); auto &ic_catalog = catalog.Cast(); From 3a63af6fc73706d0ffeb58f07eb8e178c867790e Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Wed, 22 Jan 2025 02:59:23 -0500 Subject: [PATCH 09/13] Finish implementing create/drop schema --- src/catalog_api.cpp | 127 +++++++++++++++++++++------------- src/include/catalog_api.hpp | 2 + src/storage/ic_schema_set.cpp | 23 +++--- 3 files changed, 91 insertions(+), 61 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index bcdf7dc..d0790af 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -12,7 +12,7 @@ namespace duckdb { //! We use a global here to store the path that is selected on the IBAPI::InitializeCurl call static string SELECTED_CURL_CERT_PATH = ""; -static size_t GetRequestWriteCallback(void *contents, size_t size, size_t nmemb, void *userp) { +static size_t RequestWriteCallback(void *contents, size_t size, size_t nmemb, void *userp) { ((std::string *)userp)->append((char *)contents, size * nmemb); return size * nmemb; } @@ -89,6 +89,36 @@ static string TryGetStrFromObject(duckdb_yyjson::yyjson_val *obj, const string & fail_on_missing); } +static string DeleteRequest(const string &url, const string &token = "", curl_slist *extra_headers = NULL) { + CURL *curl; + CURLcode res; + string readBuffer; + + curl = curl_easy_init(); + if (curl) { + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, RequestWriteCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer); + + if(extra_headers) { + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, extra_headers); + } + + InitializeCurlObject(curl, token); + res = curl_easy_perform(curl); + curl_easy_cleanup(curl); + + if (res != CURLcode::CURLE_OK) { + string error = curl_easy_strerror(res); + throw IOException("Curl DELETE Request to '%s' failed with error: '%s'", url, error); + } + + return readBuffer; + } + throw InternalException("Failed to initialize curl"); +} + static string GetRequest(const string &url, const string &token = "", curl_slist *extra_headers = NULL) { CURL *curl; CURLcode res; @@ -97,11 +127,13 @@ static string GetRequest(const string &url, const string &token = "", curl_slist curl = curl_easy_init(); if (curl) { curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, GetRequestWriteCallback); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, RequestWriteCallback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer); + if(extra_headers) { curl_easy_setopt(curl, CURLOPT_HTTPHEADER, extra_headers); } + InitializeCurlObject(curl, token); res = curl_easy_perform(curl); curl_easy_cleanup(curl); @@ -110,34 +142,34 @@ static string GetRequest(const string &url, const string &token = "", curl_slist string error = curl_easy_strerror(res); throw IOException("Curl Request to '%s' failed with error: '%s'", url, error); } + return readBuffer; } throw InternalException("Failed to initialize curl"); } -static string PostRequest(const string &url, const string &post_data, curl_slist *extra_headers = NULL) { +static string PostRequest( + const string &url, + const string &post_data, + const string &content_type = "x-www-form-urlencoded", + const string &token = "", + curl_slist *extra_headers = NULL) { string readBuffer; CURL *curl = curl_easy_init(); if (!curl) { throw InternalException("Failed to initialize curl"); } - // Set the URL curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - - // Set POST method curl_easy_setopt(curl, CURLOPT_POST, 1L); - - // Set POST data curl_easy_setopt(curl, CURLOPT_POSTFIELDS, post_data.c_str()); - - // Set up response handling - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, GetRequestWriteCallback); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, RequestWriteCallback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer); // Create default headers for content type struct curl_slist *headers = NULL; - headers = curl_slist_append(headers, "Content-Type: application/x-www-form-urlencoded"); + const string content_type_str = "Content-Type: application/" + content_type; + headers = curl_slist_append(headers, content_type_str.c_str()); // Append any extra headers if (extra_headers) { @@ -149,7 +181,7 @@ static string PostRequest(const string &url, const string &post_data, curl_slist } curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - InitializeCurlObject(curl, ""); + InitializeCurlObject(curl, token); // Perform the request CURLcode res = curl_easy_perform(curl); @@ -163,26 +195,27 @@ static string PostRequest(const string &url, const string &post_data, curl_slist throw IOException("Curl Request to '%s' failed with error: '%s'", url, error); } return readBuffer; - } -static duckdb_yyjson::yyjson_val *GetTableMetadata(const string &internal, const string &schema, const string &table, IBCredentials credentials) { - struct curl_slist *extra_headers = NULL; - extra_headers = curl_slist_append(extra_headers, "X-Iceberg-Access-Delegation: vended-credentials"); - auto api_result = GetRequest( - credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables/" + table, - credentials.token, - extra_headers); - duckdb_yyjson::yyjson_doc *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); - +static duckdb_yyjson::yyjson_val *api_result_to_doc(const string &api_result) { + auto *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); auto *root = yyjson_doc_get_root(doc); auto *error = yyjson_obj_get(root, "error"); if (error != NULL) { string err_msg = TryGetStrFromObject(error, "message"); throw std::runtime_error(err_msg); } + return root; +} - return yyjson_doc_get_root(doc); +static duckdb_yyjson::yyjson_val *GetTableMetadata(const string &internal, const string &schema, const string &table, IBCredentials credentials) { + struct curl_slist *extra_headers = NULL; + extra_headers = curl_slist_append(extra_headers, "X-Iceberg-Access-Delegation: vended-credentials"); + auto api_result = GetRequest( + credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables/" + table, + credentials.token, + extra_headers); + return api_result_to_doc(api_result); } void IBAPI::InitializeCurl() { @@ -205,7 +238,6 @@ static IBAPIColumnDefinition ParseColumnDefinition(duckdb_yyjson::yyjson_val *co IBAPITableCredentials IBAPI::GetTableCredentials(const string &internal, const string &schema, const string &table, IBCredentials credentials) { IBAPITableCredentials result; - duckdb_yyjson::yyjson_val *root = GetTableMetadata(internal, schema, table, credentials); auto *aws_temp_credentials = yyjson_obj_get(root, "config"); @@ -221,17 +253,7 @@ IBAPITableCredentials IBAPI::GetTableCredentials(const string &internal, const s string IBAPI::GetToken(string id, string secret, string endpoint) { string post_data = "grant_type=client_credentials&client_id=" + id + "&client_secret=" + secret + "&scope=PRINCIPAL_ROLE:ALL"; string api_result = PostRequest(endpoint + "/v1/oauth/tokens", post_data); - - // Read JSON and get root - auto *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); - auto *root = yyjson_doc_get_root(doc); - - auto *error = yyjson_obj_get(root, "error"); - if (error != NULL) { - string err_msg = TryGetStrFromObject(error, "message"); - throw std::runtime_error(err_msg); - } - + auto *root = api_result_to_doc(api_result); return TryGetStrFromObject(root, "access_token"); } @@ -291,10 +313,8 @@ IBAPITable IBAPI::GetTable( // TODO: handle out-of-order columns using position property vector IBAPI::GetTables(const string &catalog, const string &internal, const string &schema, IBCredentials credentials) { vector result; - auto api_result = GetRequest(credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables", credentials.token); - auto *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); - auto *root = yyjson_doc_get_root(doc); + auto *root = api_result_to_doc(api_result); auto *tables = yyjson_obj_get(root, "identifiers"); size_t idx, max; duckdb_yyjson::yyjson_val *table; @@ -308,18 +328,9 @@ vector IBAPI::GetTables(const string &catalog, const string &interna vector IBAPI::GetSchemas(const string &catalog, const string &internal, IBCredentials credentials) { vector result; - auto api_result = GetRequest(credentials.endpoint + "/v1/" + internal + "/namespaces", credentials.token); - - duckdb_yyjson::yyjson_doc *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); - duckdb_yyjson::yyjson_val *root = yyjson_doc_get_root(doc); - auto *error = yyjson_obj_get(root, "error"); - if (error != NULL) { - string err_msg = TryGetStrFromObject(error, "message"); - throw std::runtime_error(err_msg); - } - + auto *root = api_result_to_doc(api_result); auto *schemas = yyjson_obj_get(root, "namespaces"); size_t idx, max; duckdb_yyjson::yyjson_val *schema; @@ -334,4 +345,22 @@ vector IBAPI::GetSchemas(const string &catalog, const string &inter return result; } +IBAPISchema IBAPI::CreateSchema(const string &catalog, const string &internal, const string &schema, IBCredentials credentials) { + string post_data = "{\"namespace\":[\"" + schema + "\"]}"; + string api_result = PostRequest( + credentials.endpoint + "/v1/" + internal + "/namespaces", post_data, "json", credentials.token); + api_result_to_doc(api_result); // if the method returns, request was successful + + IBAPISchema schema_result; + schema_result.catalog_name = catalog; + schema_result.schema_name = schema; //yyjson_get_str(value); + return schema_result; +} + +void IBAPI::DropSchema(const string &internal, const string &schema, IBCredentials credentials) { + string api_result = DeleteRequest( + credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema, credentials.token); + api_result_to_doc(api_result); // if the method returns, request was successful +} + } // namespace duckdb diff --git a/src/include/catalog_api.hpp b/src/include/catalog_api.hpp index 89de9a2..65cea98 100644 --- a/src/include/catalog_api.hpp +++ b/src/include/catalog_api.hpp @@ -51,5 +51,7 @@ class IBAPI { static vector GetSchemas(const string &catalog, const string &internal, IBCredentials credentials); static vector GetTablesInSchema(const string &catalog, const string &schema, IBCredentials credentials); static string GetToken(string id, string secret, string endpoint); + static IBAPISchema CreateSchema(const string &catalog, const string &internal, const string &schema, IBCredentials credentials); + static void DropSchema(const string &internal, const string &schema, IBCredentials credentials); }; } // namespace duckdb diff --git a/src/storage/ic_schema_set.cpp b/src/storage/ic_schema_set.cpp index 2b3fd1f..fc7b1f1 100644 --- a/src/storage/ic_schema_set.cpp +++ b/src/storage/ic_schema_set.cpp @@ -1,11 +1,10 @@ -#include "storage/ic_schema_set.hpp" -#include "storage/ic_catalog.hpp" #include "catalog_api.hpp" -#include "storage/ic_transaction.hpp" #include "duckdb/parser/parsed_data/create_schema_info.hpp" +#include "duckdb/parser/parsed_data/drop_info.hpp" #include "duckdb/catalog/catalog.hpp" - -#include +#include "storage/ic_catalog.hpp" +#include "storage/ic_schema_set.hpp" +#include "storage/ic_transaction.hpp" namespace duckdb { @@ -41,17 +40,17 @@ void IBSchemaSet::FillEntry(ClientContext &context, unique_ptr &en } optional_ptr IBSchemaSet::CreateSchema(ClientContext &context, CreateSchemaInfo &info) { - // TODO: throw NotImplementedException("Schema creation"); - std::cout << " >> Create schema" << std::endl; - // CreateEntry(...) + auto &ic_catalog = catalog.Cast(); + auto schema = IBAPI::CreateSchema(catalog.GetName(), ic_catalog.internal_name, info.schema, ic_catalog.credentials); + auto schema_entry = make_uniq(catalog, info); + schema_entry->schema_data = make_uniq(schema); + return CreateEntry(std::move(schema_entry)); } void IBSchemaSet::DropSchema(ClientContext &context, DropInfo &info) { - // TODO - std::cout << " >> Drop schema" << std::endl; - + auto &ic_catalog = catalog.Cast(); + IBAPI::DropSchema(ic_catalog.internal_name, info.name, ic_catalog.credentials); DropEntry(context, info); } - } // namespace duckdb From f24fc3a3e55cfa477ddaea1a57d2d7e0a7beb858 Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Thu, 23 Jan 2025 00:53:01 -0500 Subject: [PATCH 10/13] Reorg classes for clarity --- src/include/storage/ic_catalog.hpp | 1 + src/include/storage/ic_catalog_set.hpp | 10 ---------- src/include/storage/ic_table_set.hpp | 18 ++++++++++++++---- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/include/storage/ic_catalog.hpp b/src/include/storage/ic_catalog.hpp index 0b97be4..4957618 100644 --- a/src/include/storage/ic_catalog.hpp +++ b/src/include/storage/ic_catalog.hpp @@ -26,6 +26,7 @@ class IBClearCacheFunction : public TableFunction { static void ClearCacheOnSetting(ClientContext &context, SetScope scope, Value ¶meter); }; + class IBCatalog : public Catalog { public: explicit IBCatalog(AttachedDatabase &db_p, const string &internal_name, AccessMode access_mode, diff --git a/src/include/storage/ic_catalog_set.hpp b/src/include/storage/ic_catalog_set.hpp index f486842..9f9befb 100644 --- a/src/include/storage/ic_catalog_set.hpp +++ b/src/include/storage/ic_catalog_set.hpp @@ -34,14 +34,4 @@ class IBCatalogSet { mutex entry_lock; }; -class IBInSchemaSet : public IBCatalogSet { -public: - IBInSchemaSet(IBSchemaEntry &schema); - - optional_ptr CreateEntry(unique_ptr entry) override; - -protected: - IBSchemaEntry &schema; -}; - } // namespace duckdb diff --git a/src/include/storage/ic_table_set.hpp b/src/include/storage/ic_table_set.hpp index 5803086..25b7c8e 100644 --- a/src/include/storage/ic_table_set.hpp +++ b/src/include/storage/ic_table_set.hpp @@ -9,17 +9,26 @@ struct CreateTableInfo; class IBResult; class IBSchemaEntry; + +class IBInSchemaSet : public IBCatalogSet { +public: + IBInSchemaSet(IBSchemaEntry &schema); + + optional_ptr CreateEntry(unique_ptr entry) override; + +protected: + IBSchemaEntry &schema; +}; + + class IBTableSet : public IBInSchemaSet { public: explicit IBTableSet(IBSchemaEntry &schema); public: optional_ptr CreateTable(ClientContext &context, BoundCreateTableInfo &info); - - static unique_ptr GetTableInfo(ClientContext &context, IBSchemaEntry &schema, - const string &table_name); + static unique_ptr GetTableInfo(ClientContext &context, IBSchemaEntry &schema, const string &table_name); optional_ptr RefreshTable(ClientContext &context, const string &table_name); - void AlterTable(ClientContext &context, AlterTableInfo &info); protected: @@ -37,4 +46,5 @@ class IBTableSet : public IBInSchemaSet { unique_ptr _CreateCatalogEntry(ClientContext &context, IBAPITable table); }; + } // namespace duckdb From 56e4c764db42333596bc59341b56fb8a53f0a45c Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Thu, 23 Jan 2025 00:54:13 -0500 Subject: [PATCH 11/13] Minor fixes --- src/storage/ic_catalog.cpp | 2 -- src/storage/ic_schema_entry.cpp | 2 +- src/storage/ic_table_set.cpp | 2 -- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/storage/ic_catalog.cpp b/src/storage/ic_catalog.cpp index be7276a..4238651 100644 --- a/src/storage/ic_catalog.cpp +++ b/src/storage/ic_catalog.cpp @@ -63,8 +63,6 @@ string IBCatalog::GetDBPath() { return internal_name; } - - DatabaseSize IBCatalog::GetDatabaseSize(ClientContext &context) { if (default_schema.empty()) { throw InvalidInputException("Attempting to fetch the database size - but no database was provided " diff --git a/src/storage/ic_schema_entry.cpp b/src/storage/ic_schema_entry.cpp index 41cecaf..b137d1c 100644 --- a/src/storage/ic_schema_entry.cpp +++ b/src/storage/ic_schema_entry.cpp @@ -61,7 +61,7 @@ string GetUCCreateView(CreateViewInfo &info) { optional_ptr IBSchemaEntry::CreateView(CatalogTransaction transaction, CreateViewInfo &info) { if (info.sql.empty()) { - throw BinderException("Cannot create view in PC that originated from an " + throw BinderException("Cannot create view that originated from an " "empty SQL statement"); } if (info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT || diff --git a/src/storage/ic_table_set.cpp b/src/storage/ic_table_set.cpp index 5c211cb..bc65c5a 100644 --- a/src/storage/ic_table_set.cpp +++ b/src/storage/ic_table_set.cpp @@ -54,9 +54,7 @@ void IBTableSet::LoadEntries(ClientContext &context) { return; } - auto &transaction = IBTransaction::Get(context, catalog); auto &ic_catalog = catalog.Cast(); - // TODO: handle out-of-order columns using position property auto tables = IBAPI::GetTables(catalog.GetName(), catalog.GetDBPath(), schema.name, ic_catalog.credentials); From aee6c1d42645de6ec5cc501b710d9c8fb5ad47dd Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Thu, 23 Jan 2025 01:06:32 -0500 Subject: [PATCH 12/13] Rename classes for clarity --- src/catalog_api.cpp | 44 +++++++------- src/catalog_utils.cpp | 12 ++-- src/iceberg_extension.cpp | 18 +++--- src/include/catalog_api.hpp | 30 +++++----- src/include/catalog_utils.hpp | 14 ++--- src/include/storage/ic_catalog.hpp | 20 +++---- src/include/storage/ic_catalog_set.hpp | 8 +-- src/include/storage/ic_schema_entry.hpp | 14 ++--- src/include/storage/ic_schema_set.hpp | 4 +- src/include/storage/ic_table_entry.hpp | 16 ++--- src/include/storage/ic_table_set.hpp | 20 +++---- src/include/storage/ic_transaction.hpp | 18 +++--- .../storage/ic_transaction_manager.hpp | 8 +-- src/storage/ic_catalog.cpp | 46 +++++++-------- src/storage/ic_catalog_set.cpp | 24 ++++---- src/storage/ic_clear_cache.cpp | 6 +- src/storage/ic_schema_entry.cpp | 44 +++++++------- src/storage/ic_schema_set.cpp | 30 +++++----- src/storage/ic_table_entry.cpp | 18 +++--- src/storage/ic_table_set.cpp | 58 +++++++++---------- src/storage/ic_transaction.cpp | 38 ++++++------ src/storage/ic_transaction_manager.cpp | 18 +++--- 22 files changed, 255 insertions(+), 253 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index d0790af..d38d9a4 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -9,7 +9,7 @@ namespace duckdb { -//! We use a global here to store the path that is selected on the IBAPI::InitializeCurl call +//! We use a global here to store the path that is selected on the ICAPI::InitializeCurl call static string SELECTED_CURL_CERT_PATH = ""; static size_t RequestWriteCallback(void *contents, size_t size, size_t nmemb, void *userp) { @@ -208,7 +208,7 @@ static duckdb_yyjson::yyjson_val *api_result_to_doc(const string &api_result) { return root; } -static duckdb_yyjson::yyjson_val *GetTableMetadata(const string &internal, const string &schema, const string &table, IBCredentials credentials) { +static duckdb_yyjson::yyjson_val *GetTableMetadata(const string &internal, const string &schema, const string &table, ICCredentials credentials) { struct curl_slist *extra_headers = NULL; extra_headers = curl_slist_append(extra_headers, "X-Iceberg-Access-Delegation: vended-credentials"); auto api_result = GetRequest( @@ -218,16 +218,16 @@ static duckdb_yyjson::yyjson_val *GetTableMetadata(const string &internal, const return api_result_to_doc(api_result); } -void IBAPI::InitializeCurl() { +void ICAPI::InitializeCurl() { SelectCurlCertPath(); } -vector IBAPI::GetCatalogs(const string &catalog, IBCredentials credentials) { - throw NotImplementedException("IBAPI::GetCatalogs"); +vector ICAPI::GetCatalogs(const string &catalog, ICCredentials credentials) { + throw NotImplementedException("ICAPI::GetCatalogs"); } -static IBAPIColumnDefinition ParseColumnDefinition(duckdb_yyjson::yyjson_val *column_def) { - IBAPIColumnDefinition result; +static ICAPIColumnDefinition ParseColumnDefinition(duckdb_yyjson::yyjson_val *column_def) { + ICAPIColumnDefinition result; result.name = TryGetStrFromObject(column_def, "name"); result.type_text = TryGetStrFromObject(column_def, "type"); result.precision = (result.type_text == "decimal") ? TryGetNumFromObject(column_def, "type_precision") : -1; @@ -236,8 +236,8 @@ static IBAPIColumnDefinition ParseColumnDefinition(duckdb_yyjson::yyjson_val *co return result; } -IBAPITableCredentials IBAPI::GetTableCredentials(const string &internal, const string &schema, const string &table, IBCredentials credentials) { - IBAPITableCredentials result; +ICAPITableCredentials ICAPI::GetTableCredentials(const string &internal, const string &schema, const string &table, ICCredentials credentials) { + ICAPITableCredentials result; duckdb_yyjson::yyjson_val *root = GetTableMetadata(internal, schema, table, credentials); auto *aws_temp_credentials = yyjson_obj_get(root, "config"); @@ -250,17 +250,17 @@ IBAPITableCredentials IBAPI::GetTableCredentials(const string &internal, const s return result; } -string IBAPI::GetToken(string id, string secret, string endpoint) { +string ICAPI::GetToken(string id, string secret, string endpoint) { string post_data = "grant_type=client_credentials&client_id=" + id + "&client_secret=" + secret + "&scope=PRINCIPAL_ROLE:ALL"; string api_result = PostRequest(endpoint + "/v1/oauth/tokens", post_data); auto *root = api_result_to_doc(api_result); return TryGetStrFromObject(root, "access_token"); } -IBAPITable IBAPI::GetTable( - const string &catalog, const string &internal, const string &schema, const string &table, std::optional credentials) { +ICAPITable ICAPI::GetTable( + const string &catalog, const string &internal, const string &schema, const string &table, std::optional credentials) { - IBAPITable table_result; + ICAPITable table_result; table_result.catalog_name = catalog; table_result.schema_name = schema; table_result.name = table; @@ -298,7 +298,7 @@ IBAPITable IBAPI::GetTable( } } else { // Skip fetching metadata, we'll do it later when we access the table - IBAPIColumnDefinition col; + ICAPIColumnDefinition col; col.name = "__"; col.type_text = "int"; col.precision = -1; @@ -311,8 +311,8 @@ IBAPITable IBAPI::GetTable( } // TODO: handle out-of-order columns using position property -vector IBAPI::GetTables(const string &catalog, const string &internal, const string &schema, IBCredentials credentials) { - vector result; +vector ICAPI::GetTables(const string &catalog, const string &internal, const string &schema, ICCredentials credentials) { + vector result; auto api_result = GetRequest(credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables", credentials.token); auto *root = api_result_to_doc(api_result); auto *tables = yyjson_obj_get(root, "identifiers"); @@ -326,8 +326,8 @@ vector IBAPI::GetTables(const string &catalog, const string &interna return result; } -vector IBAPI::GetSchemas(const string &catalog, const string &internal, IBCredentials credentials) { - vector result; +vector ICAPI::GetSchemas(const string &catalog, const string &internal, ICCredentials credentials) { + vector result; auto api_result = GetRequest(credentials.endpoint + "/v1/" + internal + "/namespaces", credentials.token); auto *root = api_result_to_doc(api_result); @@ -335,7 +335,7 @@ vector IBAPI::GetSchemas(const string &catalog, const string &inter size_t idx, max; duckdb_yyjson::yyjson_val *schema; yyjson_arr_foreach(schemas, idx, max, schema) { - IBAPISchema schema_result; + ICAPISchema schema_result; schema_result.catalog_name = catalog; duckdb_yyjson::yyjson_val *value = yyjson_arr_get(schema, 0); schema_result.schema_name = yyjson_get_str(value); @@ -345,19 +345,19 @@ vector IBAPI::GetSchemas(const string &catalog, const string &inter return result; } -IBAPISchema IBAPI::CreateSchema(const string &catalog, const string &internal, const string &schema, IBCredentials credentials) { +ICAPISchema ICAPI::CreateSchema(const string &catalog, const string &internal, const string &schema, ICCredentials credentials) { string post_data = "{\"namespace\":[\"" + schema + "\"]}"; string api_result = PostRequest( credentials.endpoint + "/v1/" + internal + "/namespaces", post_data, "json", credentials.token); api_result_to_doc(api_result); // if the method returns, request was successful - IBAPISchema schema_result; + ICAPISchema schema_result; schema_result.catalog_name = catalog; schema_result.schema_name = schema; //yyjson_get_str(value); return schema_result; } -void IBAPI::DropSchema(const string &internal, const string &schema, IBCredentials credentials) { +void ICAPI::DropSchema(const string &internal, const string &schema, ICCredentials credentials) { string api_result = DeleteRequest( credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema, credentials.token); api_result_to_doc(api_result); // if the method returns, request was successful diff --git a/src/catalog_utils.cpp b/src/catalog_utils.cpp index 87777df..f71b41e 100644 --- a/src/catalog_utils.cpp +++ b/src/catalog_utils.cpp @@ -7,7 +7,7 @@ namespace duckdb { -string IBUtils::TypeToString(const LogicalType &input) { +string ICUtils::TypeToString(const LogicalType &input) { switch (input.id()) { case LogicalType::VARCHAR: return "TEXT"; @@ -28,7 +28,7 @@ string IBUtils::TypeToString(const LogicalType &input) { } } -LogicalType IBUtils::TypeToLogicalType(ClientContext &context, const string &type_text) { +LogicalType ICUtils::TypeToLogicalType(ClientContext &context, const string &type_text) { if (type_text == "tinyint") { return LogicalType::TINYINT; } else if (type_text == "smallint") { @@ -69,7 +69,7 @@ LogicalType IBUtils::TypeToLogicalType(ClientContext &context, const string &typ size_t type_end = type_text.rfind('>'); // find last, to deal with nested if (type_end != string::npos) { auto child_type_str = type_text.substr(6, type_end - 6); - auto child_type = IBUtils::TypeToLogicalType(context, child_type_str); + auto child_type = ICUtils::TypeToLogicalType(context, child_type_str); return LogicalType::LIST(child_type); } } else if (type_text.find("map<") == 0) { @@ -94,7 +94,7 @@ LogicalType IBUtils::TypeToLogicalType(ClientContext &context, const string &typ } } auto child_str = type_text.substr(cur, next_sep - cur); - auto child_type = IBUtils::TypeToLogicalType(context, child_str); + auto child_type = ICUtils::TypeToLogicalType(context, child_str); key_val.push_back(child_type); if (next_sep == type_end) { break; @@ -132,7 +132,7 @@ LogicalType IBUtils::TypeToLogicalType(ClientContext &context, const string &typ throw NotImplementedException("Invalid struct child type specifier: %s", child_str); } auto child_name = child_str.substr(0, type_sep); - auto child_type = IBUtils::TypeToLogicalType(context, child_str.substr(type_sep + 1, string::npos)); + auto child_type = ICUtils::TypeToLogicalType(context, child_str.substr(type_sep + 1, string::npos)); children.push_back({child_name, child_type}); if (next_sep == type_end) { break; @@ -148,7 +148,7 @@ LogicalType IBUtils::TypeToLogicalType(ClientContext &context, const string &typ return LogicalType::VARCHAR; } -LogicalType IBUtils::ToUCType(const LogicalType &input) { +LogicalType ICUtils::ToUCType(const LogicalType &input) { // todo do we need this mapping? throw NotImplementedException("ToUCType not yet implemented"); switch (input.id()) { diff --git a/src/iceberg_extension.cpp b/src/iceberg_extension.cpp index ca5971c..114b3ff 100644 --- a/src/iceberg_extension.cpp +++ b/src/iceberg_extension.cpp @@ -42,7 +42,7 @@ static unique_ptr CreateCatalogSecretFunction(ClientContext &, Creat } // Get token from catalog - result->secret_map["token"] = IBAPI::GetToken( + result->secret_map["token"] = ICAPI::GetToken( result->secret_map["client_id"].ToString(), result->secret_map["client_secret"].ToString(), result->secret_map["endpoint"].ToString()); @@ -80,7 +80,7 @@ unique_ptr GetSecret(ClientContext &context, const string &secret_n static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_info, ClientContext &context, AttachedDatabase &db, const string &name, AttachInfo &info, AccessMode access_mode) { - IBCredentials credentials; + ICCredentials credentials; // check if we have a secret provided string secret_name; @@ -131,18 +131,18 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in // TODO: Check catalog with name actually exists! - return make_uniq(db, info.path, access_mode, credentials); + return make_uniq(db, info.path, access_mode, credentials); } static unique_ptr CreateTransactionManager(StorageExtensionInfo *storage_info, AttachedDatabase &db, Catalog &catalog) { - auto &ic_catalog = catalog.Cast(); - return make_uniq(db, ic_catalog); + auto &ic_catalog = catalog.Cast(); + return make_uniq(db, ic_catalog); } -class IBCatalogStorageExtension : public StorageExtension { +class ICCatalogStorageExtension : public StorageExtension { public: - IBCatalogStorageExtension() { + ICCatalogStorageExtension() { attach = IcebergCatalogAttach; create_transaction_manager = CreateTransactionManager; } @@ -168,7 +168,7 @@ static void LoadInternal(DatabaseInstance &instance) { ExtensionUtil::RegisterFunction(instance, fun); } - IBAPI::InitializeCurl(); + ICAPI::InitializeCurl(); SecretType secret_type; secret_type.name = "iceberg"; @@ -180,7 +180,7 @@ static void LoadInternal(DatabaseInstance &instance) { SetCatalogSecretParameters(secret_function); ExtensionUtil::RegisterFunction(instance, secret_function); - config.storage_extensions["iceberg"] = make_uniq(); + config.storage_extensions["iceberg"] = make_uniq(); } void IcebergExtension::Load(DuckDB &db) { diff --git a/src/include/catalog_api.hpp b/src/include/catalog_api.hpp index 65cea98..b2d0683 100644 --- a/src/include/catalog_api.hpp +++ b/src/include/catalog_api.hpp @@ -4,9 +4,9 @@ #include "duckdb/common/types.hpp" namespace duckdb { -struct IBCredentials; +struct ICCredentials; -struct IBAPIColumnDefinition { +struct ICAPIColumnDefinition { string name; string type_text; idx_t precision; @@ -14,7 +14,7 @@ struct IBAPIColumnDefinition { idx_t position; }; -struct IBAPITable { +struct ICAPITable { string table_id; string name; @@ -25,33 +25,33 @@ struct IBAPITable { string storage_location; - vector columns; + vector columns; }; -struct IBAPISchema { +struct ICAPISchema { string schema_name; string catalog_name; }; -struct IBAPITableCredentials { +struct ICAPITableCredentials { string key_id; string secret; string session_token; }; -class IBAPI { +class ICAPI { public: //! WARNING: not thread-safe. To be called once on extension initialization static void InitializeCurl(); - static IBAPITableCredentials GetTableCredentials(const string &internal, const string &schema, const string &table, IBCredentials credentials); - static vector GetCatalogs(const string &catalog, IBCredentials credentials); - static vector GetTables(const string &catalog, const string &internal, const string &schema, IBCredentials credentials); - static IBAPITable GetTable(const string &catalog, const string &internal, const string &schema, const string &table, std::optional credentials); - static vector GetSchemas(const string &catalog, const string &internal, IBCredentials credentials); - static vector GetTablesInSchema(const string &catalog, const string &schema, IBCredentials credentials); + static ICAPITableCredentials GetTableCredentials(const string &internal, const string &schema, const string &table, ICCredentials credentials); + static vector GetCatalogs(const string &catalog, ICCredentials credentials); + static vector GetTables(const string &catalog, const string &internal, const string &schema, ICCredentials credentials); + static ICAPITable GetTable(const string &catalog, const string &internal, const string &schema, const string &table, std::optional credentials); + static vector GetSchemas(const string &catalog, const string &internal, ICCredentials credentials); + static vector GetTablesInSchema(const string &catalog, const string &schema, ICCredentials credentials); static string GetToken(string id, string secret, string endpoint); - static IBAPISchema CreateSchema(const string &catalog, const string &internal, const string &schema, IBCredentials credentials); - static void DropSchema(const string &internal, const string &schema, IBCredentials credentials); + static ICAPISchema CreateSchema(const string &catalog, const string &internal, const string &schema, ICCredentials credentials); + static void DropSchema(const string &internal, const string &schema, ICCredentials credentials); }; } // namespace duckdb diff --git a/src/include/catalog_utils.hpp b/src/include/catalog_utils.hpp index ee11eb8..240e63b 100644 --- a/src/include/catalog_utils.hpp +++ b/src/include/catalog_utils.hpp @@ -5,18 +5,18 @@ #include "catalog_api.hpp" namespace duckdb { -class IBSchemaEntry; -class IBTransaction; +class ICSchemaEntry; +class ICTransaction; -enum class IBTypeAnnotation { STANDARD, CAST_TO_VARCHAR, NUMERIC_AS_DOUBLE, CTID, JSONB, FIXED_LENGTH_CHAR }; +enum class ICTypeAnnotation { STANDARD, CAST_TO_VARCHAR, NUMERIC_AS_DOUBLE, CTID, JSONB, FIXED_LENGTH_CHAR }; -struct IBType { +struct ICType { idx_t oid = 0; - IBTypeAnnotation info = IBTypeAnnotation::STANDARD; - vector children; + ICTypeAnnotation info = ICTypeAnnotation::STANDARD; + vector children; }; -class IBUtils { +class ICUtils { public: static LogicalType ToUCType(const LogicalType &input); static LogicalType TypeToLogicalType(ClientContext &context, const string &columnDefinition); diff --git a/src/include/storage/ic_catalog.hpp b/src/include/storage/ic_catalog.hpp index 4957618..1c0abee 100644 --- a/src/include/storage/ic_catalog.hpp +++ b/src/include/storage/ic_catalog.hpp @@ -7,9 +7,9 @@ #include "storage/ic_schema_set.hpp" namespace duckdb { -class IBSchemaEntry; +class ICSchemaEntry; -struct IBCredentials { +struct ICCredentials { string endpoint; string client_id; string client_secret; @@ -19,23 +19,23 @@ struct IBCredentials { string token; }; -class IBClearCacheFunction : public TableFunction { +class ICClearCacheFunction : public TableFunction { public: - IBClearCacheFunction(); + ICClearCacheFunction(); static void ClearCacheOnSetting(ClientContext &context, SetScope scope, Value ¶meter); }; -class IBCatalog : public Catalog { +class ICCatalog : public Catalog { public: - explicit IBCatalog(AttachedDatabase &db_p, const string &internal_name, AccessMode access_mode, - IBCredentials credentials); - ~IBCatalog(); + explicit ICCatalog(AttachedDatabase &db_p, const string &internal_name, AccessMode access_mode, + ICCredentials credentials); + ~ICCatalog(); string internal_name; AccessMode access_mode; - IBCredentials credentials; + ICCredentials credentials; public: void Initialize(bool load_builtin) override; @@ -74,7 +74,7 @@ class IBCatalog : public Catalog { void DropSchema(ClientContext &context, DropInfo &info) override; private: - IBSchemaSet schemas; + ICSchemaSet schemas; string default_schema; }; diff --git a/src/include/storage/ic_catalog_set.hpp b/src/include/storage/ic_catalog_set.hpp index 9f9befb..927c6a2 100644 --- a/src/include/storage/ic_catalog_set.hpp +++ b/src/include/storage/ic_catalog_set.hpp @@ -7,12 +7,12 @@ namespace duckdb { struct DropInfo; -class IBSchemaEntry; -class IBTransaction; +class ICSchemaEntry; +class ICTransaction; -class IBCatalogSet { +class ICCatalogSet { public: - IBCatalogSet(Catalog &catalog); + ICCatalogSet(Catalog &catalog); optional_ptr GetEntry(ClientContext &context, const string &name); virtual void DropEntry(ClientContext &context, DropInfo &info); diff --git a/src/include/storage/ic_schema_entry.hpp b/src/include/storage/ic_schema_entry.hpp index c8d5e1c..344066b 100644 --- a/src/include/storage/ic_schema_entry.hpp +++ b/src/include/storage/ic_schema_entry.hpp @@ -6,14 +6,14 @@ #include "storage/ic_table_set.hpp" namespace duckdb { -class IBTransaction; +class ICTransaction; -class IBSchemaEntry : public SchemaCatalogEntry { +class ICSchemaEntry : public SchemaCatalogEntry { public: - IBSchemaEntry(Catalog &catalog, CreateSchemaInfo &info); - ~IBSchemaEntry() override; + ICSchemaEntry(Catalog &catalog, CreateSchemaInfo &info); + ~ICSchemaEntry() override; - unique_ptr schema_data; + unique_ptr schema_data; public: optional_ptr CreateTable(CatalogTransaction transaction, BoundCreateTableInfo &info) override; @@ -37,10 +37,10 @@ class IBSchemaEntry : public SchemaCatalogEntry { optional_ptr GetEntry(CatalogTransaction transaction, CatalogType type, const string &name) override; private: - IBCatalogSet &GetCatalogSet(CatalogType type); + ICCatalogSet &GetCatalogSet(CatalogType type); private: - IBTableSet tables; + ICTableSet tables; }; } // namespace duckdb diff --git a/src/include/storage/ic_schema_set.hpp b/src/include/storage/ic_schema_set.hpp index 6e8332f..6a5068c 100644 --- a/src/include/storage/ic_schema_set.hpp +++ b/src/include/storage/ic_schema_set.hpp @@ -7,9 +7,9 @@ namespace duckdb { struct CreateSchemaInfo; -class IBSchemaSet : public IBCatalogSet { +class ICSchemaSet : public ICCatalogSet { public: - explicit IBSchemaSet(Catalog &catalog); + explicit ICSchemaSet(Catalog &catalog); public: optional_ptr CreateSchema(ClientContext &context, CreateSchemaInfo &info); diff --git a/src/include/storage/ic_table_entry.hpp b/src/include/storage/ic_table_entry.hpp index f648237..e415c1c 100644 --- a/src/include/storage/ic_table_entry.hpp +++ b/src/include/storage/ic_table_entry.hpp @@ -7,14 +7,14 @@ namespace duckdb { -struct IBTableInfo { - IBTableInfo() { +struct ICTableInfo { + ICTableInfo() { create_info = make_uniq(); } - IBTableInfo(const string &schema, const string &table) { + ICTableInfo(const string &schema, const string &table) { create_info = make_uniq(string(), schema, table); } - IBTableInfo(const SchemaCatalogEntry &schema, const string &table) { + ICTableInfo(const SchemaCatalogEntry &schema, const string &table) { create_info = make_uniq((SchemaCatalogEntry &)schema, table); } @@ -25,12 +25,12 @@ struct IBTableInfo { unique_ptr create_info; }; -class IBTableEntry : public TableCatalogEntry { +class ICTableEntry : public TableCatalogEntry { public: - IBTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTableInfo &info); - IBTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, IBTableInfo &info); + ICTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTableInfo &info); + ICTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, ICTableInfo &info); - unique_ptr table_data; + unique_ptr table_data; public: unique_ptr GetStatistics(ClientContext &context, column_t column_id) override; diff --git a/src/include/storage/ic_table_set.hpp b/src/include/storage/ic_table_set.hpp index 25b7c8e..7e41593 100644 --- a/src/include/storage/ic_table_set.hpp +++ b/src/include/storage/ic_table_set.hpp @@ -6,28 +6,28 @@ namespace duckdb { struct CreateTableInfo; -class IBResult; -class IBSchemaEntry; +class ICResult; +class ICSchemaEntry; -class IBInSchemaSet : public IBCatalogSet { +class ICInSchemaSet : public ICCatalogSet { public: - IBInSchemaSet(IBSchemaEntry &schema); + ICInSchemaSet(ICSchemaEntry &schema); optional_ptr CreateEntry(unique_ptr entry) override; protected: - IBSchemaEntry &schema; + ICSchemaEntry &schema; }; -class IBTableSet : public IBInSchemaSet { +class ICTableSet : public ICInSchemaSet { public: - explicit IBTableSet(IBSchemaEntry &schema); + explicit ICTableSet(ICSchemaEntry &schema); public: optional_ptr CreateTable(ClientContext &context, BoundCreateTableInfo &info); - static unique_ptr GetTableInfo(ClientContext &context, IBSchemaEntry &schema, const string &table_name); + static unique_ptr GetTableInfo(ClientContext &context, ICSchemaEntry &schema, const string &table_name); optional_ptr RefreshTable(ClientContext &context, const string &table_name); void AlterTable(ClientContext &context, AlterTableInfo &info); @@ -40,10 +40,10 @@ class IBTableSet : public IBInSchemaSet { void AlterTable(ClientContext &context, AddColumnInfo &info); void AlterTable(ClientContext &context, RemoveColumnInfo &info); - static void AddColumn(ClientContext &context, IBResult &result, IBTableInfo &table_info, idx_t column_offset = 0); + static void AddColumn(ClientContext &context, ICResult &result, ICTableInfo &table_info, idx_t column_offset = 0); private: - unique_ptr _CreateCatalogEntry(ClientContext &context, IBAPITable table); + unique_ptr _CreateCatalogEntry(ClientContext &context, ICAPITable table); }; diff --git a/src/include/storage/ic_transaction.hpp b/src/include/storage/ic_transaction.hpp index 7dba882..79baf62 100644 --- a/src/include/storage/ic_transaction.hpp +++ b/src/include/storage/ic_transaction.hpp @@ -4,28 +4,28 @@ #include "duckdb/transaction/transaction.hpp" namespace duckdb { -class IBCatalog; -class IBSchemaEntry; -class IBTableEntry; +class ICCatalog; +class ICSchemaEntry; +class ICTableEntry; -enum class IBTransactionState { TRANSACTION_NOT_YET_STARTED, TRANSACTION_STARTED, TRANSACTION_FINISHED }; +enum class ICTransactionState { TRANSACTION_NOT_YET_STARTED, TRANSACTION_STARTED, TRANSACTION_FINISHED }; -class IBTransaction : public Transaction { +class ICTransaction : public Transaction { public: - IBTransaction(IBCatalog &ic_catalog, TransactionManager &manager, ClientContext &context); - ~IBTransaction() override; + ICTransaction(ICCatalog &ic_catalog, TransactionManager &manager, ClientContext &context); + ~ICTransaction() override; void Start(); void Commit(); void Rollback(); - static IBTransaction &Get(ClientContext &context, Catalog &catalog); + static ICTransaction &Get(ClientContext &context, Catalog &catalog); AccessMode GetAccessMode() const { return access_mode; } private: - IBTransactionState transaction_state; + ICTransactionState transaction_state; AccessMode access_mode; }; diff --git a/src/include/storage/ic_transaction_manager.hpp b/src/include/storage/ic_transaction_manager.hpp index 4d02af7..5337234 100644 --- a/src/include/storage/ic_transaction_manager.hpp +++ b/src/include/storage/ic_transaction_manager.hpp @@ -7,9 +7,9 @@ namespace duckdb { -class IBTransactionManager : public TransactionManager { +class ICTransactionManager : public TransactionManager { public: - IBTransactionManager(AttachedDatabase &db_p, IBCatalog &ic_catalog); + ICTransactionManager(AttachedDatabase &db_p, ICCatalog &ic_catalog); Transaction &StartTransaction(ClientContext &context) override; ErrorData CommitTransaction(ClientContext &context, Transaction &transaction) override; @@ -18,9 +18,9 @@ class IBTransactionManager : public TransactionManager { void Checkpoint(ClientContext &context, bool force = false) override; private: - IBCatalog &ic_catalog; + ICCatalog &ic_catalog; mutex transaction_lock; - reference_map_t> transactions; + reference_map_t> transactions; }; } // namespace duckdb diff --git a/src/storage/ic_catalog.cpp b/src/storage/ic_catalog.cpp index 4238651..a920c29 100644 --- a/src/storage/ic_catalog.cpp +++ b/src/storage/ic_catalog.cpp @@ -8,18 +8,18 @@ namespace duckdb { -IBCatalog::IBCatalog(AttachedDatabase &db_p, const string &internal_name, AccessMode access_mode, - IBCredentials credentials) +ICCatalog::ICCatalog(AttachedDatabase &db_p, const string &internal_name, AccessMode access_mode, + ICCredentials credentials) : Catalog(db_p), internal_name(internal_name), access_mode(access_mode), credentials(std::move(credentials)), schemas(*this) { } -IBCatalog::~IBCatalog() = default; +ICCatalog::~ICCatalog() = default; -void IBCatalog::Initialize(bool load_builtin) { +void ICCatalog::Initialize(bool load_builtin) { } -optional_ptr IBCatalog::CreateSchema(CatalogTransaction transaction, CreateSchemaInfo &info) { +optional_ptr ICCatalog::CreateSchema(CatalogTransaction transaction, CreateSchemaInfo &info) { if (info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) { DropInfo try_drop; try_drop.type = CatalogType::SCHEMA_ENTRY; @@ -31,15 +31,15 @@ optional_ptr IBCatalog::CreateSchema(CatalogTransaction transactio return schemas.CreateSchema(transaction.GetContext(), info); } -void IBCatalog::DropSchema(ClientContext &context, DropInfo &info) { +void ICCatalog::DropSchema(ClientContext &context, DropInfo &info) { return schemas.DropSchema(context, info); } -void IBCatalog::ScanSchemas(ClientContext &context, std::function callback) { - schemas.Scan(context, [&](CatalogEntry &schema) { callback(schema.Cast()); }); +void ICCatalog::ScanSchemas(ClientContext &context, std::function callback) { + schemas.Scan(context, [&](CatalogEntry &schema) { callback(schema.Cast()); }); } -optional_ptr IBCatalog::GetSchema(CatalogTransaction transaction, const string &schema_name, +optional_ptr ICCatalog::GetSchema(CatalogTransaction transaction, const string &schema_name, OnEntryNotFound if_not_found, QueryErrorContext error_context) { if (schema_name == DEFAULT_SCHEMA) { if (default_schema.empty()) { @@ -55,15 +55,15 @@ optional_ptr IBCatalog::GetSchema(CatalogTransaction transac return reinterpret_cast(entry.get()); } -bool IBCatalog::InMemory() { +bool ICCatalog::InMemory() { return false; } -string IBCatalog::GetDBPath() { +string ICCatalog::GetDBPath() { return internal_name; } -DatabaseSize IBCatalog::GetDatabaseSize(ClientContext &context) { +DatabaseSize ICCatalog::GetDatabaseSize(ClientContext &context) { if (default_schema.empty()) { throw InvalidInputException("Attempting to fetch the database size - but no database was provided " "in the connection string"); @@ -72,29 +72,29 @@ DatabaseSize IBCatalog::GetDatabaseSize(ClientContext &context) { return size; } -void IBCatalog::ClearCache() { +void ICCatalog::ClearCache() { schemas.ClearEntries(); } -unique_ptr IBCatalog::PlanInsert(ClientContext &context, LogicalInsert &op, +unique_ptr ICCatalog::PlanInsert(ClientContext &context, LogicalInsert &op, unique_ptr plan) { - throw NotImplementedException("IBCatalog PlanInsert"); + throw NotImplementedException("ICCatalog PlanInsert"); } -unique_ptr IBCatalog::PlanCreateTableAs(ClientContext &context, LogicalCreateTable &op, +unique_ptr ICCatalog::PlanCreateTableAs(ClientContext &context, LogicalCreateTable &op, unique_ptr plan) { - throw NotImplementedException("IBCatalog PlanCreateTableAs"); + throw NotImplementedException("ICCatalog PlanCreateTableAs"); } -unique_ptr IBCatalog::PlanDelete(ClientContext &context, LogicalDelete &op, +unique_ptr ICCatalog::PlanDelete(ClientContext &context, LogicalDelete &op, unique_ptr plan) { - throw NotImplementedException("IBCatalog PlanDelete"); + throw NotImplementedException("ICCatalog PlanDelete"); } -unique_ptr IBCatalog::PlanUpdate(ClientContext &context, LogicalUpdate &op, +unique_ptr ICCatalog::PlanUpdate(ClientContext &context, LogicalUpdate &op, unique_ptr plan) { - throw NotImplementedException("IBCatalog PlanUpdate"); + throw NotImplementedException("ICCatalog PlanUpdate"); } -unique_ptr IBCatalog::BindCreateIndex(Binder &binder, CreateStatement &stmt, TableCatalogEntry &table, +unique_ptr ICCatalog::BindCreateIndex(Binder &binder, CreateStatement &stmt, TableCatalogEntry &table, unique_ptr plan) { - throw NotImplementedException("IBCatalog BindCreateIndex"); + throw NotImplementedException("ICCatalog BindCreateIndex"); } } // namespace duckdb diff --git a/src/storage/ic_catalog_set.cpp b/src/storage/ic_catalog_set.cpp index cc00d54..c3b01a1 100644 --- a/src/storage/ic_catalog_set.cpp +++ b/src/storage/ic_catalog_set.cpp @@ -5,10 +5,10 @@ namespace duckdb { -IBCatalogSet::IBCatalogSet(Catalog &catalog) : catalog(catalog) { +ICCatalogSet::ICCatalogSet(Catalog &catalog) : catalog(catalog) { } -optional_ptr IBCatalogSet::GetEntry(ClientContext &context, const string &name) { +optional_ptr ICCatalogSet::GetEntry(ClientContext &context, const string &name) { LoadEntries(context); lock_guard l(entry_lock); auto entry = entries.find(name); @@ -19,16 +19,18 @@ optional_ptr IBCatalogSet::GetEntry(ClientContext &context, const return entry->second.get(); } -void IBCatalogSet::DropEntry(ClientContext &context, DropInfo &info) { +void ICCatalogSet::DropEntry(ClientContext &context, DropInfo &info) { + std::cout << "ICCatalogSet::DropEntry" << info.type << std::endl; + EraseEntryInternal(info.name); } -void IBCatalogSet::EraseEntryInternal(const string &name) { +void ICCatalogSet::EraseEntryInternal(const string &name) { lock_guard l(entry_lock); entries.erase(name); } -void IBCatalogSet::Scan(ClientContext &context, const std::function &callback) { +void ICCatalogSet::Scan(ClientContext &context, const std::function &callback) { LoadEntries(context); lock_guard l(entry_lock); @@ -37,28 +39,28 @@ void IBCatalogSet::Scan(ClientContext &context, const std::function IBCatalogSet::CreateEntry(unique_ptr entry) { +optional_ptr ICCatalogSet::CreateEntry(unique_ptr entry) { lock_guard l(entry_lock); auto result = entry.get(); if (result->name.empty()) { - throw InternalException("IBCatalogSet::CreateEntry called with empty name"); + throw InternalException("ICCatalogSet::CreateEntry called with empty name"); } entries.insert(make_pair(result->name, std::move(entry))); return result; } -void IBCatalogSet::ClearEntries() { +void ICCatalogSet::ClearEntries() { entries.clear(); } -IBInSchemaSet::IBInSchemaSet(IBSchemaEntry &schema) : IBCatalogSet(schema.ParentCatalog()), schema(schema) { +ICInSchemaSet::ICInSchemaSet(ICSchemaEntry &schema) : ICCatalogSet(schema.ParentCatalog()), schema(schema) { } -optional_ptr IBInSchemaSet::CreateEntry(unique_ptr entry) { +optional_ptr ICInSchemaSet::CreateEntry(unique_ptr entry) { if (!entry->internal) { entry->internal = schema.internal; } - return IBCatalogSet::CreateEntry(std::move(entry)); + return ICCatalogSet::CreateEntry(std::move(entry)); } } // namespace duckdb diff --git a/src/storage/ic_clear_cache.cpp b/src/storage/ic_clear_cache.cpp index 99db62a..0e07887 100644 --- a/src/storage/ic_clear_cache.cpp +++ b/src/storage/ic_clear_cache.cpp @@ -28,7 +28,7 @@ static void ClearUCCaches(ClientContext &context) { if (catalog.GetCatalogType() != "iceberg") { continue; } - catalog.Cast().ClearCache(); + catalog.Cast().ClearCache(); } } @@ -41,10 +41,10 @@ static void ClearCacheFunction(ClientContext &context, TableFunctionInput &data_ data.finished = true; } -void IBClearCacheFunction::ClearCacheOnSetting(ClientContext &context, SetScope scope, Value ¶meter) { +void ICClearCacheFunction::ClearCacheOnSetting(ClientContext &context, SetScope scope, Value ¶meter) { ClearUCCaches(context); } -IBClearCacheFunction::IBClearCacheFunction() : TableFunction("pc_clear_cache", {}, ClearCacheFunction, ClearCacheBind) { +ICClearCacheFunction::ICClearCacheFunction() : TableFunction("pc_clear_cache", {}, ClearCacheFunction, ClearCacheBind) { } } // namespace duckdb diff --git a/src/storage/ic_schema_entry.cpp b/src/storage/ic_schema_entry.cpp index b137d1c..691c9b0 100644 --- a/src/storage/ic_schema_entry.cpp +++ b/src/storage/ic_schema_entry.cpp @@ -13,21 +13,21 @@ namespace duckdb { -IBSchemaEntry::IBSchemaEntry(Catalog &catalog, CreateSchemaInfo &info) +ICSchemaEntry::ICSchemaEntry(Catalog &catalog, CreateSchemaInfo &info) : SchemaCatalogEntry(catalog, info), tables(*this) { } -IBSchemaEntry::~IBSchemaEntry() { +ICSchemaEntry::~ICSchemaEntry() { } -IBTransaction &GetUCTransaction(CatalogTransaction transaction) { +ICTransaction &GetUCTransaction(CatalogTransaction transaction) { if (!transaction.transaction) { throw InternalException("No transaction!?"); } - return transaction.transaction->Cast(); + return transaction.transaction->Cast(); } -optional_ptr IBSchemaEntry::CreateTable(CatalogTransaction transaction, BoundCreateTableInfo &info) { +optional_ptr ICSchemaEntry::CreateTable(CatalogTransaction transaction, BoundCreateTableInfo &info) { auto &base_info = info.Base(); auto table_name = base_info.table; if (base_info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) { @@ -36,21 +36,21 @@ optional_ptr IBSchemaEntry::CreateTable(CatalogTransaction transac return tables.CreateTable(transaction.GetContext(), info); } -optional_ptr IBSchemaEntry::CreateFunction(CatalogTransaction transaction, CreateFunctionInfo &info) { +optional_ptr ICSchemaEntry::CreateFunction(CatalogTransaction transaction, CreateFunctionInfo &info) { throw BinderException("PC databases do not support creating functions"); } -void IBUnqualifyColumnRef(ParsedExpression &expr) { +void ICUnqualifyColumnRef(ParsedExpression &expr) { if (expr.type == ExpressionType::COLUMN_REF) { auto &colref = expr.Cast(); auto name = std::move(colref.column_names.back()); colref.column_names = {std::move(name)}; return; } - ParsedExpressionIterator::EnumerateChildren(expr, IBUnqualifyColumnRef); + ParsedExpressionIterator::EnumerateChildren(expr, ICUnqualifyColumnRef); } -optional_ptr IBSchemaEntry::CreateIndex(CatalogTransaction transaction, CreateIndexInfo &info, +optional_ptr ICSchemaEntry::CreateIndex(CatalogTransaction transaction, CreateIndexInfo &info, TableCatalogEntry &table) { throw NotImplementedException("CreateIndex"); } @@ -59,7 +59,7 @@ string GetUCCreateView(CreateViewInfo &info) { throw NotImplementedException("GetCreateView"); } -optional_ptr IBSchemaEntry::CreateView(CatalogTransaction transaction, CreateViewInfo &info) { +optional_ptr ICSchemaEntry::CreateView(CatalogTransaction transaction, CreateViewInfo &info) { if (info.sql.empty()) { throw BinderException("Cannot create view that originated from an " "empty SQL statement"); @@ -79,34 +79,34 @@ optional_ptr IBSchemaEntry::CreateView(CatalogTransaction transact return tables.RefreshTable(transaction.GetContext(), info.view_name); } -optional_ptr IBSchemaEntry::CreateType(CatalogTransaction transaction, CreateTypeInfo &info) { +optional_ptr ICSchemaEntry::CreateType(CatalogTransaction transaction, CreateTypeInfo &info) { throw BinderException("PC databases do not support creating types"); } -optional_ptr IBSchemaEntry::CreateSequence(CatalogTransaction transaction, CreateSequenceInfo &info) { +optional_ptr ICSchemaEntry::CreateSequence(CatalogTransaction transaction, CreateSequenceInfo &info) { throw BinderException("PC databases do not support creating sequences"); } -optional_ptr IBSchemaEntry::CreateTableFunction(CatalogTransaction transaction, +optional_ptr ICSchemaEntry::CreateTableFunction(CatalogTransaction transaction, CreateTableFunctionInfo &info) { throw BinderException("PC databases do not support creating table functions"); } -optional_ptr IBSchemaEntry::CreateCopyFunction(CatalogTransaction transaction, +optional_ptr ICSchemaEntry::CreateCopyFunction(CatalogTransaction transaction, CreateCopyFunctionInfo &info) { throw BinderException("PC databases do not support creating copy functions"); } -optional_ptr IBSchemaEntry::CreatePragmaFunction(CatalogTransaction transaction, +optional_ptr ICSchemaEntry::CreatePragmaFunction(CatalogTransaction transaction, CreatePragmaFunctionInfo &info) { throw BinderException("PC databases do not support creating pragma functions"); } -optional_ptr IBSchemaEntry::CreateCollation(CatalogTransaction transaction, CreateCollationInfo &info) { +optional_ptr ICSchemaEntry::CreateCollation(CatalogTransaction transaction, CreateCollationInfo &info) { throw BinderException("PC databases do not support creating collations"); } -void IBSchemaEntry::Alter(CatalogTransaction transaction, AlterInfo &info) { +void ICSchemaEntry::Alter(CatalogTransaction transaction, AlterInfo &info) { if (info.type != AlterType::ALTER_TABLE) { throw BinderException("Only altering tables is supported for now"); } @@ -125,22 +125,22 @@ bool CatalogTypeIsSupported(CatalogType type) { } } -void IBSchemaEntry::Scan(ClientContext &context, CatalogType type, +void ICSchemaEntry::Scan(ClientContext &context, CatalogType type, const std::function &callback) { if (!CatalogTypeIsSupported(type)) { return; } GetCatalogSet(type).Scan(context, callback); } -void IBSchemaEntry::Scan(CatalogType type, const std::function &callback) { +void ICSchemaEntry::Scan(CatalogType type, const std::function &callback) { throw NotImplementedException("Scan without context not supported"); } -void IBSchemaEntry::DropEntry(ClientContext &context, DropInfo &info) { +void ICSchemaEntry::DropEntry(ClientContext &context, DropInfo &info) { GetCatalogSet(info.type).DropEntry(context, info); } -optional_ptr IBSchemaEntry::GetEntry(CatalogTransaction transaction, CatalogType type, +optional_ptr ICSchemaEntry::GetEntry(CatalogTransaction transaction, CatalogType type, const string &name) { if (!CatalogTypeIsSupported(type)) { return nullptr; @@ -148,7 +148,7 @@ optional_ptr IBSchemaEntry::GetEntry(CatalogTransaction transactio return GetCatalogSet(type).GetEntry(transaction.GetContext(), name); } -IBCatalogSet &IBSchemaEntry::GetCatalogSet(CatalogType type) { +ICCatalogSet &ICSchemaEntry::GetCatalogSet(CatalogType type) { switch (type) { case CatalogType::TABLE_ENTRY: case CatalogType::VIEW_ENTRY: diff --git a/src/storage/ic_schema_set.cpp b/src/storage/ic_schema_set.cpp index fc7b1f1..b35b624 100644 --- a/src/storage/ic_schema_set.cpp +++ b/src/storage/ic_schema_set.cpp @@ -8,7 +8,7 @@ namespace duckdb { -IBSchemaSet::IBSchemaSet(Catalog &catalog) : IBCatalogSet(catalog) { +ICSchemaSet::ICSchemaSet(Catalog &catalog) : ICCatalogSet(catalog) { } static bool IsInternalTable(const string &catalog, const string &schema) { @@ -18,38 +18,38 @@ static bool IsInternalTable(const string &catalog, const string &schema) { return false; } -void IBSchemaSet::LoadEntries(ClientContext &context) { +void ICSchemaSet::LoadEntries(ClientContext &context) { if (!entries.empty()) { return; } - auto &ic_catalog = catalog.Cast(); - auto schemas = IBAPI::GetSchemas(catalog.GetName(), ic_catalog.internal_name, ic_catalog.credentials); + auto &ic_catalog = catalog.Cast(); + auto schemas = ICAPI::GetSchemas(catalog.GetName(), ic_catalog.internal_name, ic_catalog.credentials); for (const auto &schema : schemas) { CreateSchemaInfo info; info.schema = schema.schema_name; info.internal = IsInternalTable(schema.catalog_name, schema.schema_name); - auto schema_entry = make_uniq(catalog, info); - schema_entry->schema_data = make_uniq(schema); + auto schema_entry = make_uniq(catalog, info); + schema_entry->schema_data = make_uniq(schema); CreateEntry(std::move(schema_entry)); } } -void IBSchemaSet::FillEntry(ClientContext &context, unique_ptr &entry) { +void ICSchemaSet::FillEntry(ClientContext &context, unique_ptr &entry) { // Nothing to do } -optional_ptr IBSchemaSet::CreateSchema(ClientContext &context, CreateSchemaInfo &info) { - auto &ic_catalog = catalog.Cast(); - auto schema = IBAPI::CreateSchema(catalog.GetName(), ic_catalog.internal_name, info.schema, ic_catalog.credentials); - auto schema_entry = make_uniq(catalog, info); - schema_entry->schema_data = make_uniq(schema); +optional_ptr ICSchemaSet::CreateSchema(ClientContext &context, CreateSchemaInfo &info) { + auto &ic_catalog = catalog.Cast(); + auto schema = ICAPI::CreateSchema(catalog.GetName(), ic_catalog.internal_name, info.schema, ic_catalog.credentials); + auto schema_entry = make_uniq(catalog, info); + schema_entry->schema_data = make_uniq(schema); return CreateEntry(std::move(schema_entry)); } -void IBSchemaSet::DropSchema(ClientContext &context, DropInfo &info) { - auto &ic_catalog = catalog.Cast(); - IBAPI::DropSchema(ic_catalog.internal_name, info.name, ic_catalog.credentials); +void ICSchemaSet::DropSchema(ClientContext &context, DropInfo &info) { + auto &ic_catalog = catalog.Cast(); + ICAPI::DropSchema(ic_catalog.internal_name, info.name, ic_catalog.credentials); DropEntry(context, info); } diff --git a/src/storage/ic_table_entry.cpp b/src/storage/ic_table_entry.cpp index e625ee8..18b9501 100644 --- a/src/storage/ic_table_entry.cpp +++ b/src/storage/ic_table_entry.cpp @@ -19,21 +19,21 @@ namespace duckdb { -IBTableEntry::IBTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTableInfo &info) +ICTableEntry::ICTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTableInfo &info) : TableCatalogEntry(catalog, schema, info) { this->internal = false; } -IBTableEntry::IBTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, IBTableInfo &info) +ICTableEntry::ICTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, ICTableInfo &info) : TableCatalogEntry(catalog, schema, *info.create_info) { this->internal = false; } -unique_ptr IBTableEntry::GetStatistics(ClientContext &context, column_t column_id) { +unique_ptr ICTableEntry::GetStatistics(ClientContext &context, column_t column_id) { return nullptr; } -void IBTableEntry::BindUpdateConstraints(Binder &binder, LogicalGet &, LogicalProjection &, LogicalUpdate &, +void ICTableEntry::BindUpdateConstraints(Binder &binder, LogicalGet &, LogicalProjection &, LogicalUpdate &, ClientContext &) { throw NotImplementedException("BindUpdateConstraints"); } @@ -55,9 +55,9 @@ struct MyIcebergFunctionData : public FunctionData { } }; -TableFunction IBTableEntry::GetScanFunction(ClientContext &context, unique_ptr &bind_data) { +TableFunction ICTableEntry::GetScanFunction(ClientContext &context, unique_ptr &bind_data) { auto &db = DatabaseInstance::GetDatabase(context); - auto &ic_catalog = catalog.Cast(); + auto &ic_catalog = catalog.Cast(); auto &parquet_function_set = ExtensionUtil::GetTableFunction(db, "parquet_scan"); auto parquet_scan_function = parquet_function_set.functions.GetFunctionByArguments(context, {LogicalType::VARCHAR}); @@ -74,8 +74,8 @@ TableFunction IBTableEntry::GetScanFunction(ClientContext &context, unique_ptrstorage_location.find("file://") != 0) { auto &secret_manager = SecretManager::Get(context); - // Get Credentials from IBAPI - auto table_credentials = IBAPI::GetTableCredentials( + // Get Credentials from ICAPI + auto table_credentials = ICAPI::GetTableCredentials( ic_catalog.internal_name, table_data->schema_name, table_data->name, ic_catalog.credentials); // Inject secret into secret manager scoped to this path @@ -138,7 +138,7 @@ TableFunction IBTableEntry::GetScanFunction(ClientContext &context, unique_ptr IBTableSet::_CreateCatalogEntry(ClientContext &context, IBAPITable table) { +unique_ptr ICTableSet::_CreateCatalogEntry(ClientContext &context, ICAPITable table) { D_ASSERT(schema.name == table.schema_name); CreateTableInfo info; info.table = table.name; @@ -33,30 +33,30 @@ unique_ptr IBTableSet::_CreateCatalogEntry(ClientContext &context, info.columns.AddColumn(CreateColumnDefinition(context, col)); } - auto table_entry = make_uniq(catalog, schema, info); - table_entry->table_data = make_uniq(table); + auto table_entry = make_uniq(catalog, schema, info); + table_entry->table_data = make_uniq(table); return table_entry; } -void IBTableSet::FillEntry(ClientContext &context, unique_ptr &entry) { - auto* derived = static_cast(entry.get()); +void ICTableSet::FillEntry(ClientContext &context, unique_ptr &entry) { + auto* derived = static_cast(entry.get()); if (!derived->table_data->storage_location.empty()) { return; } - auto &ic_catalog = catalog.Cast(); - auto table = IBAPI::GetTable(catalog.GetName(), catalog.GetDBPath(), schema.name, entry->name, ic_catalog.credentials); + auto &ic_catalog = catalog.Cast(); + auto table = ICAPI::GetTable(catalog.GetName(), catalog.GetDBPath(), schema.name, entry->name, ic_catalog.credentials); entry = _CreateCatalogEntry(context, table); } -void IBTableSet::LoadEntries(ClientContext &context) { +void ICTableSet::LoadEntries(ClientContext &context) { if (!entries.empty()) { return; } - auto &ic_catalog = catalog.Cast(); + auto &ic_catalog = catalog.Cast(); // TODO: handle out-of-order columns using position property - auto tables = IBAPI::GetTables(catalog.GetName(), catalog.GetDBPath(), schema.name, ic_catalog.credentials); + auto tables = ICAPI::GetTables(catalog.GetName(), catalog.GetDBPath(), schema.name, ic_catalog.credentials); for (auto &table : tables) { auto entry = _CreateCatalogEntry(context, table); @@ -64,41 +64,41 @@ void IBTableSet::LoadEntries(ClientContext &context) { } } -optional_ptr IBTableSet::RefreshTable(ClientContext &context, const string &table_name) { +optional_ptr ICTableSet::RefreshTable(ClientContext &context, const string &table_name) { auto table_info = GetTableInfo(context, schema, table_name); - auto table_entry = make_uniq(catalog, schema, *table_info); + auto table_entry = make_uniq(catalog, schema, *table_info); auto table_ptr = table_entry.get(); CreateEntry(std::move(table_entry)); return table_ptr; } -unique_ptr IBTableSet::GetTableInfo(ClientContext &context, IBSchemaEntry &schema, +unique_ptr ICTableSet::GetTableInfo(ClientContext &context, ICSchemaEntry &schema, const string &table_name) { - throw NotImplementedException("IBTableSet::CreateTable"); + throw NotImplementedException("ICTableSet::CreateTable"); } -optional_ptr IBTableSet::CreateTable(ClientContext &context, BoundCreateTableInfo &info) { - throw NotImplementedException("IBTableSet::CreateTable"); +optional_ptr ICTableSet::CreateTable(ClientContext &context, BoundCreateTableInfo &info) { + throw NotImplementedException("ICTableSet::CreateTable"); } -void IBTableSet::AlterTable(ClientContext &context, RenameTableInfo &info) { - throw NotImplementedException("IBTableSet::AlterTable"); +void ICTableSet::AlterTable(ClientContext &context, RenameTableInfo &info) { + throw NotImplementedException("ICTableSet::AlterTable"); } -void IBTableSet::AlterTable(ClientContext &context, RenameColumnInfo &info) { - throw NotImplementedException("IBTableSet::AlterTable"); +void ICTableSet::AlterTable(ClientContext &context, RenameColumnInfo &info) { + throw NotImplementedException("ICTableSet::AlterTable"); } -void IBTableSet::AlterTable(ClientContext &context, AddColumnInfo &info) { - throw NotImplementedException("IBTableSet::AlterTable"); +void ICTableSet::AlterTable(ClientContext &context, AddColumnInfo &info) { + throw NotImplementedException("ICTableSet::AlterTable"); } -void IBTableSet::AlterTable(ClientContext &context, RemoveColumnInfo &info) { - throw NotImplementedException("IBTableSet::AlterTable"); +void ICTableSet::AlterTable(ClientContext &context, RemoveColumnInfo &info) { + throw NotImplementedException("ICTableSet::AlterTable"); } -void IBTableSet::AlterTable(ClientContext &context, AlterTableInfo &alter) { - throw NotImplementedException("IBTableSet::AlterTable"); +void ICTableSet::AlterTable(ClientContext &context, AlterTableInfo &alter) { + throw NotImplementedException("ICTableSet::AlterTable"); } } // namespace duckdb diff --git a/src/storage/ic_transaction.cpp b/src/storage/ic_transaction.cpp index 57c2d2b..70c29c7 100644 --- a/src/storage/ic_transaction.cpp +++ b/src/storage/ic_transaction.cpp @@ -6,32 +6,32 @@ namespace duckdb { -IBTransaction::IBTransaction(IBCatalog &ic_catalog, TransactionManager &manager, ClientContext &context) +ICTransaction::ICTransaction(ICCatalog &ic_catalog, TransactionManager &manager, ClientContext &context) : Transaction(manager, context), access_mode(ic_catalog.access_mode) { - // connection = IBConnection::Open(ic_catalog.path); + // connection = ICConnection::Open(ic_catalog.path); } -IBTransaction::~IBTransaction() = default; +ICTransaction::~ICTransaction() = default; -void IBTransaction::Start() { - transaction_state = IBTransactionState::TRANSACTION_NOT_YET_STARTED; +void ICTransaction::Start() { + transaction_state = ICTransactionState::TRANSACTION_NOT_YET_STARTED; } -void IBTransaction::Commit() { - if (transaction_state == IBTransactionState::TRANSACTION_STARTED) { - transaction_state = IBTransactionState::TRANSACTION_FINISHED; +void ICTransaction::Commit() { + if (transaction_state == ICTransactionState::TRANSACTION_STARTED) { + transaction_state = ICTransactionState::TRANSACTION_FINISHED; // connection.Execute("COMMIT"); } } -void IBTransaction::Rollback() { - if (transaction_state == IBTransactionState::TRANSACTION_STARTED) { - transaction_state = IBTransactionState::TRANSACTION_FINISHED; +void ICTransaction::Rollback() { + if (transaction_state == ICTransactionState::TRANSACTION_STARTED) { + transaction_state = ICTransactionState::TRANSACTION_FINISHED; // connection.Execute("ROLLBACK"); } } -// IBConnection &IBTransaction::GetConnection() { -// if (transaction_state == IBTransactionState::TRANSACTION_NOT_YET_STARTED) { -// transaction_state = IBTransactionState::TRANSACTION_STARTED; +// ICConnection &ICTransaction::GetConnection() { +// if (transaction_state == ICTransactionState::TRANSACTION_NOT_YET_STARTED) { +// transaction_state = ICTransactionState::TRANSACTION_STARTED; // string query = "START TRANSACTION"; // if (access_mode == AccessMode::READ_ONLY) { // query += " READ ONLY"; @@ -41,9 +41,9 @@ void IBTransaction::Rollback() { // return connection; //} -// unique_ptr IBTransaction::Query(const string &query) { -// if (transaction_state == IBTransactionState::TRANSACTION_NOT_YET_STARTED) { -// transaction_state = IBTransactionState::TRANSACTION_STARTED; +// unique_ptr ICTransaction::Query(const string &query) { +// if (transaction_state == ICTransactionState::TRANSACTION_NOT_YET_STARTED) { +// transaction_state = ICTransactionState::TRANSACTION_STARTED; // string transaction_start = "START TRANSACTION"; // if (access_mode == AccessMode::READ_ONLY) { // transaction_start += " READ ONLY"; @@ -54,8 +54,8 @@ void IBTransaction::Rollback() { // return connection.Query(query); //} -IBTransaction &IBTransaction::Get(ClientContext &context, Catalog &catalog) { - return Transaction::Get(context, catalog).Cast(); +ICTransaction &ICTransaction::Get(ClientContext &context, Catalog &catalog) { + return Transaction::Get(context, catalog).Cast(); } } // namespace duckdb diff --git a/src/storage/ic_transaction_manager.cpp b/src/storage/ic_transaction_manager.cpp index 6b2d3b9..f034de9 100644 --- a/src/storage/ic_transaction_manager.cpp +++ b/src/storage/ic_transaction_manager.cpp @@ -3,12 +3,12 @@ namespace duckdb { -IBTransactionManager::IBTransactionManager(AttachedDatabase &db_p, IBCatalog &ic_catalog) +ICTransactionManager::ICTransactionManager(AttachedDatabase &db_p, ICCatalog &ic_catalog) : TransactionManager(db_p), ic_catalog(ic_catalog) { } -Transaction &IBTransactionManager::StartTransaction(ClientContext &context) { - auto transaction = make_uniq(ic_catalog, *this, context); +Transaction &ICTransactionManager::StartTransaction(ClientContext &context) { + auto transaction = make_uniq(ic_catalog, *this, context); transaction->Start(); auto &result = *transaction; lock_guard l(transaction_lock); @@ -16,23 +16,23 @@ Transaction &IBTransactionManager::StartTransaction(ClientContext &context) { return result; } -ErrorData IBTransactionManager::CommitTransaction(ClientContext &context, Transaction &transaction) { - auto &ic_transaction = transaction.Cast(); +ErrorData ICTransactionManager::CommitTransaction(ClientContext &context, Transaction &transaction) { + auto &ic_transaction = transaction.Cast(); ic_transaction.Commit(); lock_guard l(transaction_lock); transactions.erase(transaction); return ErrorData(); } -void IBTransactionManager::RollbackTransaction(Transaction &transaction) { - auto &ic_transaction = transaction.Cast(); +void ICTransactionManager::RollbackTransaction(Transaction &transaction) { + auto &ic_transaction = transaction.Cast(); ic_transaction.Rollback(); lock_guard l(transaction_lock); transactions.erase(transaction); } -void IBTransactionManager::Checkpoint(ClientContext &context, bool force) { - auto &transaction = IBTransaction::Get(context, db.GetCatalog()); +void ICTransactionManager::Checkpoint(ClientContext &context, bool force) { + auto &transaction = ICTransaction::Get(context, db.GetCatalog()); // auto &db = transaction.GetConnection(); // db.Execute("CHECKPOINT"); } From dc0ebb36f3ab3c738e32313f1983544569060367 Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Fri, 24 Jan 2025 03:05:02 -0500 Subject: [PATCH 13/13] Complete table create & drop --- src/catalog_api.cpp | 202 +++++++++++++++++++-------- src/catalog_utils.cpp | 43 ++++++ src/include/catalog_api.hpp | 7 +- src/include/catalog_utils.hpp | 1 + src/include/storage/ic_table_set.hpp | 1 + src/storage/ic_catalog_set.cpp | 2 - src/storage/ic_schema_entry.cpp | 12 +- src/storage/ic_table_entry.cpp | 1 - src/storage/ic_table_set.cpp | 14 +- 9 files changed, 214 insertions(+), 69 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index d38d9a4..9beafb5 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -1,12 +1,14 @@ #include "catalog_api.hpp" +#include "catalog_utils.hpp" #include "storage/ic_catalog.hpp" #include "yyjson.hpp" + #include #include - #include #include +using namespace duckdb_yyjson; namespace duckdb { //! We use a global here to store the path that is selected on the ICAPI::InitializeCurl call @@ -33,6 +35,15 @@ static string certFileLocations[] = { "/etc/ssl/cert.pem" }; +struct YyjsonDocDeleter { + void operator()(yyjson_doc* doc) { + yyjson_doc_free(doc); + } + void operator()(yyjson_mut_doc* doc) { + yyjson_mut_doc_free(doc); + } +}; + // Look through the the above locations and if one of the files exists, set that as the location curl should use. static bool SelectCurlCertPath() { for (string& caFile : certFileLocations) { @@ -61,8 +72,8 @@ static void InitializeCurlObject(CURL * curl, const string &token) { SetCurlCAFileInfo(curl); } -template -static TYPE TemplatedTryGetYYJson(duckdb_yyjson::yyjson_val *obj, const string &field, TYPE default_val, +template +static TYPE TemplatedTryGetYYJson(yyjson_val *obj, const string &field, TYPE default_val, bool fail_on_missing = true) { auto val = yyjson_obj_get(obj, field.c_str()); if (val && yyjson_get_type(val) == TYPE_NUM) { @@ -73,19 +84,19 @@ static TYPE TemplatedTryGetYYJson(duckdb_yyjson::yyjson_val *obj, const string & throw IOException("Invalid field found while parsing field: " + field); } -static uint64_t TryGetNumFromObject(duckdb_yyjson::yyjson_val *obj, const string &field, bool fail_on_missing = true, +static uint64_t TryGetNumFromObject(yyjson_val *obj, const string &field, bool fail_on_missing = true, uint64_t default_val = 0) { - return TemplatedTryGetYYJson(obj, field, default_val, + return TemplatedTryGetYYJson(obj, field, default_val, fail_on_missing); } -static bool TryGetBoolFromObject(duckdb_yyjson::yyjson_val *obj, const string &field, bool fail_on_missing = false, +static bool TryGetBoolFromObject(yyjson_val *obj, const string &field, bool fail_on_missing = false, bool default_val = false) { - return TemplatedTryGetYYJson(obj, field, default_val, + return TemplatedTryGetYYJson(obj, field, default_val, fail_on_missing); } -static string TryGetStrFromObject(duckdb_yyjson::yyjson_val *obj, const string &field, bool fail_on_missing = true, +static string TryGetStrFromObject(yyjson_val *obj, const string &field, bool fail_on_missing = true, const char *default_val = "") { - return TemplatedTryGetYYJson(obj, field, default_val, + return TemplatedTryGetYYJson(obj, field, default_val, fail_on_missing); } @@ -197,25 +208,26 @@ static string PostRequest( return readBuffer; } -static duckdb_yyjson::yyjson_val *api_result_to_doc(const string &api_result) { - auto *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); +static yyjson_doc *api_result_to_doc(const string &api_result) { + auto *doc = yyjson_read(api_result.c_str(), api_result.size(), 0); auto *root = yyjson_doc_get_root(doc); auto *error = yyjson_obj_get(root, "error"); if (error != NULL) { string err_msg = TryGetStrFromObject(error, "message"); throw std::runtime_error(err_msg); } - return root; + return doc; } -static duckdb_yyjson::yyjson_val *GetTableMetadata(const string &internal, const string &schema, const string &table, ICCredentials credentials) { +static string GetTableMetadata(const string &internal, const string &schema, const string &table, ICCredentials credentials) { struct curl_slist *extra_headers = NULL; extra_headers = curl_slist_append(extra_headers, "X-Iceberg-Access-Delegation: vended-credentials"); - auto api_result = GetRequest( + string api_result = GetRequest( credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables/" + table, credentials.token, extra_headers); - return api_result_to_doc(api_result); + curl_slist_free_all(extra_headers); + return api_result; } void ICAPI::InitializeCurl() { @@ -226,7 +238,7 @@ vector ICAPI::GetCatalogs(const string &catalog, ICCredentials credentia throw NotImplementedException("ICAPI::GetCatalogs"); } -static ICAPIColumnDefinition ParseColumnDefinition(duckdb_yyjson::yyjson_val *column_def) { +static ICAPIColumnDefinition ParseColumnDefinition(yyjson_val *column_def) { ICAPIColumnDefinition result; result.name = TryGetStrFromObject(column_def, "name"); result.type_text = TryGetStrFromObject(column_def, "type"); @@ -238,64 +250,77 @@ static ICAPIColumnDefinition ParseColumnDefinition(duckdb_yyjson::yyjson_val *co ICAPITableCredentials ICAPI::GetTableCredentials(const string &internal, const string &schema, const string &table, ICCredentials credentials) { ICAPITableCredentials result; - duckdb_yyjson::yyjson_val *root = GetTableMetadata(internal, schema, table, credentials); + string api_result = GetTableMetadata(internal, schema, table, credentials); + std::unique_ptr doc(api_result_to_doc(api_result)); + auto *root = yyjson_doc_get_root(doc.get()); auto *aws_temp_credentials = yyjson_obj_get(root, "config"); - if (aws_temp_credentials) { result.key_id = TryGetStrFromObject(aws_temp_credentials, "s3.access-key-id"); result.secret = TryGetStrFromObject(aws_temp_credentials, "s3.secret-access-key"); result.session_token = TryGetStrFromObject(aws_temp_credentials, "s3.session-token"); + return result; } - return result; + throw std::runtime_error("No AWS credentials found for table"); } string ICAPI::GetToken(string id, string secret, string endpoint) { string post_data = "grant_type=client_credentials&client_id=" + id + "&client_secret=" + secret + "&scope=PRINCIPAL_ROLE:ALL"; string api_result = PostRequest(endpoint + "/v1/oauth/tokens", post_data); - auto *root = api_result_to_doc(api_result); + std::unique_ptr doc(api_result_to_doc(api_result)); + auto *root = yyjson_doc_get_root(doc.get()); return TryGetStrFromObject(root, "access_token"); } -ICAPITable ICAPI::GetTable( - const string &catalog, const string &internal, const string &schema, const string &table, std::optional credentials) { - +static void populateTableMetadata(ICAPITable &table, yyjson_val *metadata_root) { + table.storage_location = TryGetStrFromObject(metadata_root, "metadata-location"); + auto *metadata = yyjson_obj_get(metadata_root, "metadata"); + //table_result.table_id = TryGetStrFromObject(metadata, "table-uuid"); + + uint64_t current_schema_id = TryGetNumFromObject(metadata, "current-schema-id"); + auto *schemas = yyjson_obj_get(metadata, "schemas"); + yyjson_val *schema; + size_t schema_idx, schema_max; + bool found = false; + yyjson_arr_foreach(schemas, schema_idx, schema_max, schema) { + uint64_t schema_id = TryGetNumFromObject(schema, "schema-id"); + if (schema_id == current_schema_id) { + found = true; + auto *columns = yyjson_obj_get(schema, "fields"); + yyjson_val *col; + size_t col_idx, col_max; + yyjson_arr_foreach(columns, col_idx, col_max, col) { + auto column_definition = ParseColumnDefinition(col); + table.columns.push_back(column_definition); + } + } + } + + if (!found) { + throw InternalException("Current schema not found"); + } +} + +static ICAPITable createTable(const string &catalog, const string &schema, const string &table_name) { ICAPITable table_result; table_result.catalog_name = catalog; table_result.schema_name = schema; - table_result.name = table; + table_result.name = table_name; table_result.data_source_format = "ICEBERG"; table_result.table_id = "uuid-" + schema + "-" + "table"; std::replace(table_result.table_id.begin(), table_result.table_id.end(), '_', '-'); + return table_result; +} +ICAPITable ICAPI::GetTable( + const string &catalog, const string &internal, const string &schema, const string &table_name, std::optional credentials) { + + ICAPITable table_result = createTable(catalog, schema, table_name); if (credentials) { - auto *metadata_root = GetTableMetadata(internal, schema, table_result.name, *credentials); - table_result.storage_location = TryGetStrFromObject(metadata_root, "metadata-location"); - auto *metadata = yyjson_obj_get(metadata_root, "metadata"); - //table_result.table_id = TryGetStrFromObject(metadata, "table-uuid"); - - uint64_t current_schema_id = TryGetNumFromObject(metadata, "current-schema-id"); - auto *schemas = yyjson_obj_get(metadata, "schemas"); - duckdb_yyjson::yyjson_val *schema; - size_t schema_idx, schema_max; - bool found = false; - yyjson_arr_foreach(schemas, schema_idx, schema_max, schema) { - uint64_t schema_id = TryGetNumFromObject(schema, "schema-id"); - if (schema_id == current_schema_id) { - found = true; - auto *columns = yyjson_obj_get(schema, "fields"); - duckdb_yyjson::yyjson_val *col; - size_t col_idx, col_max; - yyjson_arr_foreach(columns, col_idx, col_max, col) { - auto column_definition = ParseColumnDefinition(col); - table_result.columns.push_back(column_definition); - } - } - } - - if (!found) { - throw InternalException("Current schema not found"); - } + string result = GetTableMetadata(internal, schema, table_result.name, *credentials); + std::unique_ptr doc(api_result_to_doc(result)); + auto *metadata_root = yyjson_doc_get_root(doc.get()); + populateTableMetadata(table_result, metadata_root); } else { // Skip fetching metadata, we'll do it later when we access the table ICAPIColumnDefinition col; @@ -313,11 +338,12 @@ ICAPITable ICAPI::GetTable( // TODO: handle out-of-order columns using position property vector ICAPI::GetTables(const string &catalog, const string &internal, const string &schema, ICCredentials credentials) { vector result; - auto api_result = GetRequest(credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables", credentials.token); - auto *root = api_result_to_doc(api_result); + string api_result = GetRequest(credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables", credentials.token); + std::unique_ptr doc(api_result_to_doc(api_result)); + auto *root = yyjson_doc_get_root(doc.get()); auto *tables = yyjson_obj_get(root, "identifiers"); size_t idx, max; - duckdb_yyjson::yyjson_val *table; + yyjson_val *table; yyjson_arr_foreach(tables, idx, max, table) { auto table_result = GetTable(catalog, internal, schema, TryGetStrFromObject(table, "name"), std::nullopt); result.push_back(table_result); @@ -328,16 +354,17 @@ vector ICAPI::GetTables(const string &catalog, const string &interna vector ICAPI::GetSchemas(const string &catalog, const string &internal, ICCredentials credentials) { vector result; - auto api_result = + string api_result = GetRequest(credentials.endpoint + "/v1/" + internal + "/namespaces", credentials.token); - auto *root = api_result_to_doc(api_result); + std::unique_ptr doc(api_result_to_doc(api_result)); + auto *root = yyjson_doc_get_root(doc.get()); auto *schemas = yyjson_obj_get(root, "namespaces"); size_t idx, max; - duckdb_yyjson::yyjson_val *schema; + yyjson_val *schema; yyjson_arr_foreach(schemas, idx, max, schema) { ICAPISchema schema_result; schema_result.catalog_name = catalog; - duckdb_yyjson::yyjson_val *value = yyjson_arr_get(schema, 0); + yyjson_val *value = yyjson_arr_get(schema, 0); schema_result.schema_name = yyjson_get_str(value); result.push_back(schema_result); } @@ -363,4 +390,63 @@ void ICAPI::DropSchema(const string &internal, const string &schema, ICCredentia api_result_to_doc(api_result); // if the method returns, request was successful } +void ICAPI::DropTable(const string &catalog, const string &internal, const string &schema, string &table_name, ICCredentials credentials) { + string api_result = DeleteRequest( + credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables/" + table_name + "?purgeRequested=true", + credentials.token); + api_result_to_doc(api_result); // if the method returns, request was successful +} + +static std::string json_to_string(yyjson_mut_doc *doc, yyjson_write_flag flags = YYJSON_WRITE_PRETTY) { + char *json_chars = yyjson_mut_write(doc, flags, NULL); + std::string json_str(json_chars); + free(json_chars); + return json_str; +} + +ICAPITable ICAPI::CreateTable(const string &catalog, const string &internal, const string &schema, ICCredentials credentials, CreateTableInfo *table_info) { + std::unique_ptr dd(yyjson_mut_doc_new(NULL)); + yyjson_mut_val *rr = yyjson_mut_obj(dd.get()); + yyjson_mut_doc_set_root(dd.get(), rr); + yyjson_mut_obj_add_str(dd.get(), rr, "name", table_info->table.c_str()); + + yyjson_mut_val *sch = yyjson_mut_obj(dd.get()); + yyjson_mut_obj_add_val(dd.get(), rr, "schema", sch); + yyjson_mut_obj_add_str(dd.get(), sch, "type", "struct"); + + yyjson_mut_val *fields = yyjson_mut_arr(dd.get()); + yyjson_mut_obj_add_val(dd.get(), sch, "fields", fields); + + std::vector column_names; + std::vector column_types; + for (auto &col : table_info->columns.Logical()) { + // Store column name and type in vectors + column_names.push_back(col.GetName()); + column_types.push_back(ICUtils::LogicalToIcebergType(col.GetType())); + // Add column object to JSON + yyjson_mut_val *col_obj = yyjson_mut_obj(dd.get()); + yyjson_mut_obj_add_int(dd.get(), col_obj, "id", col.Oid()); + yyjson_mut_obj_add_bool(dd.get(), col_obj, "required", true); + yyjson_mut_obj_add_str(dd.get(), col_obj, "name", column_names.back().c_str()); + yyjson_mut_obj_add_str(dd.get(), col_obj, "type", column_types.back().c_str()); + yyjson_mut_arr_add_val(fields, col_obj); + } + + yyjson_mut_val *props = yyjson_mut_obj(dd.get()); + yyjson_mut_obj_add_val(dd.get(), rr, "properties", props); + yyjson_mut_obj_add_str(dd.get(), props, "write.parquet.compression-codec", "snappy"); + + ICAPITable table_result = createTable(catalog, schema, table_info->table); + string post_data = json_to_string(dd.get()); + struct curl_slist *extra_headers = NULL; + extra_headers = curl_slist_append(extra_headers, "X-Iceberg-Access-Delegation: vended-credentials"); + string api_result = PostRequest( + credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables", post_data, "json", credentials.token, extra_headers); + curl_slist_free_all(extra_headers); + std::unique_ptr doc(api_result_to_doc(api_result)); + auto *root = yyjson_doc_get_root(doc.get()); + populateTableMetadata(table_result, root); + return table_result; +} + } // namespace duckdb diff --git a/src/catalog_utils.cpp b/src/catalog_utils.cpp index f71b41e..075392f 100644 --- a/src/catalog_utils.cpp +++ b/src/catalog_utils.cpp @@ -7,6 +7,49 @@ namespace duckdb { +string ICUtils::LogicalToIcebergType(const LogicalType &input) { + switch (input.id()) { + case LogicalType::TINYINT: + case LogicalType::UTINYINT: + return "tinyint"; + case LogicalType::SMALLINT: + case LogicalType::USMALLINT: + return "smallint"; + case LogicalType::INTEGER: + case LogicalType::UINTEGER: + return "int"; + case LogicalType::BIGINT: + case LogicalType::UBIGINT: + return "long"; + case LogicalType::VARCHAR: + return "string"; + case LogicalType::DOUBLE: + return "double"; + case LogicalType::FLOAT: + return "float"; + case LogicalType::BOOLEAN: + return "boolean"; + case LogicalType::TIMESTAMP: + return "timestamp"; + case LogicalType::TIMESTAMP_TZ: + return "timestamptz"; + case LogicalType::BLOB: + return "binary"; + case LogicalType::DATE: + return "date"; + case LogicalTypeId::DECIMAL: + uint8_t precision = DecimalType::GetWidth(input); + uint8_t scale = DecimalType::GetScale(input); + return "decimal(" + std::to_string(precision) + ", " + std::to_string(scale) + ")"; + // case LogicalTypeId::ARRAY: + // case LogicalTypeId::STRUCT: + // case LogicalTypeId::MAP: + // default: + } + + throw std::runtime_error("Unsupported type: " + input.ToString()); +} + string ICUtils::TypeToString(const LogicalType &input) { switch (input.id()) { case LogicalType::VARCHAR: diff --git a/src/include/catalog_api.hpp b/src/include/catalog_api.hpp index b2d0683..18613dd 100644 --- a/src/include/catalog_api.hpp +++ b/src/include/catalog_api.hpp @@ -2,6 +2,7 @@ #pragma once #include "duckdb/common/types.hpp" +#include "duckdb/parser/parsed_data/create_table_info.hpp" namespace duckdb { struct ICCredentials; @@ -22,7 +23,6 @@ struct ICAPITable { string schema_name; string table_type; string data_source_format; - string storage_location; vector columns; @@ -47,11 +47,14 @@ class ICAPI { static ICAPITableCredentials GetTableCredentials(const string &internal, const string &schema, const string &table, ICCredentials credentials); static vector GetCatalogs(const string &catalog, ICCredentials credentials); static vector GetTables(const string &catalog, const string &internal, const string &schema, ICCredentials credentials); - static ICAPITable GetTable(const string &catalog, const string &internal, const string &schema, const string &table, std::optional credentials); + static ICAPITable GetTable(const string &catalog, const string &internal, const string &schema, const string &table_name, std::optional credentials); static vector GetSchemas(const string &catalog, const string &internal, ICCredentials credentials); static vector GetTablesInSchema(const string &catalog, const string &schema, ICCredentials credentials); static string GetToken(string id, string secret, string endpoint); static ICAPISchema CreateSchema(const string &catalog, const string &internal, const string &schema, ICCredentials credentials); static void DropSchema(const string &internal, const string &schema, ICCredentials credentials); + static ICAPITable CreateTable(const string &catalog, const string &internal, const string &schema, ICCredentials credentials, CreateTableInfo *table_info); + static void DropTable(const string &catalog, const string &internal, const string &schema, string &table_name, ICCredentials credentials); }; + } // namespace duckdb diff --git a/src/include/catalog_utils.hpp b/src/include/catalog_utils.hpp index 240e63b..9814272 100644 --- a/src/include/catalog_utils.hpp +++ b/src/include/catalog_utils.hpp @@ -21,6 +21,7 @@ class ICUtils { static LogicalType ToUCType(const LogicalType &input); static LogicalType TypeToLogicalType(ClientContext &context, const string &columnDefinition); static string TypeToString(const LogicalType &input); + static string LogicalToIcebergType(const LogicalType &input); }; } // namespace duckdb diff --git a/src/include/storage/ic_table_set.hpp b/src/include/storage/ic_table_set.hpp index 7e41593..f1746db 100644 --- a/src/include/storage/ic_table_set.hpp +++ b/src/include/storage/ic_table_set.hpp @@ -30,6 +30,7 @@ class ICTableSet : public ICInSchemaSet { static unique_ptr GetTableInfo(ClientContext &context, ICSchemaEntry &schema, const string &table_name); optional_ptr RefreshTable(ClientContext &context, const string &table_name); void AlterTable(ClientContext &context, AlterTableInfo &info); + void DropTable(ClientContext &context, DropInfo &info); protected: void LoadEntries(ClientContext &context) override; diff --git a/src/storage/ic_catalog_set.cpp b/src/storage/ic_catalog_set.cpp index c3b01a1..6297623 100644 --- a/src/storage/ic_catalog_set.cpp +++ b/src/storage/ic_catalog_set.cpp @@ -20,8 +20,6 @@ optional_ptr ICCatalogSet::GetEntry(ClientContext &context, const } void ICCatalogSet::DropEntry(ClientContext &context, DropInfo &info) { - std::cout << "ICCatalogSet::DropEntry" << info.type << std::endl; - EraseEntryInternal(info.name); } diff --git a/src/storage/ic_schema_entry.cpp b/src/storage/ic_schema_entry.cpp index 691c9b0..4046ac7 100644 --- a/src/storage/ic_schema_entry.cpp +++ b/src/storage/ic_schema_entry.cpp @@ -36,6 +36,14 @@ optional_ptr ICSchemaEntry::CreateTable(CatalogTransaction transac return tables.CreateTable(transaction.GetContext(), info); } +void ICSchemaEntry::DropEntry(ClientContext &context, DropInfo &info) { + if (info.type != CatalogType::TABLE_ENTRY) { + throw BinderException("Expecting table entry"); + } + tables.DropTable(context, info); + GetCatalogSet(info.type).DropEntry(context, info); +} + optional_ptr ICSchemaEntry::CreateFunction(CatalogTransaction transaction, CreateFunctionInfo &info) { throw BinderException("PC databases do not support creating functions"); } @@ -136,10 +144,6 @@ void ICSchemaEntry::Scan(CatalogType type, const std::function ICSchemaEntry::GetEntry(CatalogTransaction transaction, CatalogType type, const string &name) { if (!CatalogTypeIsSupported(type)) { diff --git a/src/storage/ic_table_entry.cpp b/src/storage/ic_table_entry.cpp index 18b9501..5bf7511 100644 --- a/src/storage/ic_table_entry.cpp +++ b/src/storage/ic_table_entry.cpp @@ -77,7 +77,6 @@ TableFunction ICTableEntry::GetScanFunction(ClientContext &context, unique_ptrschema_name, table_data->name, ic_catalog.credentials); - // Inject secret into secret manager scoped to this path CreateSecretInfo info(OnCreateConflict::REPLACE_ON_CONFLICT, SecretPersistType::TEMPORARY); info.name = "__internal_ic_" + table_data->table_id; diff --git a/src/storage/ic_table_set.cpp b/src/storage/ic_table_set.cpp index 6866ea9..6e1f284 100644 --- a/src/storage/ic_table_set.cpp +++ b/src/storage/ic_table_set.cpp @@ -9,6 +9,7 @@ #include "duckdb/parser/constraints/unique_constraint.hpp" #include "duckdb/parser/expression/constant_expression.hpp" #include "duckdb/planner/parsed_data/bound_create_table_info.hpp" +#include "duckdb/parser/parsed_data/drop_info.hpp" #include "duckdb/catalog/dependency_list.hpp" #include "duckdb/parser/parsed_data/create_table_info.hpp" #include "duckdb/parser/constraints/list.hpp" @@ -74,11 +75,20 @@ optional_ptr ICTableSet::RefreshTable(ClientContext &context, cons unique_ptr ICTableSet::GetTableInfo(ClientContext &context, ICSchemaEntry &schema, const string &table_name) { - throw NotImplementedException("ICTableSet::CreateTable"); + throw NotImplementedException("ICTableSet::GetTableInfo"); } optional_ptr ICTableSet::CreateTable(ClientContext &context, BoundCreateTableInfo &info) { - throw NotImplementedException("ICTableSet::CreateTable"); + auto &ic_catalog = catalog.Cast(); + auto *table_info = dynamic_cast(info.base.get()); + auto table = ICAPI::CreateTable(catalog.GetName(), ic_catalog.internal_name, schema.name, ic_catalog.credentials, table_info); + auto entry = _CreateCatalogEntry(context, table); + return CreateEntry(std::move(entry)); +} + +void ICTableSet::DropTable(ClientContext &context, DropInfo &info) { + auto &ic_catalog = catalog.Cast(); + ICAPI::DropTable(catalog.GetName(), ic_catalog.internal_name, schema.name, info.name, ic_catalog.credentials); } void ICTableSet::AlterTable(ClientContext &context, RenameTableInfo &info) {