From 1710ff9908aecaa9ed8f113b5ab293a7a00b388b Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Tue, 14 Jan 2025 02:49:51 -0500 Subject: [PATCH 01/26] Merge is done --- CMakeLists.txt | 20 +- src/iceberg_extension.cpp | 147 ++++++++ src/include/storage/uc_catalog.hpp | 87 +++++ src/include/storage/uc_catalog_set.hpp | 54 +++ src/include/storage/uc_schema_entry.hpp | 53 +++ src/include/storage/uc_schema_set.hpp | 28 ++ src/include/storage/uc_table_entry.hpp | 53 +++ src/include/storage/uc_table_set.hpp | 43 +++ src/include/storage/uc_transaction.hpp | 42 +++ .../storage/uc_transaction_manager.hpp | 33 ++ src/include/uc_api.hpp | 61 ++++ src/include/uc_utils.hpp | 33 ++ src/storage/uc_catalog.cpp | 102 ++++++ src/storage/uc_catalog_set.cpp | 69 ++++ src/storage/uc_clear_cache.cpp | 50 +++ src/storage/uc_schema_entry.cpp | 161 +++++++++ src/storage/uc_schema_set.cpp | 38 +++ src/storage/uc_table_entry.cpp | 147 ++++++++ src/storage/uc_table_set.cpp | 87 +++++ src/storage/uc_transaction.cpp | 61 ++++ src/storage/uc_transaction_manager.cpp | 40 +++ src/uc_api.cpp | 320 ++++++++++++++++++ src/uc_utils.cpp | 193 +++++++++++ vcpkg.json | 1 + 24 files changed, 1920 insertions(+), 3 deletions(-) create mode 100644 src/include/storage/uc_catalog.hpp create mode 100644 src/include/storage/uc_catalog_set.hpp create mode 100644 src/include/storage/uc_schema_entry.hpp create mode 100644 src/include/storage/uc_schema_set.hpp create mode 100644 src/include/storage/uc_table_entry.hpp create mode 100644 src/include/storage/uc_table_set.hpp create mode 100644 src/include/storage/uc_transaction.hpp create mode 100644 src/include/storage/uc_transaction_manager.hpp create mode 100644 src/include/uc_api.hpp create mode 100644 src/include/uc_utils.hpp create mode 100644 src/storage/uc_catalog.cpp create mode 100644 src/storage/uc_catalog_set.cpp create mode 100644 src/storage/uc_clear_cache.cpp create mode 100644 src/storage/uc_schema_entry.cpp create mode 100644 src/storage/uc_schema_set.cpp create mode 100644 src/storage/uc_table_entry.cpp create mode 100644 src/storage/uc_table_set.cpp create mode 100644 src/storage/uc_transaction.cpp create mode 100644 src/storage/uc_transaction_manager.cpp create mode 100644 src/uc_api.cpp create mode 100644 src/uc_utils.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index fe79363..db7837f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,12 +13,24 @@ include_directories(src/include) set(EXTENSION_SOURCES src/iceberg_extension.cpp src/iceberg_functions.cpp + src/uc_api.cpp + src/uc_utils.cpp src/common/utils.cpp src/common/schema.cpp src/common/iceberg.cpp src/iceberg_functions/iceberg_snapshots.cpp src/iceberg_functions/iceberg_scan.cpp - src/iceberg_functions/iceberg_metadata.cpp) + src/iceberg_functions/iceberg_metadata.cpp + src/storage/uc_catalog.cpp + src/storage/uc_catalog_set.cpp + src/storage/uc_clear_cache.cpp + src/storage/uc_schema_entry.cpp + src/storage/uc_schema_set.cpp + src/storage/uc_table_entry.cpp + src/storage/uc_table_set.cpp + src/storage/uc_transaction.cpp + src/storage/uc_transaction_manager.cpp + ) add_library(${EXTENSION_NAME} STATIC ${EXTENSION_SOURCES}) @@ -73,10 +85,12 @@ target_link_libraries( Snappy::snappy ZLIB::ZLIB) +find_package(CURL REQUIRED) + # Link dependencies into extension -target_link_libraries(${EXTENSION_NAME} PUBLIC optimized avro_static_release +target_link_libraries(${EXTENSION_NAME} PUBLIC optimized avro_static_release CURL::libcurl debug avro_static_debug) -target_link_libraries(${TARGET_NAME}_loadable_extension optimized +target_link_libraries(${TARGET_NAME}_loadable_extension optimized CURL::libcurl avro_static_release debug avro_static_debug) install( diff --git a/src/iceberg_extension.cpp b/src/iceberg_extension.cpp index f82c6b4..2d4a3b9 100644 --- a/src/iceberg_extension.cpp +++ b/src/iceberg_extension.cpp @@ -1,20 +1,153 @@ #define DUCKDB_EXTENSION_MAIN #include "iceberg_extension.hpp" +#include "storage/uc_catalog.hpp" +#include "storage/uc_transaction_manager.hpp" #include "duckdb.hpp" +#include "duckdb/main/secret/secret_manager.hpp" #include "duckdb/common/exception.hpp" #include "duckdb/common/string_util.hpp" #include "duckdb/function/scalar_function.hpp" +#include "duckdb/main/extension_util.hpp" #include "duckdb/catalog/catalog_entry/macro_catalog_entry.hpp" #include "duckdb/catalog/default/default_functions.hpp" +#include "duckdb/parser/parsed_data/create_scalar_function_info.hpp" +#include "duckdb/parser/parsed_data/attach_info.hpp" +#include "duckdb/storage/storage_extension.hpp" #include "iceberg_functions.hpp" #include "yyjson.hpp" #include "duckdb/main/extension_util.hpp" #include +#include "uc_api.hpp" namespace duckdb { +static unique_ptr CreatePolarisSecretFunction(ClientContext &, CreateSecretInput &input) { + // apply any overridden settings + vector prefix_paths; + auto result = make_uniq(prefix_paths, "iceberg", "config", input.name); + + for (const auto &named_param : input.options) { + auto lower_name = StringUtil::Lower(named_param.first); + + if (lower_name == "client_id" || + lower_name == "client_secret" || + lower_name == "endpoint" || + lower_name == "aws_region") { + result->secret_map[lower_name] = named_param.second.ToString(); + } else { + throw InternalException("Unknown named parameter passed to CreateUCSecretFunction: " + lower_name); + } + } + + // Get token from catalog + result->secret_map["token"] = UCAPI::GetToken( + result->secret_map["client_id"].ToString(), + result->secret_map["client_secret"].ToString(), + result->secret_map["endpoint"].ToString()); + + //! Set redact keys + result->redact_keys = {"token", "client_id", "client_secret"}; + + return std::move(result); +} + +static void SetPolarisSecretParameters(CreateSecretFunction &function) { + function.named_parameters["client_id"] = LogicalType::VARCHAR; + function.named_parameters["client_secret"] = LogicalType::VARCHAR; + function.named_parameters["endpoint"] = LogicalType::VARCHAR; + function.named_parameters["aws_region"] = LogicalType::VARCHAR; + function.named_parameters["token"] = LogicalType::VARCHAR; +} + +unique_ptr GetSecret(ClientContext &context, const string &secret_name) { + auto &secret_manager = SecretManager::Get(context); + auto transaction = CatalogTransaction::GetSystemCatalogTransaction(context); + // FIXME: this should be adjusted once the `GetSecretByName` API supports this + // use case + auto secret_entry = secret_manager.GetSecretByName(transaction, secret_name, "memory"); + if (secret_entry) { + return secret_entry; + } + secret_entry = secret_manager.GetSecretByName(transaction, secret_name, "local_file"); + if (secret_entry) { + return secret_entry; + } + return nullptr; +} + +static unique_ptr PolarisCatalogAttach(StorageExtensionInfo *storage_info, ClientContext &context, + AttachedDatabase &db, const string &name, AttachInfo &info, + AccessMode access_mode) { + UCCredentials credentials; + + // check if we have a secret provided + string secret_name; + for (auto &entry : info.options) { + auto lower_name = StringUtil::Lower(entry.first); + if (lower_name == "type" || lower_name == "read_only") { + // already handled + } else if (lower_name == "secret") { + secret_name = entry.second.ToString(); + } else { + throw BinderException("Unrecognized option for PC attach: %s", entry.first); + } + } + + // if no secret is specified we default to the unnamed mysql secret, if it + // exists + bool explicit_secret = !secret_name.empty(); + if (!explicit_secret) { + // look up settings from the default unnamed mysql secret if none is + // provided + secret_name = "__default_iceberg"; + } + + string connection_string = info.path; + auto secret_entry = GetSecret(context, secret_name); + if (secret_entry) { + // secret found - read data + const auto &kv_secret = dynamic_cast(*secret_entry->secret); + string new_connection_info; + + Value token_val = kv_secret.TryGetValue("token"); + if (token_val.IsNull()) { + throw std::runtime_error("Token is blank"); + } + credentials.token = token_val.ToString(); + + Value endpoint_val = kv_secret.TryGetValue("endpoint"); + credentials.endpoint = endpoint_val.IsNull() ? "" : endpoint_val.ToString(); + StringUtil::RTrim(credentials.endpoint, "/"); + + Value aws_region_val = kv_secret.TryGetValue("aws_region"); + credentials.aws_region = endpoint_val.IsNull() ? "" : aws_region_val.ToString(); + + } else if (explicit_secret) { + // secret not found and one was explicitly provided - throw an error + throw BinderException("Secret with name \"%s\" not found", secret_name); + } + + // TODO: Check catalog with name actually exists! + + return make_uniq(db, info.path, access_mode, credentials); +} + +static unique_ptr CreateTransactionManager(StorageExtensionInfo *storage_info, AttachedDatabase &db, + Catalog &catalog) { + auto &uc_catalog = catalog.Cast(); + return make_uniq(db, uc_catalog); +} + +class UCCatalogStorageExtension : public StorageExtension { +public: + UCCatalogStorageExtension() { + attach = PolarisCatalogAttach; + create_transaction_manager = CreateTransactionManager; + } +}; + static void LoadInternal(DatabaseInstance &instance) { auto &config = DBConfig::GetConfig(instance); @@ -34,6 +167,20 @@ static void LoadInternal(DatabaseInstance &instance) { for (auto &fun : IcebergFunctions::GetScalarFunctions()) { ExtensionUtil::RegisterFunction(instance, fun); } + + UCAPI::InitializeCurl(); + + SecretType secret_type; + secret_type.name = "iceberg"; + secret_type.deserializer = KeyValueSecret::Deserialize; + secret_type.default_provider = "config"; + + ExtensionUtil::RegisterSecretType(instance, secret_type); + CreateSecretFunction secret_function = {"iceberg", "config", CreatePolarisSecretFunction}; + SetPolarisSecretParameters(secret_function); + ExtensionUtil::RegisterFunction(instance, secret_function); + + config.storage_extensions["iceberg"] = make_uniq(); } void IcebergExtension::Load(DuckDB &db) { diff --git a/src/include/storage/uc_catalog.hpp b/src/include/storage/uc_catalog.hpp new file mode 100644 index 0000000..22df4f0 --- /dev/null +++ b/src/include/storage/uc_catalog.hpp @@ -0,0 +1,87 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// storage/uc_catalog.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/function/table_function.hpp" +#include "duckdb/common/enums/access_mode.hpp" +#include "storage/uc_schema_set.hpp" + +namespace duckdb { +class UCSchemaEntry; + +struct UCCredentials { + string endpoint; + string client_id; + string client_secret; + // required to query s3 tables + string aws_region; + // Catalog generates the token using client id & secret + string token; +}; + +class UCClearCacheFunction : public TableFunction { +public: + UCClearCacheFunction(); + + static void ClearCacheOnSetting(ClientContext &context, SetScope scope, Value ¶meter); +}; + +class UCCatalog : public Catalog { +public: + explicit UCCatalog(AttachedDatabase &db_p, const string &internal_name, AccessMode access_mode, + UCCredentials credentials); + ~UCCatalog(); + + string internal_name; + AccessMode access_mode; + UCCredentials credentials; + +public: + void Initialize(bool load_builtin) override; + string GetCatalogType() override { + return "iceberg"; + } + + optional_ptr CreateSchema(CatalogTransaction transaction, CreateSchemaInfo &info) override; + + void ScanSchemas(ClientContext &context, std::function callback) override; + + optional_ptr GetSchema(CatalogTransaction transaction, const string &schema_name, + OnEntryNotFound if_not_found, + QueryErrorContext error_context = QueryErrorContext()) override; + + unique_ptr PlanInsert(ClientContext &context, LogicalInsert &op, + unique_ptr plan) override; + unique_ptr PlanCreateTableAs(ClientContext &context, LogicalCreateTable &op, + unique_ptr plan) override; + unique_ptr PlanDelete(ClientContext &context, LogicalDelete &op, + unique_ptr plan) override; + unique_ptr PlanUpdate(ClientContext &context, LogicalUpdate &op, + unique_ptr plan) override; + unique_ptr BindCreateIndex(Binder &binder, CreateStatement &stmt, TableCatalogEntry &table, + unique_ptr plan) override; + + DatabaseSize GetDatabaseSize(ClientContext &context) override; + + //! Whether or not this is an in-memory PC database + bool InMemory() override; + string GetDBPath() override; + + void ClearCache(); + +private: + void DropSchema(ClientContext &context, DropInfo &info) override; + +private: + UCSchemaSet schemas; + string default_schema; +}; + +} // namespace duckdb diff --git a/src/include/storage/uc_catalog_set.hpp b/src/include/storage/uc_catalog_set.hpp new file mode 100644 index 0000000..8cd9ba3 --- /dev/null +++ b/src/include/storage/uc_catalog_set.hpp @@ -0,0 +1,54 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// storage/uc_catalog_set.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/transaction/transaction.hpp" +#include "duckdb/common/case_insensitive_map.hpp" +#include "duckdb/common/mutex.hpp" + +namespace duckdb { +struct DropInfo; +class UCSchemaEntry; +class UCTransaction; + +class UCCatalogSet { +public: + UCCatalogSet(Catalog &catalog); + + optional_ptr GetEntry(ClientContext &context, const string &name); + virtual void DropEntry(ClientContext &context, DropInfo &info); + void Scan(ClientContext &context, const std::function &callback); + virtual optional_ptr CreateEntry(unique_ptr entry); + void ClearEntries(); + +protected: + virtual void LoadEntries(ClientContext &context) = 0; + + void EraseEntryInternal(const string &name); + +protected: + Catalog &catalog; + +private: + mutex entry_lock; + case_insensitive_map_t> entries; + bool is_loaded; +}; + +class UCInSchemaSet : public UCCatalogSet { +public: + UCInSchemaSet(UCSchemaEntry &schema); + + optional_ptr CreateEntry(unique_ptr entry) override; + +protected: + UCSchemaEntry &schema; +}; + +} // namespace duckdb diff --git a/src/include/storage/uc_schema_entry.hpp b/src/include/storage/uc_schema_entry.hpp new file mode 100644 index 0000000..b9d11f1 --- /dev/null +++ b/src/include/storage/uc_schema_entry.hpp @@ -0,0 +1,53 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// storage/uc_schema_entry.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "uc_api.hpp" +#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" +#include "storage/uc_table_set.hpp" + +namespace duckdb { +class UCTransaction; + +class UCSchemaEntry : public SchemaCatalogEntry { +public: + UCSchemaEntry(Catalog &catalog, CreateSchemaInfo &info); + ~UCSchemaEntry() override; + + unique_ptr schema_data; + +public: + optional_ptr CreateTable(CatalogTransaction transaction, BoundCreateTableInfo &info) override; + optional_ptr CreateFunction(CatalogTransaction transaction, CreateFunctionInfo &info) override; + optional_ptr CreateIndex(CatalogTransaction transaction, CreateIndexInfo &info, + TableCatalogEntry &table) override; + optional_ptr CreateView(CatalogTransaction transaction, CreateViewInfo &info) override; + optional_ptr CreateSequence(CatalogTransaction transaction, CreateSequenceInfo &info) override; + optional_ptr CreateTableFunction(CatalogTransaction transaction, + CreateTableFunctionInfo &info) override; + optional_ptr CreateCopyFunction(CatalogTransaction transaction, + CreateCopyFunctionInfo &info) override; + optional_ptr CreatePragmaFunction(CatalogTransaction transaction, + CreatePragmaFunctionInfo &info) override; + optional_ptr CreateCollation(CatalogTransaction transaction, CreateCollationInfo &info) override; + optional_ptr CreateType(CatalogTransaction transaction, CreateTypeInfo &info) override; + void Alter(CatalogTransaction transaction, AlterInfo &info) override; + void Scan(ClientContext &context, CatalogType type, const std::function &callback) override; + void Scan(CatalogType type, const std::function &callback) override; + void DropEntry(ClientContext &context, DropInfo &info) override; + optional_ptr GetEntry(CatalogTransaction transaction, CatalogType type, const string &name) override; + +private: + UCCatalogSet &GetCatalogSet(CatalogType type); + +private: + UCTableSet tables; +}; + +} // namespace duckdb diff --git a/src/include/storage/uc_schema_set.hpp b/src/include/storage/uc_schema_set.hpp new file mode 100644 index 0000000..38bfb2f --- /dev/null +++ b/src/include/storage/uc_schema_set.hpp @@ -0,0 +1,28 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// storage/uc_schema_set.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "storage/uc_catalog_set.hpp" +#include "storage/uc_schema_entry.hpp" + +namespace duckdb { +struct CreateSchemaInfo; + +class UCSchemaSet : public UCCatalogSet { +public: + explicit UCSchemaSet(Catalog &catalog); + +public: + optional_ptr CreateSchema(ClientContext &context, CreateSchemaInfo &info); + +protected: + void LoadEntries(ClientContext &context) override; +}; + +} // namespace duckdb diff --git a/src/include/storage/uc_table_entry.hpp b/src/include/storage/uc_table_entry.hpp new file mode 100644 index 0000000..50b2528 --- /dev/null +++ b/src/include/storage/uc_table_entry.hpp @@ -0,0 +1,53 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// storage/uc_table_entry.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "uc_api.hpp" +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" +#include "duckdb/parser/parsed_data/create_table_info.hpp" + +namespace duckdb { + +struct UCTableInfo { + UCTableInfo() { + create_info = make_uniq(); + } + UCTableInfo(const string &schema, const string &table) { + create_info = make_uniq(string(), schema, table); + } + UCTableInfo(const SchemaCatalogEntry &schema, const string &table) { + create_info = make_uniq((SchemaCatalogEntry &)schema, table); + } + + const string &GetTableName() const { + return create_info->table; + } + + unique_ptr create_info; +}; + +class UCTableEntry : public TableCatalogEntry { +public: + UCTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTableInfo &info); + UCTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, UCTableInfo &info); + + unique_ptr table_data; + +public: + unique_ptr GetStatistics(ClientContext &context, column_t column_id) override; + + TableFunction GetScanFunction(ClientContext &context, unique_ptr &bind_data) override; + + TableStorageInfo GetStorageInfo(ClientContext &context) override; + + void BindUpdateConstraints(Binder &binder, LogicalGet &get, LogicalProjection &proj, LogicalUpdate &update, + ClientContext &context) override; +}; + +} // namespace duckdb diff --git a/src/include/storage/uc_table_set.hpp b/src/include/storage/uc_table_set.hpp new file mode 100644 index 0000000..fd8ec0b --- /dev/null +++ b/src/include/storage/uc_table_set.hpp @@ -0,0 +1,43 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// storage/uc_table_set.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "storage/uc_catalog_set.hpp" +#include "storage/uc_table_entry.hpp" + +namespace duckdb { +struct CreateTableInfo; +class UCResult; +class UCSchemaEntry; + +class UCTableSet : public UCInSchemaSet { +public: + explicit UCTableSet(UCSchemaEntry &schema); + +public: + optional_ptr CreateTable(ClientContext &context, BoundCreateTableInfo &info); + + static unique_ptr GetTableInfo(ClientContext &context, UCSchemaEntry &schema, + const string &table_name); + optional_ptr RefreshTable(ClientContext &context, const string &table_name); + + void AlterTable(ClientContext &context, AlterTableInfo &info); + +protected: + void LoadEntries(ClientContext &context) override; + + void AlterTable(ClientContext &context, RenameTableInfo &info); + void AlterTable(ClientContext &context, RenameColumnInfo &info); + void AlterTable(ClientContext &context, AddColumnInfo &info); + void AlterTable(ClientContext &context, RemoveColumnInfo &info); + + static void AddColumn(ClientContext &context, UCResult &result, UCTableInfo &table_info, idx_t column_offset = 0); +}; + +} // namespace duckdb diff --git a/src/include/storage/uc_transaction.hpp b/src/include/storage/uc_transaction.hpp new file mode 100644 index 0000000..86a1423 --- /dev/null +++ b/src/include/storage/uc_transaction.hpp @@ -0,0 +1,42 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// storage/uc_transaction.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/transaction/transaction.hpp" + +namespace duckdb { +class UCCatalog; +class UCSchemaEntry; +class UCTableEntry; + +enum class UCTransactionState { TRANSACTION_NOT_YET_STARTED, TRANSACTION_STARTED, TRANSACTION_FINISHED }; + +class UCTransaction : public Transaction { +public: + UCTransaction(UCCatalog &uc_catalog, TransactionManager &manager, ClientContext &context); + ~UCTransaction() override; + + void Start(); + void Commit(); + void Rollback(); + + // UCConnection &GetConnection(); + // unique_ptr Query(const string &query); + static UCTransaction &Get(ClientContext &context, Catalog &catalog); + AccessMode GetAccessMode() const { + return access_mode; + } + +private: + // UCConnection connection; + UCTransactionState transaction_state; + AccessMode access_mode; +}; + +} // namespace duckdb diff --git a/src/include/storage/uc_transaction_manager.hpp b/src/include/storage/uc_transaction_manager.hpp new file mode 100644 index 0000000..b368f51 --- /dev/null +++ b/src/include/storage/uc_transaction_manager.hpp @@ -0,0 +1,33 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// storage/uc_transaction_manager.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/transaction/transaction_manager.hpp" +#include "storage/uc_catalog.hpp" +#include "storage/uc_transaction.hpp" + +namespace duckdb { + +class UCTransactionManager : public TransactionManager { +public: + UCTransactionManager(AttachedDatabase &db_p, UCCatalog &uc_catalog); + + Transaction &StartTransaction(ClientContext &context) override; + ErrorData CommitTransaction(ClientContext &context, Transaction &transaction) override; + void RollbackTransaction(Transaction &transaction) override; + + void Checkpoint(ClientContext &context, bool force = false) override; + +private: + UCCatalog &uc_catalog; + mutex transaction_lock; + reference_map_t> transactions; +}; + +} // namespace duckdb diff --git a/src/include/uc_api.hpp b/src/include/uc_api.hpp new file mode 100644 index 0000000..a65f208 --- /dev/null +++ b/src/include/uc_api.hpp @@ -0,0 +1,61 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// src/include/uc_api.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/common/types.hpp" + +namespace duckdb { +struct UCCredentials; + +struct UCAPIColumnDefinition { + string name; + string type_text; + idx_t precision; + idx_t scale; + idx_t position; +}; + +struct UCAPITable { + string table_id; + + string name; + string catalog_name; + string schema_name; + string table_type; + string data_source_format; + + string storage_location; + + vector columns; +}; + +struct UCAPISchema { + string schema_name; + string catalog_name; +}; + +struct UCAPITableCredentials { + string key_id; + string secret; + string session_token; +}; + +class UCAPI { +public: + //! WARNING: not thread-safe. To be called once on extension initialization + static void InitializeCurl(); + + static UCAPITableCredentials GetTableCredentials(const string &internal, const string &schema, const string &table, UCCredentials credentials); + static vector GetCatalogs(const string &catalog, UCCredentials credentials); + static vector GetTables(const string &catalog, const string &internal, const string &schema, UCCredentials credentials); + static vector GetSchemas(const string &catalog, const string &internal, UCCredentials credentials); + static vector GetTablesInSchema(const string &catalog, const string &schema, UCCredentials credentials); + static string GetToken(string id, string secret, string endpoint); +}; +} // namespace duckdb diff --git a/src/include/uc_utils.hpp b/src/include/uc_utils.hpp new file mode 100644 index 0000000..8927943 --- /dev/null +++ b/src/include/uc_utils.hpp @@ -0,0 +1,33 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// mysql_utils.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb.hpp" +#include "uc_api.hpp" + +namespace duckdb { +class UCSchemaEntry; +class UCTransaction; + +enum class UCTypeAnnotation { STANDARD, CAST_TO_VARCHAR, NUMERIC_AS_DOUBLE, CTID, JSONB, FIXED_LENGTH_CHAR }; + +struct UCType { + idx_t oid = 0; + UCTypeAnnotation info = UCTypeAnnotation::STANDARD; + vector children; +}; + +class UCUtils { +public: + static LogicalType ToUCType(const LogicalType &input); + static LogicalType TypeToLogicalType(ClientContext &context, const string &columnDefinition); + static string TypeToString(const LogicalType &input); +}; + +} // namespace duckdb diff --git a/src/storage/uc_catalog.cpp b/src/storage/uc_catalog.cpp new file mode 100644 index 0000000..3c23771 --- /dev/null +++ b/src/storage/uc_catalog.cpp @@ -0,0 +1,102 @@ +#include "storage/uc_catalog.hpp" +#include "storage/uc_schema_entry.hpp" +#include "storage/uc_transaction.hpp" +#include "duckdb/storage/database_size.hpp" +#include "duckdb/parser/parsed_data/drop_info.hpp" +#include "duckdb/parser/parsed_data/create_schema_info.hpp" +#include "duckdb/main/attached_database.hpp" + +namespace duckdb { + +UCCatalog::UCCatalog(AttachedDatabase &db_p, const string &internal_name, AccessMode access_mode, + UCCredentials credentials) + : Catalog(db_p), internal_name(internal_name), access_mode(access_mode), credentials(std::move(credentials)), + schemas(*this) { +} + +UCCatalog::~UCCatalog() = default; + +void UCCatalog::Initialize(bool load_builtin) { +} + +optional_ptr UCCatalog::CreateSchema(CatalogTransaction transaction, CreateSchemaInfo &info) { + if (info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) { + DropInfo try_drop; + try_drop.type = CatalogType::SCHEMA_ENTRY; + try_drop.name = info.schema; + try_drop.if_not_found = OnEntryNotFound::RETURN_NULL; + try_drop.cascade = false; + schemas.DropEntry(transaction.GetContext(), try_drop); + } + return schemas.CreateSchema(transaction.GetContext(), info); +} + +void UCCatalog::DropSchema(ClientContext &context, DropInfo &info) { + return schemas.DropEntry(context, info); +} + +void UCCatalog::ScanSchemas(ClientContext &context, std::function callback) { + schemas.Scan(context, [&](CatalogEntry &schema) { callback(schema.Cast()); }); +} + +optional_ptr UCCatalog::GetSchema(CatalogTransaction transaction, const string &schema_name, + OnEntryNotFound if_not_found, QueryErrorContext error_context) { + if (schema_name == DEFAULT_SCHEMA) { + if (default_schema.empty()) { + throw InvalidInputException("Attempting to fetch the default schema - but no database was " + "provided in the connection string"); + } + return GetSchema(transaction, default_schema, if_not_found, error_context); + } + auto entry = schemas.GetEntry(transaction.GetContext(), schema_name); + if (!entry && if_not_found != OnEntryNotFound::RETURN_NULL) { + throw BinderException("Schema with name \"%s\" not found", schema_name); + } + return reinterpret_cast(entry.get()); +} + +bool UCCatalog::InMemory() { + return false; +} + +string UCCatalog::GetDBPath() { + return internal_name; +} + + + +DatabaseSize UCCatalog::GetDatabaseSize(ClientContext &context) { + if (default_schema.empty()) { + throw InvalidInputException("Attempting to fetch the database size - but no database was provided " + "in the connection string"); + } + DatabaseSize size; + return size; +} + +void UCCatalog::ClearCache() { + schemas.ClearEntries(); +} + +unique_ptr UCCatalog::PlanInsert(ClientContext &context, LogicalInsert &op, + unique_ptr plan) { + throw NotImplementedException("UCCatalog PlanInsert"); +} +unique_ptr UCCatalog::PlanCreateTableAs(ClientContext &context, LogicalCreateTable &op, + unique_ptr plan) { + throw NotImplementedException("UCCatalog PlanCreateTableAs"); +} +unique_ptr UCCatalog::PlanDelete(ClientContext &context, LogicalDelete &op, + unique_ptr plan) { + throw NotImplementedException("UCCatalog PlanDelete"); +} +unique_ptr UCCatalog::PlanUpdate(ClientContext &context, LogicalUpdate &op, + unique_ptr plan) { + throw NotImplementedException("UCCatalog PlanUpdate"); +} +unique_ptr UCCatalog::BindCreateIndex(Binder &binder, CreateStatement &stmt, TableCatalogEntry &table, + unique_ptr plan) { + throw NotImplementedException("UCCatalog BindCreateIndex"); +} + +} // namespace duckdb diff --git a/src/storage/uc_catalog_set.cpp b/src/storage/uc_catalog_set.cpp new file mode 100644 index 0000000..6451087 --- /dev/null +++ b/src/storage/uc_catalog_set.cpp @@ -0,0 +1,69 @@ +#include "storage/uc_catalog_set.hpp" +#include "storage/uc_transaction.hpp" +#include "duckdb/parser/parsed_data/drop_info.hpp" +#include "storage/uc_schema_entry.hpp" + +namespace duckdb { + +UCCatalogSet::UCCatalogSet(Catalog &catalog) : catalog(catalog), is_loaded(false) { +} + +optional_ptr UCCatalogSet::GetEntry(ClientContext &context, const string &name) { + if (!is_loaded) { + is_loaded = true; + LoadEntries(context); + } + lock_guard l(entry_lock); + auto entry = entries.find(name); + if (entry == entries.end()) { + return nullptr; + } + return entry->second.get(); +} + +void UCCatalogSet::DropEntry(ClientContext &context, DropInfo &info) { + throw NotImplementedException("UCCatalogSet::DropEntry"); +} + +void UCCatalogSet::EraseEntryInternal(const string &name) { + lock_guard l(entry_lock); + entries.erase(name); +} + +void UCCatalogSet::Scan(ClientContext &context, const std::function &callback) { + if (!is_loaded) { + is_loaded = true; + LoadEntries(context); + } + lock_guard l(entry_lock); + for (auto &entry : entries) { + callback(*entry.second); + } +} + +optional_ptr UCCatalogSet::CreateEntry(unique_ptr entry) { + lock_guard l(entry_lock); + auto result = entry.get(); + if (result->name.empty()) { + throw InternalException("UCCatalogSet::CreateEntry called with empty name"); + } + entries.insert(make_pair(result->name, std::move(entry))); + return result; +} + +void UCCatalogSet::ClearEntries() { + entries.clear(); + is_loaded = false; +} + +UCInSchemaSet::UCInSchemaSet(UCSchemaEntry &schema) : UCCatalogSet(schema.ParentCatalog()), schema(schema) { +} + +optional_ptr UCInSchemaSet::CreateEntry(unique_ptr entry) { + if (!entry->internal) { + entry->internal = schema.internal; + } + return UCCatalogSet::CreateEntry(std::move(entry)); +} + +} // namespace duckdb diff --git a/src/storage/uc_clear_cache.cpp b/src/storage/uc_clear_cache.cpp new file mode 100644 index 0000000..2c51abb --- /dev/null +++ b/src/storage/uc_clear_cache.cpp @@ -0,0 +1,50 @@ +#include "duckdb.hpp" + +#include "duckdb/parser/parsed_data/create_table_function_info.hpp" +#include "duckdb/main/database_manager.hpp" +#include "duckdb/main/attached_database.hpp" +#include "storage/uc_catalog.hpp" + +namespace duckdb { + +struct ClearCacheFunctionData : public TableFunctionData { + bool finished = false; +}; + +static unique_ptr ClearCacheBind(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, vector &names) { + + auto result = make_uniq(); + return_types.push_back(LogicalType::BOOLEAN); + names.emplace_back("Success"); + return std::move(result); +} + +static void ClearUCCaches(ClientContext &context) { + auto databases = DatabaseManager::Get(context).GetDatabases(context); + for (auto &db_ref : databases) { + auto &db = db_ref.get(); + auto &catalog = db.GetCatalog(); + if (catalog.GetCatalogType() != "iceberg") { + continue; + } + catalog.Cast().ClearCache(); + } +} + +static void ClearCacheFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { + auto &data = data_p.bind_data->CastNoConst(); + if (data.finished) { + return; + } + ClearUCCaches(context); + data.finished = true; +} + +void UCClearCacheFunction::ClearCacheOnSetting(ClientContext &context, SetScope scope, Value ¶meter) { + ClearUCCaches(context); +} + +UCClearCacheFunction::UCClearCacheFunction() : TableFunction("pc_clear_cache", {}, ClearCacheFunction, ClearCacheBind) { +} +} // namespace duckdb diff --git a/src/storage/uc_schema_entry.cpp b/src/storage/uc_schema_entry.cpp new file mode 100644 index 0000000..7bc93ce --- /dev/null +++ b/src/storage/uc_schema_entry.cpp @@ -0,0 +1,161 @@ +#include "storage/uc_schema_entry.hpp" +#include "storage/uc_table_entry.hpp" +#include "storage/uc_transaction.hpp" +#include "duckdb/parser/parsed_data/create_view_info.hpp" +#include "duckdb/parser/parsed_data/create_index_info.hpp" +#include "duckdb/planner/parsed_data/bound_create_table_info.hpp" +#include "duckdb/parser/parsed_data/drop_info.hpp" +#include "duckdb/parser/constraints/list.hpp" +#include "duckdb/common/unordered_set.hpp" +#include "duckdb/parser/parsed_data/alter_info.hpp" +#include "duckdb/parser/parsed_data/alter_table_info.hpp" +#include "duckdb/parser/parsed_expression_iterator.hpp" + +namespace duckdb { + +UCSchemaEntry::UCSchemaEntry(Catalog &catalog, CreateSchemaInfo &info) + : SchemaCatalogEntry(catalog, info), tables(*this) { +} + +UCSchemaEntry::~UCSchemaEntry() { +} + +UCTransaction &GetUCTransaction(CatalogTransaction transaction) { + if (!transaction.transaction) { + throw InternalException("No transaction!?"); + } + return transaction.transaction->Cast(); +} + +optional_ptr UCSchemaEntry::CreateTable(CatalogTransaction transaction, BoundCreateTableInfo &info) { + auto &base_info = info.Base(); + auto table_name = base_info.table; + if (base_info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) { + throw NotImplementedException("REPLACE ON CONFLICT in CreateTable"); + } + return tables.CreateTable(transaction.GetContext(), info); +} + +optional_ptr UCSchemaEntry::CreateFunction(CatalogTransaction transaction, CreateFunctionInfo &info) { + throw BinderException("PC databases do not support creating functions"); +} + +void UCUnqualifyColumnRef(ParsedExpression &expr) { + if (expr.type == ExpressionType::COLUMN_REF) { + auto &colref = expr.Cast(); + auto name = std::move(colref.column_names.back()); + colref.column_names = {std::move(name)}; + return; + } + ParsedExpressionIterator::EnumerateChildren(expr, UCUnqualifyColumnRef); +} + +optional_ptr UCSchemaEntry::CreateIndex(CatalogTransaction transaction, CreateIndexInfo &info, + TableCatalogEntry &table) { + throw NotImplementedException("CreateIndex"); +} + +string GetUCCreateView(CreateViewInfo &info) { + throw NotImplementedException("GetCreateView"); +} + +optional_ptr UCSchemaEntry::CreateView(CatalogTransaction transaction, CreateViewInfo &info) { + if (info.sql.empty()) { + throw BinderException("Cannot create view in PC that originated from an " + "empty SQL statement"); + } + if (info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT || + info.on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT) { + auto current_entry = GetEntry(transaction, CatalogType::VIEW_ENTRY, info.view_name); + if (current_entry) { + if (info.on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT) { + return current_entry; + } + throw NotImplementedException("REPLACE ON CONFLICT in CreateView"); + } + } + auto &uc_transaction = GetUCTransaction(transaction); + // uc_transaction.Query(GetUCCreateView(info)); + return tables.RefreshTable(transaction.GetContext(), info.view_name); +} + +optional_ptr UCSchemaEntry::CreateType(CatalogTransaction transaction, CreateTypeInfo &info) { + throw BinderException("PC databases do not support creating types"); +} + +optional_ptr UCSchemaEntry::CreateSequence(CatalogTransaction transaction, CreateSequenceInfo &info) { + throw BinderException("PC databases do not support creating sequences"); +} + +optional_ptr UCSchemaEntry::CreateTableFunction(CatalogTransaction transaction, + CreateTableFunctionInfo &info) { + throw BinderException("PC databases do not support creating table functions"); +} + +optional_ptr UCSchemaEntry::CreateCopyFunction(CatalogTransaction transaction, + CreateCopyFunctionInfo &info) { + throw BinderException("PC databases do not support creating copy functions"); +} + +optional_ptr UCSchemaEntry::CreatePragmaFunction(CatalogTransaction transaction, + CreatePragmaFunctionInfo &info) { + throw BinderException("PC databases do not support creating pragma functions"); +} + +optional_ptr UCSchemaEntry::CreateCollation(CatalogTransaction transaction, CreateCollationInfo &info) { + throw BinderException("PC databases do not support creating collations"); +} + +void UCSchemaEntry::Alter(CatalogTransaction transaction, AlterInfo &info) { + if (info.type != AlterType::ALTER_TABLE) { + throw BinderException("Only altering tables is supported for now"); + } + auto &alter = info.Cast(); + tables.AlterTable(transaction.GetContext(), alter); +} + +bool CatalogTypeIsSupported(CatalogType type) { + switch (type) { + case CatalogType::INDEX_ENTRY: + case CatalogType::TABLE_ENTRY: + case CatalogType::VIEW_ENTRY: + return true; + default: + return false; + } +} + +void UCSchemaEntry::Scan(ClientContext &context, CatalogType type, + const std::function &callback) { + if (!CatalogTypeIsSupported(type)) { + return; + } + GetCatalogSet(type).Scan(context, callback); +} +void UCSchemaEntry::Scan(CatalogType type, const std::function &callback) { + throw NotImplementedException("Scan without context not supported"); +} + +void UCSchemaEntry::DropEntry(ClientContext &context, DropInfo &info) { + GetCatalogSet(info.type).DropEntry(context, info); +} + +optional_ptr UCSchemaEntry::GetEntry(CatalogTransaction transaction, CatalogType type, + const string &name) { + if (!CatalogTypeIsSupported(type)) { + return nullptr; + } + return GetCatalogSet(type).GetEntry(transaction.GetContext(), name); +} + +UCCatalogSet &UCSchemaEntry::GetCatalogSet(CatalogType type) { + switch (type) { + case CatalogType::TABLE_ENTRY: + case CatalogType::VIEW_ENTRY: + return tables; + default: + throw InternalException("Type not supported for GetCatalogSet"); + } +} + +} // namespace duckdb diff --git a/src/storage/uc_schema_set.cpp b/src/storage/uc_schema_set.cpp new file mode 100644 index 0000000..6efb318 --- /dev/null +++ b/src/storage/uc_schema_set.cpp @@ -0,0 +1,38 @@ +#include "storage/uc_schema_set.hpp" +#include "storage/uc_catalog.hpp" +#include "uc_api.hpp" +#include "storage/uc_transaction.hpp" +#include "duckdb/parser/parsed_data/create_schema_info.hpp" +#include "duckdb/catalog/catalog.hpp" + +namespace duckdb { + +UCSchemaSet::UCSchemaSet(Catalog &catalog) : UCCatalogSet(catalog) { +} + +static bool IsInternalTable(const string &catalog, const string &schema) { + if (schema == "information_schema") { + return true; + } + return false; +} +void UCSchemaSet::LoadEntries(ClientContext &context) { + + auto &uc_catalog = catalog.Cast(); + auto tables = UCAPI::GetSchemas(catalog.GetName(), uc_catalog.internal_name, uc_catalog.credentials); + + for (const auto &schema : tables) { + CreateSchemaInfo info; + info.schema = schema.schema_name; + info.internal = IsInternalTable(schema.catalog_name, schema.schema_name); + auto schema_entry = make_uniq(catalog, info); + schema_entry->schema_data = make_uniq(schema); + CreateEntry(std::move(schema_entry)); + } +} + +optional_ptr UCSchemaSet::CreateSchema(ClientContext &context, CreateSchemaInfo &info) { + throw NotImplementedException("Schema creation"); +} + +} // namespace duckdb diff --git a/src/storage/uc_table_entry.cpp b/src/storage/uc_table_entry.cpp new file mode 100644 index 0000000..7691a2f --- /dev/null +++ b/src/storage/uc_table_entry.cpp @@ -0,0 +1,147 @@ +#include "storage/uc_catalog.hpp" +#include "storage/uc_schema_entry.hpp" +#include "storage/uc_table_entry.hpp" +#include "storage/uc_transaction.hpp" +#include "duckdb/storage/statistics/base_statistics.hpp" +#include "duckdb/storage/table_storage_info.hpp" +#include "duckdb/main/extension_util.hpp" +#include "duckdb/main/database.hpp" +#include "duckdb/main/secret/secret_manager.hpp" +#include "duckdb/catalog/catalog_entry/table_function_catalog_entry.hpp" +#include "duckdb/parser/tableref/table_function_ref.hpp" +#include "uc_api.hpp" +#include "../../duckdb/third_party/catch/catch.hpp" +#include "duckdb/planner/binder.hpp" +#include "duckdb/planner/tableref/bound_table_function.hpp" +#include "duckdb/planner/logical_operator.hpp" +#include "duckdb/planner/operator/logical_get.hpp" + + +namespace duckdb { + +UCTableEntry::UCTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTableInfo &info) + : TableCatalogEntry(catalog, schema, info) { + this->internal = false; +} + +UCTableEntry::UCTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, UCTableInfo &info) + : TableCatalogEntry(catalog, schema, *info.create_info) { + this->internal = false; +} + +unique_ptr UCTableEntry::GetStatistics(ClientContext &context, column_t column_id) { + return nullptr; +} + +void UCTableEntry::BindUpdateConstraints(Binder &binder, LogicalGet &, LogicalProjection &, LogicalUpdate &, + ClientContext &) { + throw NotImplementedException("BindUpdateConstraints"); +} + +struct MyIcebergFunctionData : public FunctionData { + std::string path; // store the path or any other relevant info here + + // Optional: implement Copy for caching/pushdown logic + unique_ptr Copy() const override { + auto copy = make_uniq(); + copy->path = path; + return copy; + } + + // Optional: implement Equals for caching + bool Equals(const FunctionData &other_p) const override { + auto &other = (const MyIcebergFunctionData &)other_p; + return path == other.path; + } +}; + +TableFunction UCTableEntry::GetScanFunction(ClientContext &context, unique_ptr &bind_data) { + auto &db = DatabaseInstance::GetDatabase(context); + auto &uc_catalog = catalog.Cast(); + + auto &parquet_function_set = ExtensionUtil::GetTableFunction(db, "parquet_scan"); + auto parquet_scan_function = parquet_function_set.functions.GetFunctionByArguments(context, {LogicalType::VARCHAR}); + + auto &iceberg_function_set = ExtensionUtil::GetTableFunction(db, "iceberg_scan"); + auto iceberg_scan_function = iceberg_function_set.functions.GetFunctionByArguments(context, {LogicalType::VARCHAR}); + + D_ASSERT(table_data); + + if (table_data->data_source_format != "ICEBERG") { + throw NotImplementedException("Table '%s' is of unsupported format '%s', ", table_data->name, + table_data->data_source_format); + } + + if (table_data->storage_location.find("file://") != 0) { + auto &secret_manager = SecretManager::Get(context); + // Get Credentials from UCAPI + auto table_credentials = UCAPI::GetTableCredentials( + uc_catalog.internal_name, table_data->schema_name, table_data->name, uc_catalog.credentials); + + // Inject secret into secret manager scoped to this path + CreateSecretInfo info(OnCreateConflict::REPLACE_ON_CONFLICT, SecretPersistType::TEMPORARY); + info.name = "__internal_uc_" + table_data->table_id; + info.type = "s3"; + info.provider = "config"; + info.storage_type = "memory"; + info.options = { + {"key_id", table_credentials.key_id}, + {"secret", table_credentials.secret}, + {"session_token", table_credentials.session_token}, + {"region", uc_catalog.credentials.aws_region}, + }; + + std::string lc_storage_location; + lc_storage_location.resize(table_data->storage_location.size()); + std::transform(table_data->storage_location.begin(), table_data->storage_location.end(), lc_storage_location.begin(), ::tolower); + size_t metadata_pos = lc_storage_location.find("metadata"); + if (metadata_pos != std::string::npos) { + info.scope = {lc_storage_location.substr(0, metadata_pos)}; + } else { + throw std::runtime_error("Substring not found"); + } + + auto my_secret = secret_manager.CreateSecret(context, info); + } + + named_parameter_map_t param_map; + vector return_types; + vector names; + TableFunctionRef empty_ref; + + // Set the S3 path as input to table function + vector inputs = {table_data->storage_location}; + + TableFunctionBindInput bind_input(inputs, param_map, return_types, names, nullptr, nullptr, + iceberg_scan_function, empty_ref); + + auto table_ref = iceberg_scan_function.bind_replace(context, bind_input); + + // 1) Create a Binder and bind the parser-level TableRef -> BoundTableRef + auto binder = Binder::CreateBinder(context); + auto bound_ref = binder->Bind(*table_ref); + + // 2) Create a logical plan from the bound reference + unique_ptr logical_plan = binder->CreatePlan(*bound_ref); + + // 3) Recursively search the logical plan for a LogicalGet node + // For a single table function, you often have just one operator: LogicalGet + LogicalOperator *op = logical_plan.get(); + if (op->type != LogicalOperatorType::LOGICAL_GET) { + throw std::runtime_error("Expected a LogicalGet, but got something else!"); + } + + // 4) Access the bind_data inside LogicalGet + auto &get = (LogicalGet &)*op; + bind_data = std::move(get.bind_data); + + return parquet_scan_function; +} + +TableStorageInfo UCTableEntry::GetStorageInfo(ClientContext &context) { + TableStorageInfo result; + // TODO fill info + return result; +} + +} // namespace duckdb diff --git a/src/storage/uc_table_set.cpp b/src/storage/uc_table_set.cpp new file mode 100644 index 0000000..494060c --- /dev/null +++ b/src/storage/uc_table_set.cpp @@ -0,0 +1,87 @@ +#include "uc_api.hpp" +#include "uc_utils.hpp" + +#include "storage/uc_catalog.hpp" +#include "storage/uc_table_set.hpp" +#include "storage/uc_transaction.hpp" +#include "duckdb/parser/parsed_data/create_table_info.hpp" +#include "duckdb/parser/constraints/not_null_constraint.hpp" +#include "duckdb/parser/constraints/unique_constraint.hpp" +#include "duckdb/parser/expression/constant_expression.hpp" +#include "duckdb/planner/parsed_data/bound_create_table_info.hpp" +#include "duckdb/catalog/dependency_list.hpp" +#include "duckdb/parser/parsed_data/create_table_info.hpp" +#include "duckdb/parser/constraints/list.hpp" +#include "storage/uc_schema_entry.hpp" +#include "duckdb/parser/parser.hpp" + +namespace duckdb { + +UCTableSet::UCTableSet(UCSchemaEntry &schema) : UCInSchemaSet(schema) { +} + +static ColumnDefinition CreateColumnDefinition(ClientContext &context, UCAPIColumnDefinition &coldef) { + return {coldef.name, UCUtils::TypeToLogicalType(context, coldef.type_text)}; +} + +void UCTableSet::LoadEntries(ClientContext &context) { + auto &transaction = UCTransaction::Get(context, catalog); + + auto &uc_catalog = catalog.Cast(); + + // TODO: handle out-of-order columns using position property + auto tables = UCAPI::GetTables(catalog.GetName(), catalog.GetDBPath(), schema.name, uc_catalog.credentials); + + for (auto &table : tables) { + D_ASSERT(schema.name == table.schema_name); + CreateTableInfo info; + for (auto &col : table.columns) { + info.columns.AddColumn(CreateColumnDefinition(context, col)); + } + + info.table = table.name; + auto table_entry = make_uniq(catalog, schema, info); + table_entry->table_data = make_uniq(table); + + CreateEntry(std::move(table_entry)); + } +} + +optional_ptr UCTableSet::RefreshTable(ClientContext &context, const string &table_name) { + auto table_info = GetTableInfo(context, schema, table_name); + auto table_entry = make_uniq(catalog, schema, *table_info); + auto table_ptr = table_entry.get(); + CreateEntry(std::move(table_entry)); + return table_ptr; +} + +unique_ptr UCTableSet::GetTableInfo(ClientContext &context, UCSchemaEntry &schema, + const string &table_name) { + throw NotImplementedException("UCTableSet::CreateTable"); +} + +optional_ptr UCTableSet::CreateTable(ClientContext &context, BoundCreateTableInfo &info) { + throw NotImplementedException("UCTableSet::CreateTable"); +} + +void UCTableSet::AlterTable(ClientContext &context, RenameTableInfo &info) { + throw NotImplementedException("UCTableSet::AlterTable"); +} + +void UCTableSet::AlterTable(ClientContext &context, RenameColumnInfo &info) { + throw NotImplementedException("UCTableSet::AlterTable"); +} + +void UCTableSet::AlterTable(ClientContext &context, AddColumnInfo &info) { + throw NotImplementedException("UCTableSet::AlterTable"); +} + +void UCTableSet::AlterTable(ClientContext &context, RemoveColumnInfo &info) { + throw NotImplementedException("UCTableSet::AlterTable"); +} + +void UCTableSet::AlterTable(ClientContext &context, AlterTableInfo &alter) { + throw NotImplementedException("UCTableSet::AlterTable"); +} + +} // namespace duckdb diff --git a/src/storage/uc_transaction.cpp b/src/storage/uc_transaction.cpp new file mode 100644 index 0000000..2942458 --- /dev/null +++ b/src/storage/uc_transaction.cpp @@ -0,0 +1,61 @@ +#include "storage/uc_transaction.hpp" +#include "storage/uc_catalog.hpp" +#include "duckdb/parser/parsed_data/create_view_info.hpp" +#include "duckdb/catalog/catalog_entry/index_catalog_entry.hpp" +#include "duckdb/catalog/catalog_entry/view_catalog_entry.hpp" + +namespace duckdb { + +UCTransaction::UCTransaction(UCCatalog &uc_catalog, TransactionManager &manager, ClientContext &context) + : Transaction(manager, context), access_mode(uc_catalog.access_mode) { + // connection = UCConnection::Open(uc_catalog.path); +} + +UCTransaction::~UCTransaction() = default; + +void UCTransaction::Start() { + transaction_state = UCTransactionState::TRANSACTION_NOT_YET_STARTED; +} +void UCTransaction::Commit() { + if (transaction_state == UCTransactionState::TRANSACTION_STARTED) { + transaction_state = UCTransactionState::TRANSACTION_FINISHED; + // connection.Execute("COMMIT"); + } +} +void UCTransaction::Rollback() { + if (transaction_state == UCTransactionState::TRANSACTION_STARTED) { + transaction_state = UCTransactionState::TRANSACTION_FINISHED; + // connection.Execute("ROLLBACK"); + } +} + +// UCConnection &UCTransaction::GetConnection() { +// if (transaction_state == UCTransactionState::TRANSACTION_NOT_YET_STARTED) { +// transaction_state = UCTransactionState::TRANSACTION_STARTED; +// string query = "START TRANSACTION"; +// if (access_mode == AccessMode::READ_ONLY) { +// query += " READ ONLY"; +// } +//// conne/**/ction.Execute(query); +// } +// return connection; +//} + +// unique_ptr UCTransaction::Query(const string &query) { +// if (transaction_state == UCTransactionState::TRANSACTION_NOT_YET_STARTED) { +// transaction_state = UCTransactionState::TRANSACTION_STARTED; +// string transaction_start = "START TRANSACTION"; +// if (access_mode == AccessMode::READ_ONLY) { +// transaction_start += " READ ONLY"; +// } +// connection.Query(transaction_start); +// return connection.Query(query); +// } +// return connection.Query(query); +//} + +UCTransaction &UCTransaction::Get(ClientContext &context, Catalog &catalog) { + return Transaction::Get(context, catalog).Cast(); +} + +} // namespace duckdb diff --git a/src/storage/uc_transaction_manager.cpp b/src/storage/uc_transaction_manager.cpp new file mode 100644 index 0000000..afd7e07 --- /dev/null +++ b/src/storage/uc_transaction_manager.cpp @@ -0,0 +1,40 @@ +#include "storage/uc_transaction_manager.hpp" +#include "duckdb/main/attached_database.hpp" + +namespace duckdb { + +UCTransactionManager::UCTransactionManager(AttachedDatabase &db_p, UCCatalog &uc_catalog) + : TransactionManager(db_p), uc_catalog(uc_catalog) { +} + +Transaction &UCTransactionManager::StartTransaction(ClientContext &context) { + auto transaction = make_uniq(uc_catalog, *this, context); + transaction->Start(); + auto &result = *transaction; + lock_guard l(transaction_lock); + transactions[result] = std::move(transaction); + return result; +} + +ErrorData UCTransactionManager::CommitTransaction(ClientContext &context, Transaction &transaction) { + auto &uc_transaction = transaction.Cast(); + uc_transaction.Commit(); + lock_guard l(transaction_lock); + transactions.erase(transaction); + return ErrorData(); +} + +void UCTransactionManager::RollbackTransaction(Transaction &transaction) { + auto &uc_transaction = transaction.Cast(); + uc_transaction.Rollback(); + lock_guard l(transaction_lock); + transactions.erase(transaction); +} + +void UCTransactionManager::Checkpoint(ClientContext &context, bool force) { + auto &transaction = UCTransaction::Get(context, db.GetCatalog()); + // auto &db = transaction.GetConnection(); + // db.Execute("CHECKPOINT"); +} + +} // namespace duckdb diff --git a/src/uc_api.cpp b/src/uc_api.cpp new file mode 100644 index 0000000..6a6e8d7 --- /dev/null +++ b/src/uc_api.cpp @@ -0,0 +1,320 @@ +#include "uc_api.hpp" +#include "storage/uc_catalog.hpp" +#include "yyjson.hpp" +#include +#include + +#include + +namespace duckdb { + +//! We use a global here to store the path that is selected on the UCAPI::InitializeCurl call +static string SELECTED_CURL_CERT_PATH = ""; + +static size_t GetRequestWriteCallback(void *contents, size_t size, size_t nmemb, void *userp) { + ((std::string *)userp)->append((char *)contents, size * nmemb); + return size * nmemb; +} + +// we statically compile in libcurl, which means the cert file location of the build machine is the +// place curl will look. But not every distro has this file in the same location, so we search a +// number of common locations and use the first one we find. +static string certFileLocations[] = { + // Arch, Debian-based, Gentoo + "/etc/ssl/certs/ca-certificates.crt", + // RedHat 7 based + "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", + // Redhat 6 based + "/etc/pki/tls/certs/ca-bundle.crt", + // OpenSUSE + "/etc/ssl/ca-bundle.pem", + // Alpine + "/etc/ssl/cert.pem" +}; + +// Look through the the above locations and if one of the files exists, set that as the location curl should use. +static bool SelectCurlCertPath() { + for (string& caFile : certFileLocations) { + struct stat buf; + if (stat(caFile.c_str(), &buf) == 0) { + SELECTED_CURL_CERT_PATH = caFile; + } + } + return false; +} + +static bool SetCurlCAFileInfo(CURL* curl) { + if (!SELECTED_CURL_CERT_PATH.empty()) { + curl_easy_setopt(curl, CURLOPT_CAINFO, SELECTED_CURL_CERT_PATH.c_str()); + return true; + } + return false; +} + +// Note: every curl object we use should set this, because without it some linux distro's may not find the CA certificate. +static void InitializeCurlObject(CURL * curl, const string &token) { + if (!token.empty()) { + curl_easy_setopt(curl, CURLOPT_XOAUTH2_BEARER, token.c_str()); + curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_BEARER); + } + SetCurlCAFileInfo(curl); +} + +static string GetRequest(const string &url, const string &token = "", curl_slist *extra_headers = NULL) { + CURL *curl; + CURLcode res; + string readBuffer; + + curl = curl_easy_init(); + if (curl) { + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, GetRequestWriteCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer); + if(extra_headers) { + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, extra_headers); + } + InitializeCurlObject(curl, token); + res = curl_easy_perform(curl); + curl_easy_cleanup(curl); + + if (res != CURLcode::CURLE_OK) { + string error = curl_easy_strerror(res); + throw IOException("Curl Request to '%s' failed with error: '%s'", url, error); + } + return readBuffer; + } + throw InternalException("Failed to initialize curl"); +} + +static string PostRequest(const string &url, const string &post_data, curl_slist *extra_headers = NULL) { + string readBuffer; + CURL *curl = curl_easy_init(); + if (!curl) { + throw InternalException("Failed to initialize curl"); + } + + // Set the URL + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + + // Set POST method + curl_easy_setopt(curl, CURLOPT_POST, 1L); + + // Set POST data + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, post_data.c_str()); + + // Set up response handling + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, GetRequestWriteCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer); + + // Create default headers for content type + struct curl_slist *headers = NULL; + headers = curl_slist_append(headers, "Content-Type: application/x-www-form-urlencoded"); + + // Append any extra headers + if (extra_headers) { + struct curl_slist *temp = extra_headers; + while (temp) { + headers = curl_slist_append(headers, temp->data); + temp = temp->next; + } + } + + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + InitializeCurlObject(curl, ""); + + // Perform the request + CURLcode res = curl_easy_perform(curl); + + // Clean up + curl_slist_free_all(headers); + curl_easy_cleanup(curl); + + if (res != CURLcode::CURLE_OK) { + string error = curl_easy_strerror(res); + throw IOException("Curl Request to '%s' failed with error: '%s'", url, error); + } + return readBuffer; + +} + +static duckdb_yyjson::yyjson_val *GetTableMetadata(const string &internal, const string &schema, const string &table, UCCredentials credentials) { + struct curl_slist *extra_headers = NULL; + extra_headers = curl_slist_append(extra_headers, "X-Iceberg-Access-Delegation: vended-credentials"); + auto api_result = GetRequest( + credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables/" + table, + credentials.token, + extra_headers); + duckdb_yyjson::yyjson_doc *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); + return yyjson_doc_get_root(doc); +} + +template +static TYPE TemplatedTryGetYYJson(duckdb_yyjson::yyjson_val *obj, const string &field, TYPE default_val, + bool fail_on_missing = true) { + auto val = yyjson_obj_get(obj, field.c_str()); + if (val && yyjson_get_type(val) == TYPE_NUM) { + return get_function(val); + } else if (!fail_on_missing) { + return default_val; + } + throw IOException("Invalid field found while parsing field: " + field); +} + +static uint64_t TryGetNumFromObject(duckdb_yyjson::yyjson_val *obj, const string &field, bool fail_on_missing = true, + uint64_t default_val = 0) { + return TemplatedTryGetYYJson(obj, field, default_val, + fail_on_missing); +} +static bool TryGetBoolFromObject(duckdb_yyjson::yyjson_val *obj, const string &field, bool fail_on_missing = false, + bool default_val = false) { + return TemplatedTryGetYYJson(obj, field, default_val, + fail_on_missing); +} +static string TryGetStrFromObject(duckdb_yyjson::yyjson_val *obj, const string &field, bool fail_on_missing = true, + const char *default_val = "") { + return TemplatedTryGetYYJson(obj, field, default_val, + fail_on_missing); +} + +void UCAPI::InitializeCurl() { + SelectCurlCertPath(); +} + +vector UCAPI::GetCatalogs(const string &catalog, UCCredentials credentials) { + throw NotImplementedException("UCAPI::GetCatalogs"); +} + +static UCAPIColumnDefinition ParseColumnDefinition(duckdb_yyjson::yyjson_val *column_def) { + UCAPIColumnDefinition result; + + result.name = TryGetStrFromObject(column_def, "name"); + result.type_text = TryGetStrFromObject(column_def, "type"); + result.precision = -1; //TODO: TryGetNumFromObject(column_def, "type_precision"); + result.scale = -1; //TODO: TryGetNumFromObject(column_def, "type_scale"); + result.position = TryGetNumFromObject(column_def, "id") - 1; + + return result; +} + +UCAPITableCredentials UCAPI::GetTableCredentials(const string &internal, const string &schema, const string &table, UCCredentials credentials) { + UCAPITableCredentials result; + + duckdb_yyjson::yyjson_val *root = GetTableMetadata(internal, schema, table, credentials); + auto *aws_temp_credentials = yyjson_obj_get(root, "config"); + + if (aws_temp_credentials) { + result.key_id = TryGetStrFromObject(aws_temp_credentials, "s3.access-key-id"); + result.secret = TryGetStrFromObject(aws_temp_credentials, "s3.secret-access-key"); + result.session_token = TryGetStrFromObject(aws_temp_credentials, "s3.session-token"); + } + + return result; +} + +string UCAPI::GetToken(string id, string secret, string endpoint) { + string post_data = "grant_type=client_credentials&client_id=" + id + "&client_secret=" + secret + "&scope=PRINCIPAL_ROLE:ALL"; + string api_result = PostRequest(endpoint + "/v1/oauth/tokens", post_data); + + // Read JSON and get root + auto *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); + auto *root = yyjson_doc_get_root(doc); + + auto *error = yyjson_obj_get(root, "error"); + if (error != NULL) { + string err_msg = TryGetStrFromObject(error, "message"); + throw std::runtime_error(err_msg); + } + + return TryGetStrFromObject(root, "access_token"); +} + +vector UCAPI::GetTables(const string &catalog, const string &internal, const string &schema, UCCredentials credentials) { + vector result; + + auto api_result = GetRequest(credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables", credentials.token); + + // Read JSON and get root + auto *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); + auto *root = yyjson_doc_get_root(doc); + + // Get root["hits"], iterate over the array + auto *tables = yyjson_obj_get(root, "identifiers"); + size_t idx, max; + duckdb_yyjson::yyjson_val *table; + yyjson_arr_foreach(tables, idx, max, table) { + UCAPITable table_result; + table_result.catalog_name = catalog; + table_result.schema_name = schema; + table_result.name = TryGetStrFromObject(table, "name"); + + auto *metadata_root = GetTableMetadata(internal, schema, table_result.name, credentials); + table_result.data_source_format = "ICEBERG"; + table_result.storage_location = TryGetStrFromObject(metadata_root, "metadata-location"); + auto *metadata = yyjson_obj_get(metadata_root, "metadata"); + table_result.table_id = TryGetStrFromObject(metadata, "table-uuid"); + + uint64_t current_schema_id = TryGetNumFromObject(metadata, "current-schema-id"); + auto *schemas = yyjson_obj_get(metadata, "schemas"); + duckdb_yyjson::yyjson_val *schema; + size_t schema_idx, schema_max; + bool found = false; + yyjson_arr_foreach(schemas, schema_idx, schema_max, schema) { + uint64_t schema_id = TryGetNumFromObject(schema, "schema-id"); + if (schema_id == current_schema_id) { + found = true; + auto *columns = yyjson_obj_get(schema, "fields"); + duckdb_yyjson::yyjson_val *col; + size_t col_idx, col_max; + yyjson_arr_foreach(columns, col_idx, col_max, col) { + auto column_definition = ParseColumnDefinition(col); + table_result.columns.push_back(column_definition); + } + result.push_back(table_result); + + // This could take a while so display found tables to indicate progress + // TODO: Is there a way to show progress some other way? + std::cout << table_result.schema_name << " | " << table_result.name << std::endl; + } + } + + if (!found) { + throw InternalException("Current schema not found"); + } + } + + return result; +} + +vector UCAPI::GetSchemas(const string &catalog, const string &internal, UCCredentials credentials) { + vector result; + + auto api_result = + GetRequest(credentials.endpoint + "/v1/" + internal + "/namespaces", credentials.token); + + // Read JSON and get root + duckdb_yyjson::yyjson_doc *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); + duckdb_yyjson::yyjson_val *root = yyjson_doc_get_root(doc); + + auto *error = yyjson_obj_get(root, "error"); + if (error != NULL) { + string err_msg = TryGetStrFromObject(error, "message"); + throw std::runtime_error(err_msg); + } + + // Get root["hits"], iterate over the array + auto *schemas = yyjson_obj_get(root, "namespaces"); + size_t idx, max; + duckdb_yyjson::yyjson_val *schema; + yyjson_arr_foreach(schemas, idx, max, schema) { + UCAPISchema schema_result; + + schema_result.catalog_name = catalog; + duckdb_yyjson::yyjson_val *value = yyjson_arr_get(schema, 0); + schema_result.schema_name = yyjson_get_str(value); + result.push_back(schema_result); + } + + return result; +} + +} // namespace duckdb diff --git a/src/uc_utils.cpp b/src/uc_utils.cpp new file mode 100644 index 0000000..5acae60 --- /dev/null +++ b/src/uc_utils.cpp @@ -0,0 +1,193 @@ +#include "uc_utils.hpp" +#include "duckdb/common/operator/cast_operators.hpp" +#include "storage/uc_schema_entry.hpp" +#include "storage/uc_transaction.hpp" + +#include + +namespace duckdb { + +string UCUtils::TypeToString(const LogicalType &input) { + switch (input.id()) { + case LogicalType::VARCHAR: + return "TEXT"; + case LogicalType::UTINYINT: + return "TINYINT UNSIGNED"; + case LogicalType::USMALLINT: + return "SMALLINT UNSIGNED"; + case LogicalType::UINTEGER: + return "INTEGER UNSIGNED"; + case LogicalType::UBIGINT: + return "BIGINT UNSIGNED"; + case LogicalType::TIMESTAMP: + return "DATETIME"; + case LogicalType::TIMESTAMP_TZ: + return "TIMESTAMP"; + default: + return input.ToString(); + } +} + +LogicalType UCUtils::TypeToLogicalType(ClientContext &context, const string &type_text) { + if (type_text == "tinyint") { + return LogicalType::TINYINT; + } else if (type_text == "smallint") { + return LogicalType::SMALLINT; + } else if (type_text == "bigint") { + return LogicalType::BIGINT; + } else if (type_text == "int") { + return LogicalType::INTEGER; + } else if (type_text == "long") { + return LogicalType::BIGINT; + } else if (type_text == "string") { + return LogicalType::VARCHAR; + } else if (type_text == "double") { + return LogicalType::DOUBLE; + } else if (type_text == "float") { + return LogicalType::FLOAT; + } else if (type_text == "boolean") { + return LogicalType::BOOLEAN; + } else if (type_text == "timestamp") { + return LogicalType::TIMESTAMP; + } else if (type_text == "timestamptz") { + return LogicalType::TIMESTAMP_TZ; + } else if (type_text == "binary") { + return LogicalType::BLOB; + } else if (type_text == "date") { + return LogicalType::DATE; + } else if (type_text == "timestamp") { + return LogicalType::TIMESTAMP; // TODO: Is this the right timestamp + } else if (type_text.find("decimal(") == 0) { + size_t spec_end = type_text.find(')'); + if (spec_end != string::npos) { + size_t sep = type_text.find(','); + auto prec_str = type_text.substr(8, sep - 8); + auto scale_str = type_text.substr(sep + 1, spec_end - sep - 1); + uint8_t prec = Cast::Operation(prec_str); + uint8_t scale = Cast::Operation(scale_str); + return LogicalType::DECIMAL(prec, scale); + } + } else if (type_text.find("array<") == 0) { + size_t type_end = type_text.rfind('>'); // find last, to deal with nested + if (type_end != string::npos) { + auto child_type_str = type_text.substr(6, type_end - 6); + auto child_type = UCUtils::TypeToLogicalType(context, child_type_str); + return LogicalType::LIST(child_type); + } + } else if (type_text.find("map<") == 0) { + size_t type_end = type_text.rfind('>'); // find last, to deal with nested + if (type_end != string::npos) { + // TODO: Factor this and struct parsing into an iterator over ',' separated values + vector key_val; + size_t cur = 4; + auto nested_opens = 0; + for (;;) { + size_t next_sep = cur; + // find the location of the next ',' ignoring nested commas + while (type_text[next_sep] != ',' || nested_opens > 0) { + if (type_text[next_sep] == '<') { + nested_opens++; + } else if (type_text[next_sep] == '>') { + nested_opens--; + } + next_sep++; + if (next_sep == type_end) { + break; + } + } + auto child_str = type_text.substr(cur, next_sep - cur); + auto child_type = UCUtils::TypeToLogicalType(context, child_str); + key_val.push_back(child_type); + if (next_sep == type_end) { + break; + } + cur = next_sep + 1; + } + if (key_val.size() != 2) { + throw NotImplementedException("Invalid map specification with %i types", key_val.size()); + } + return LogicalType::MAP(key_val[0], key_val[1]); + } + } else if (type_text.find("struct<") == 0) { + size_t type_end = type_text.rfind('>'); // find last, to deal with nested + if (type_end != string::npos) { + child_list_t children; + size_t cur = 7; + auto nested_opens = 0; + for (;;) { + size_t next_sep = cur; + // find the location of the next ',' ignoring nested commas + while (type_text[next_sep] != ',' || nested_opens > 0) { + if (type_text[next_sep] == '<') { + nested_opens++; + } else if (type_text[next_sep] == '>') { + nested_opens--; + } + next_sep++; + if (next_sep == type_end) { + break; + } + } + auto child_str = type_text.substr(cur, next_sep - cur); + size_t type_sep = child_str.find(':'); + if (type_sep == string::npos) { + throw NotImplementedException("Invalid struct child type specifier: %s", child_str); + } + auto child_name = child_str.substr(0, type_sep); + auto child_type = UCUtils::TypeToLogicalType(context, child_str.substr(type_sep + 1, string::npos)); + children.push_back({child_name, child_type}); + if (next_sep == type_end) { + break; + } + cur = next_sep + 1; + } + return LogicalType::STRUCT(children); + } + } + + throw NotImplementedException("Tried to fallback to unknown type for '%s'", type_text); + // fallback for unknown types + return LogicalType::VARCHAR; +} + +LogicalType UCUtils::ToUCType(const LogicalType &input) { + // todo do we need this mapping? + throw NotImplementedException("ToUCType not yet implemented"); + switch (input.id()) { + case LogicalTypeId::BOOLEAN: + case LogicalTypeId::SMALLINT: + case LogicalTypeId::INTEGER: + case LogicalTypeId::BIGINT: + case LogicalTypeId::TINYINT: + case LogicalTypeId::UTINYINT: + case LogicalTypeId::USMALLINT: + case LogicalTypeId::UINTEGER: + case LogicalTypeId::UBIGINT: + case LogicalTypeId::FLOAT: + case LogicalTypeId::DOUBLE: + case LogicalTypeId::BLOB: + case LogicalTypeId::DATE: + case LogicalTypeId::DECIMAL: + case LogicalTypeId::TIMESTAMP: + case LogicalTypeId::TIMESTAMP_TZ: + case LogicalTypeId::VARCHAR: + return input; + case LogicalTypeId::LIST: + throw NotImplementedException("PC does not support arrays - unsupported type \"%s\"", input.ToString()); + case LogicalTypeId::STRUCT: + case LogicalTypeId::MAP: + case LogicalTypeId::UNION: + throw NotImplementedException("PC does not support composite types - unsupported type \"%s\"", + input.ToString()); + case LogicalTypeId::TIMESTAMP_SEC: + case LogicalTypeId::TIMESTAMP_MS: + case LogicalTypeId::TIMESTAMP_NS: + return LogicalType::TIMESTAMP; + case LogicalTypeId::HUGEINT: + return LogicalType::DOUBLE; + default: + return LogicalType::VARCHAR; + } +} + +} // namespace duckdb diff --git a/vcpkg.json b/vcpkg.json index 94ab7f7..17661df 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -4,6 +4,7 @@ "name": "avro-cpp", "features": ["snappy"] }, + "curl", "openssl" ], "vcpkg-configuration": { From f3d5d8785704b21a6beda9661d369f22889ed5e3 Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Tue, 14 Jan 2025 22:40:33 -0500 Subject: [PATCH 02/26] Rename files uc -> ic --- CMakeLists.txt | 22 +++++++++---------- src/{uc_api.cpp => catalog_api.cpp} | 4 ++-- src/{uc_utils.cpp => catalog_utils.cpp} | 6 ++--- src/iceberg_extension.cpp | 10 ++++----- src/include/{uc_api.hpp => catalog_api.hpp} | 7 ------ .../{uc_utils.hpp => catalog_utils.hpp} | 9 +------- .../{uc_catalog.hpp => ic_catalog.hpp} | 9 +------- ...{uc_catalog_set.hpp => ic_catalog_set.hpp} | 7 ------ ...c_schema_entry.hpp => ic_schema_entry.hpp} | 11 ++-------- .../{uc_schema_set.hpp => ic_schema_set.hpp} | 11 ++-------- ...{uc_table_entry.hpp => ic_table_entry.hpp} | 9 +------- .../{uc_table_set.hpp => ic_table_set.hpp} | 11 ++-------- ...{uc_transaction.hpp => ic_transaction.hpp} | 9 +------- ...manager.hpp => ic_transaction_manager.hpp} | 15 ++++--------- .../{uc_catalog.cpp => ic_catalog.cpp} | 6 ++--- ...{uc_catalog_set.cpp => ic_catalog_set.cpp} | 6 ++--- ...{uc_clear_cache.cpp => ic_clear_cache.cpp} | 2 +- ...c_schema_entry.cpp => ic_schema_entry.cpp} | 10 ++++----- .../{uc_schema_set.cpp => ic_schema_set.cpp} | 12 +++++----- ...{uc_table_entry.cpp => ic_table_entry.cpp} | 18 +++++++-------- .../{uc_table_set.cpp => ic_table_set.cpp} | 16 +++++++------- ...{uc_transaction.cpp => ic_transaction.cpp} | 10 ++++----- ...manager.cpp => ic_transaction_manager.cpp} | 16 +++++++------- 23 files changed, 83 insertions(+), 153 deletions(-) rename src/{uc_api.cpp => catalog_api.cpp} (99%) rename src/{uc_utils.cpp => catalog_utils.cpp} (98%) rename src/include/{uc_api.hpp => catalog_api.hpp} (85%) rename src/include/{uc_utils.hpp => catalog_utils.hpp} (70%) rename src/include/storage/{uc_catalog.hpp => ic_catalog.hpp} (91%) rename src/include/storage/{uc_catalog_set.hpp => ic_catalog_set.hpp} (82%) rename src/include/storage/{uc_schema_entry.hpp => ic_schema_entry.hpp} (88%) rename src/include/storage/{uc_schema_set.hpp => ic_schema_set.hpp} (52%) rename src/include/storage/{uc_table_entry.hpp => ic_table_entry.hpp} (84%) rename src/include/storage/{uc_table_set.hpp => ic_table_set.hpp} (77%) rename src/include/storage/{uc_transaction.hpp => ic_transaction.hpp} (71%) rename src/include/storage/{uc_transaction_manager.hpp => ic_transaction_manager.hpp} (59%) rename src/storage/{uc_catalog.cpp => ic_catalog.cpp} (97%) rename src/storage/{uc_catalog_set.cpp => ic_catalog_set.cpp} (93%) rename src/storage/{uc_clear_cache.cpp => ic_clear_cache.cpp} (97%) rename src/storage/{uc_schema_entry.cpp => ic_schema_entry.cpp} (96%) rename src/storage/{uc_schema_set.cpp => ic_schema_set.cpp} (76%) rename src/storage/{uc_table_entry.cpp => ic_table_entry.cpp} (92%) rename src/storage/{uc_table_set.cpp => ic_table_set.cpp} (90%) rename src/storage/{uc_transaction.cpp => ic_transaction.cpp} (87%) rename src/storage/{uc_transaction_manager.cpp => ic_transaction_manager.cpp} (72%) diff --git a/CMakeLists.txt b/CMakeLists.txt index db7837f..0f57eb2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,23 +13,23 @@ include_directories(src/include) set(EXTENSION_SOURCES src/iceberg_extension.cpp src/iceberg_functions.cpp - src/uc_api.cpp - src/uc_utils.cpp + src/catalog_api.cpp + src/catalog_utils.cpp src/common/utils.cpp src/common/schema.cpp src/common/iceberg.cpp src/iceberg_functions/iceberg_snapshots.cpp src/iceberg_functions/iceberg_scan.cpp src/iceberg_functions/iceberg_metadata.cpp - src/storage/uc_catalog.cpp - src/storage/uc_catalog_set.cpp - src/storage/uc_clear_cache.cpp - src/storage/uc_schema_entry.cpp - src/storage/uc_schema_set.cpp - src/storage/uc_table_entry.cpp - src/storage/uc_table_set.cpp - src/storage/uc_transaction.cpp - src/storage/uc_transaction_manager.cpp + src/storage/ic_catalog.cpp + src/storage/ic_catalog_set.cpp + src/storage/ic_clear_cache.cpp + src/storage/ic_schema_entry.cpp + src/storage/ic_schema_set.cpp + src/storage/ic_table_entry.cpp + src/storage/ic_table_set.cpp + src/storage/ic_transaction.cpp + src/storage/ic_transaction_manager.cpp ) add_library(${EXTENSION_NAME} STATIC ${EXTENSION_SOURCES}) diff --git a/src/uc_api.cpp b/src/catalog_api.cpp similarity index 99% rename from src/uc_api.cpp rename to src/catalog_api.cpp index 6a6e8d7..78bc9ea 100644 --- a/src/uc_api.cpp +++ b/src/catalog_api.cpp @@ -1,5 +1,5 @@ -#include "uc_api.hpp" -#include "storage/uc_catalog.hpp" +#include "catalog_api.hpp" +#include "storage/ic_catalog.hpp" #include "yyjson.hpp" #include #include diff --git a/src/uc_utils.cpp b/src/catalog_utils.cpp similarity index 98% rename from src/uc_utils.cpp rename to src/catalog_utils.cpp index 5acae60..591f6ab 100644 --- a/src/uc_utils.cpp +++ b/src/catalog_utils.cpp @@ -1,7 +1,7 @@ -#include "uc_utils.hpp" +#include "catalog_utils.hpp" #include "duckdb/common/operator/cast_operators.hpp" -#include "storage/uc_schema_entry.hpp" -#include "storage/uc_transaction.hpp" +#include "storage/ic_schema_entry.hpp" +#include "storage/ic_transaction.hpp" #include diff --git a/src/iceberg_extension.cpp b/src/iceberg_extension.cpp index 2d4a3b9..10937ff 100644 --- a/src/iceberg_extension.cpp +++ b/src/iceberg_extension.cpp @@ -1,8 +1,8 @@ #define DUCKDB_EXTENSION_MAIN #include "iceberg_extension.hpp" -#include "storage/uc_catalog.hpp" -#include "storage/uc_transaction_manager.hpp" +#include "storage/ic_catalog.hpp" +#include "storage/ic_transaction_manager.hpp" #include "duckdb.hpp" #include "duckdb/main/secret/secret_manager.hpp" @@ -19,7 +19,7 @@ #include "yyjson.hpp" #include "duckdb/main/extension_util.hpp" #include -#include "uc_api.hpp" +#include "catalog_api.hpp" namespace duckdb { @@ -136,8 +136,8 @@ static unique_ptr PolarisCatalogAttach(StorageExtensionInfo *storage_in static unique_ptr CreateTransactionManager(StorageExtensionInfo *storage_info, AttachedDatabase &db, Catalog &catalog) { - auto &uc_catalog = catalog.Cast(); - return make_uniq(db, uc_catalog); + auto &ic_catalog = catalog.Cast(); + return make_uniq(db, ic_catalog); } class UCCatalogStorageExtension : public StorageExtension { diff --git a/src/include/uc_api.hpp b/src/include/catalog_api.hpp similarity index 85% rename from src/include/uc_api.hpp rename to src/include/catalog_api.hpp index a65f208..ff25711 100644 --- a/src/include/uc_api.hpp +++ b/src/include/catalog_api.hpp @@ -1,10 +1,3 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// src/include/uc_api.hpp -// -// -//===----------------------------------------------------------------------===// #pragma once diff --git a/src/include/uc_utils.hpp b/src/include/catalog_utils.hpp similarity index 70% rename from src/include/uc_utils.hpp rename to src/include/catalog_utils.hpp index 8927943..e7db342 100644 --- a/src/include/uc_utils.hpp +++ b/src/include/catalog_utils.hpp @@ -1,15 +1,8 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// mysql_utils.hpp -// -// -//===----------------------------------------------------------------------===// #pragma once #include "duckdb.hpp" -#include "uc_api.hpp" +#include "catalog_api.hpp" namespace duckdb { class UCSchemaEntry; diff --git a/src/include/storage/uc_catalog.hpp b/src/include/storage/ic_catalog.hpp similarity index 91% rename from src/include/storage/uc_catalog.hpp rename to src/include/storage/ic_catalog.hpp index 22df4f0..42da0b4 100644 --- a/src/include/storage/uc_catalog.hpp +++ b/src/include/storage/ic_catalog.hpp @@ -1,17 +1,10 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// storage/uc_catalog.hpp -// -// -//===----------------------------------------------------------------------===// #pragma once #include "duckdb/catalog/catalog.hpp" #include "duckdb/function/table_function.hpp" #include "duckdb/common/enums/access_mode.hpp" -#include "storage/uc_schema_set.hpp" +#include "storage/ic_schema_set.hpp" namespace duckdb { class UCSchemaEntry; diff --git a/src/include/storage/uc_catalog_set.hpp b/src/include/storage/ic_catalog_set.hpp similarity index 82% rename from src/include/storage/uc_catalog_set.hpp rename to src/include/storage/ic_catalog_set.hpp index 8cd9ba3..d21af26 100644 --- a/src/include/storage/uc_catalog_set.hpp +++ b/src/include/storage/ic_catalog_set.hpp @@ -1,10 +1,3 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// storage/uc_catalog_set.hpp -// -// -//===----------------------------------------------------------------------===// #pragma once diff --git a/src/include/storage/uc_schema_entry.hpp b/src/include/storage/ic_schema_entry.hpp similarity index 88% rename from src/include/storage/uc_schema_entry.hpp rename to src/include/storage/ic_schema_entry.hpp index b9d11f1..9cde3da 100644 --- a/src/include/storage/uc_schema_entry.hpp +++ b/src/include/storage/ic_schema_entry.hpp @@ -1,16 +1,9 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// storage/uc_schema_entry.hpp -// -// -//===----------------------------------------------------------------------===// #pragma once -#include "uc_api.hpp" +#include "catalog_api.hpp" #include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" -#include "storage/uc_table_set.hpp" +#include "storage/ic_table_set.hpp" namespace duckdb { class UCTransaction; diff --git a/src/include/storage/uc_schema_set.hpp b/src/include/storage/ic_schema_set.hpp similarity index 52% rename from src/include/storage/uc_schema_set.hpp rename to src/include/storage/ic_schema_set.hpp index 38bfb2f..c019f6e 100644 --- a/src/include/storage/uc_schema_set.hpp +++ b/src/include/storage/ic_schema_set.hpp @@ -1,15 +1,8 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// storage/uc_schema_set.hpp -// -// -//===----------------------------------------------------------------------===// #pragma once -#include "storage/uc_catalog_set.hpp" -#include "storage/uc_schema_entry.hpp" +#include "storage/ic_catalog_set.hpp" +#include "storage/ic_schema_entry.hpp" namespace duckdb { struct CreateSchemaInfo; diff --git a/src/include/storage/uc_table_entry.hpp b/src/include/storage/ic_table_entry.hpp similarity index 84% rename from src/include/storage/uc_table_entry.hpp rename to src/include/storage/ic_table_entry.hpp index 50b2528..5b25e82 100644 --- a/src/include/storage/uc_table_entry.hpp +++ b/src/include/storage/ic_table_entry.hpp @@ -1,14 +1,7 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// storage/uc_table_entry.hpp -// -// -//===----------------------------------------------------------------------===// #pragma once -#include "uc_api.hpp" +#include "catalog_api.hpp" #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" #include "duckdb/parser/parsed_data/create_table_info.hpp" diff --git a/src/include/storage/uc_table_set.hpp b/src/include/storage/ic_table_set.hpp similarity index 77% rename from src/include/storage/uc_table_set.hpp rename to src/include/storage/ic_table_set.hpp index fd8ec0b..c73b1bf 100644 --- a/src/include/storage/uc_table_set.hpp +++ b/src/include/storage/ic_table_set.hpp @@ -1,15 +1,8 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// storage/uc_table_set.hpp -// -// -//===----------------------------------------------------------------------===// #pragma once -#include "storage/uc_catalog_set.hpp" -#include "storage/uc_table_entry.hpp" +#include "storage/ic_catalog_set.hpp" +#include "storage/ic_table_entry.hpp" namespace duckdb { struct CreateTableInfo; diff --git a/src/include/storage/uc_transaction.hpp b/src/include/storage/ic_transaction.hpp similarity index 71% rename from src/include/storage/uc_transaction.hpp rename to src/include/storage/ic_transaction.hpp index 86a1423..16d6da8 100644 --- a/src/include/storage/uc_transaction.hpp +++ b/src/include/storage/ic_transaction.hpp @@ -1,10 +1,3 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// storage/uc_transaction.hpp -// -// -//===----------------------------------------------------------------------===// #pragma once @@ -19,7 +12,7 @@ enum class UCTransactionState { TRANSACTION_NOT_YET_STARTED, TRANSACTION_STARTED class UCTransaction : public Transaction { public: - UCTransaction(UCCatalog &uc_catalog, TransactionManager &manager, ClientContext &context); + UCTransaction(UCCatalog &ic_catalog, TransactionManager &manager, ClientContext &context); ~UCTransaction() override; void Start(); diff --git a/src/include/storage/uc_transaction_manager.hpp b/src/include/storage/ic_transaction_manager.hpp similarity index 59% rename from src/include/storage/uc_transaction_manager.hpp rename to src/include/storage/ic_transaction_manager.hpp index b368f51..7f9fb80 100644 --- a/src/include/storage/uc_transaction_manager.hpp +++ b/src/include/storage/ic_transaction_manager.hpp @@ -1,22 +1,15 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// storage/uc_transaction_manager.hpp -// -// -//===----------------------------------------------------------------------===// #pragma once #include "duckdb/transaction/transaction_manager.hpp" -#include "storage/uc_catalog.hpp" -#include "storage/uc_transaction.hpp" +#include "storage/ic_catalog.hpp" +#include "storage/ic_transaction.hpp" namespace duckdb { class UCTransactionManager : public TransactionManager { public: - UCTransactionManager(AttachedDatabase &db_p, UCCatalog &uc_catalog); + UCTransactionManager(AttachedDatabase &db_p, UCCatalog &ic_catalog); Transaction &StartTransaction(ClientContext &context) override; ErrorData CommitTransaction(ClientContext &context, Transaction &transaction) override; @@ -25,7 +18,7 @@ class UCTransactionManager : public TransactionManager { void Checkpoint(ClientContext &context, bool force = false) override; private: - UCCatalog &uc_catalog; + UCCatalog &ic_catalog; mutex transaction_lock; reference_map_t> transactions; }; diff --git a/src/storage/uc_catalog.cpp b/src/storage/ic_catalog.cpp similarity index 97% rename from src/storage/uc_catalog.cpp rename to src/storage/ic_catalog.cpp index 3c23771..f01387c 100644 --- a/src/storage/uc_catalog.cpp +++ b/src/storage/ic_catalog.cpp @@ -1,6 +1,6 @@ -#include "storage/uc_catalog.hpp" -#include "storage/uc_schema_entry.hpp" -#include "storage/uc_transaction.hpp" +#include "storage/ic_catalog.hpp" +#include "storage/ic_schema_entry.hpp" +#include "storage/ic_transaction.hpp" #include "duckdb/storage/database_size.hpp" #include "duckdb/parser/parsed_data/drop_info.hpp" #include "duckdb/parser/parsed_data/create_schema_info.hpp" diff --git a/src/storage/uc_catalog_set.cpp b/src/storage/ic_catalog_set.cpp similarity index 93% rename from src/storage/uc_catalog_set.cpp rename to src/storage/ic_catalog_set.cpp index 6451087..7f7a26c 100644 --- a/src/storage/uc_catalog_set.cpp +++ b/src/storage/ic_catalog_set.cpp @@ -1,7 +1,7 @@ -#include "storage/uc_catalog_set.hpp" -#include "storage/uc_transaction.hpp" +#include "storage/ic_catalog_set.hpp" +#include "storage/ic_transaction.hpp" #include "duckdb/parser/parsed_data/drop_info.hpp" -#include "storage/uc_schema_entry.hpp" +#include "storage/ic_schema_entry.hpp" namespace duckdb { diff --git a/src/storage/uc_clear_cache.cpp b/src/storage/ic_clear_cache.cpp similarity index 97% rename from src/storage/uc_clear_cache.cpp rename to src/storage/ic_clear_cache.cpp index 2c51abb..d4346b8 100644 --- a/src/storage/uc_clear_cache.cpp +++ b/src/storage/ic_clear_cache.cpp @@ -3,7 +3,7 @@ #include "duckdb/parser/parsed_data/create_table_function_info.hpp" #include "duckdb/main/database_manager.hpp" #include "duckdb/main/attached_database.hpp" -#include "storage/uc_catalog.hpp" +#include "storage/ic_catalog.hpp" namespace duckdb { diff --git a/src/storage/uc_schema_entry.cpp b/src/storage/ic_schema_entry.cpp similarity index 96% rename from src/storage/uc_schema_entry.cpp rename to src/storage/ic_schema_entry.cpp index 7bc93ce..57cd7d2 100644 --- a/src/storage/uc_schema_entry.cpp +++ b/src/storage/ic_schema_entry.cpp @@ -1,6 +1,6 @@ -#include "storage/uc_schema_entry.hpp" -#include "storage/uc_table_entry.hpp" -#include "storage/uc_transaction.hpp" +#include "storage/ic_schema_entry.hpp" +#include "storage/ic_table_entry.hpp" +#include "storage/ic_transaction.hpp" #include "duckdb/parser/parsed_data/create_view_info.hpp" #include "duckdb/parser/parsed_data/create_index_info.hpp" #include "duckdb/planner/parsed_data/bound_create_table_info.hpp" @@ -74,8 +74,8 @@ optional_ptr UCSchemaEntry::CreateView(CatalogTransaction transact throw NotImplementedException("REPLACE ON CONFLICT in CreateView"); } } - auto &uc_transaction = GetUCTransaction(transaction); - // uc_transaction.Query(GetUCCreateView(info)); + auto &ic_transaction = GetUCTransaction(transaction); + // ic_transaction.Query(GetUCCreateView(info)); return tables.RefreshTable(transaction.GetContext(), info.view_name); } diff --git a/src/storage/uc_schema_set.cpp b/src/storage/ic_schema_set.cpp similarity index 76% rename from src/storage/uc_schema_set.cpp rename to src/storage/ic_schema_set.cpp index 6efb318..3f85a09 100644 --- a/src/storage/uc_schema_set.cpp +++ b/src/storage/ic_schema_set.cpp @@ -1,7 +1,7 @@ -#include "storage/uc_schema_set.hpp" -#include "storage/uc_catalog.hpp" -#include "uc_api.hpp" -#include "storage/uc_transaction.hpp" +#include "storage/ic_schema_set.hpp" +#include "storage/ic_catalog.hpp" +#include "catalog_api.hpp" +#include "storage/ic_transaction.hpp" #include "duckdb/parser/parsed_data/create_schema_info.hpp" #include "duckdb/catalog/catalog.hpp" @@ -18,8 +18,8 @@ static bool IsInternalTable(const string &catalog, const string &schema) { } void UCSchemaSet::LoadEntries(ClientContext &context) { - auto &uc_catalog = catalog.Cast(); - auto tables = UCAPI::GetSchemas(catalog.GetName(), uc_catalog.internal_name, uc_catalog.credentials); + auto &ic_catalog = catalog.Cast(); + auto tables = UCAPI::GetSchemas(catalog.GetName(), ic_catalog.internal_name, ic_catalog.credentials); for (const auto &schema : tables) { CreateSchemaInfo info; diff --git a/src/storage/uc_table_entry.cpp b/src/storage/ic_table_entry.cpp similarity index 92% rename from src/storage/uc_table_entry.cpp rename to src/storage/ic_table_entry.cpp index 7691a2f..2928870 100644 --- a/src/storage/uc_table_entry.cpp +++ b/src/storage/ic_table_entry.cpp @@ -1,7 +1,7 @@ -#include "storage/uc_catalog.hpp" -#include "storage/uc_schema_entry.hpp" -#include "storage/uc_table_entry.hpp" -#include "storage/uc_transaction.hpp" +#include "storage/ic_catalog.hpp" +#include "storage/ic_schema_entry.hpp" +#include "storage/ic_table_entry.hpp" +#include "storage/ic_transaction.hpp" #include "duckdb/storage/statistics/base_statistics.hpp" #include "duckdb/storage/table_storage_info.hpp" #include "duckdb/main/extension_util.hpp" @@ -9,7 +9,7 @@ #include "duckdb/main/secret/secret_manager.hpp" #include "duckdb/catalog/catalog_entry/table_function_catalog_entry.hpp" #include "duckdb/parser/tableref/table_function_ref.hpp" -#include "uc_api.hpp" +#include "catalog_api.hpp" #include "../../duckdb/third_party/catch/catch.hpp" #include "duckdb/planner/binder.hpp" #include "duckdb/planner/tableref/bound_table_function.hpp" @@ -57,7 +57,7 @@ struct MyIcebergFunctionData : public FunctionData { TableFunction UCTableEntry::GetScanFunction(ClientContext &context, unique_ptr &bind_data) { auto &db = DatabaseInstance::GetDatabase(context); - auto &uc_catalog = catalog.Cast(); + auto &ic_catalog = catalog.Cast(); auto &parquet_function_set = ExtensionUtil::GetTableFunction(db, "parquet_scan"); auto parquet_scan_function = parquet_function_set.functions.GetFunctionByArguments(context, {LogicalType::VARCHAR}); @@ -76,11 +76,11 @@ TableFunction UCTableEntry::GetScanFunction(ClientContext &context, unique_ptrschema_name, table_data->name, uc_catalog.credentials); + ic_catalog.internal_name, table_data->schema_name, table_data->name, ic_catalog.credentials); // Inject secret into secret manager scoped to this path CreateSecretInfo info(OnCreateConflict::REPLACE_ON_CONFLICT, SecretPersistType::TEMPORARY); - info.name = "__internal_uc_" + table_data->table_id; + info.name = "__internal_ic_" + table_data->table_id; info.type = "s3"; info.provider = "config"; info.storage_type = "memory"; @@ -88,7 +88,7 @@ TableFunction UCTableEntry::GetScanFunction(ClientContext &context, unique_ptr(); + auto &ic_catalog = catalog.Cast(); // TODO: handle out-of-order columns using position property - auto tables = UCAPI::GetTables(catalog.GetName(), catalog.GetDBPath(), schema.name, uc_catalog.credentials); + auto tables = UCAPI::GetTables(catalog.GetName(), catalog.GetDBPath(), schema.name, ic_catalog.credentials); for (auto &table : tables) { D_ASSERT(schema.name == table.schema_name); diff --git a/src/storage/uc_transaction.cpp b/src/storage/ic_transaction.cpp similarity index 87% rename from src/storage/uc_transaction.cpp rename to src/storage/ic_transaction.cpp index 2942458..fbff966 100644 --- a/src/storage/uc_transaction.cpp +++ b/src/storage/ic_transaction.cpp @@ -1,14 +1,14 @@ -#include "storage/uc_transaction.hpp" -#include "storage/uc_catalog.hpp" +#include "storage/ic_transaction.hpp" +#include "storage/ic_catalog.hpp" #include "duckdb/parser/parsed_data/create_view_info.hpp" #include "duckdb/catalog/catalog_entry/index_catalog_entry.hpp" #include "duckdb/catalog/catalog_entry/view_catalog_entry.hpp" namespace duckdb { -UCTransaction::UCTransaction(UCCatalog &uc_catalog, TransactionManager &manager, ClientContext &context) - : Transaction(manager, context), access_mode(uc_catalog.access_mode) { - // connection = UCConnection::Open(uc_catalog.path); +UCTransaction::UCTransaction(UCCatalog &ic_catalog, TransactionManager &manager, ClientContext &context) + : Transaction(manager, context), access_mode(ic_catalog.access_mode) { + // connection = UCConnection::Open(ic_catalog.path); } UCTransaction::~UCTransaction() = default; diff --git a/src/storage/uc_transaction_manager.cpp b/src/storage/ic_transaction_manager.cpp similarity index 72% rename from src/storage/uc_transaction_manager.cpp rename to src/storage/ic_transaction_manager.cpp index afd7e07..bf0f319 100644 --- a/src/storage/uc_transaction_manager.cpp +++ b/src/storage/ic_transaction_manager.cpp @@ -1,14 +1,14 @@ -#include "storage/uc_transaction_manager.hpp" +#include "storage/ic_transaction_manager.hpp" #include "duckdb/main/attached_database.hpp" namespace duckdb { -UCTransactionManager::UCTransactionManager(AttachedDatabase &db_p, UCCatalog &uc_catalog) - : TransactionManager(db_p), uc_catalog(uc_catalog) { +UCTransactionManager::UCTransactionManager(AttachedDatabase &db_p, UCCatalog &ic_catalog) + : TransactionManager(db_p), ic_catalog(ic_catalog) { } Transaction &UCTransactionManager::StartTransaction(ClientContext &context) { - auto transaction = make_uniq(uc_catalog, *this, context); + auto transaction = make_uniq(ic_catalog, *this, context); transaction->Start(); auto &result = *transaction; lock_guard l(transaction_lock); @@ -17,16 +17,16 @@ Transaction &UCTransactionManager::StartTransaction(ClientContext &context) { } ErrorData UCTransactionManager::CommitTransaction(ClientContext &context, Transaction &transaction) { - auto &uc_transaction = transaction.Cast(); - uc_transaction.Commit(); + auto &ic_transaction = transaction.Cast(); + ic_transaction.Commit(); lock_guard l(transaction_lock); transactions.erase(transaction); return ErrorData(); } void UCTransactionManager::RollbackTransaction(Transaction &transaction) { - auto &uc_transaction = transaction.Cast(); - uc_transaction.Rollback(); + auto &ic_transaction = transaction.Cast(); + ic_transaction.Rollback(); lock_guard l(transaction_lock); transactions.erase(transaction); } From 1be2fa33cae9fc657b4625f77cfadae4dd8b37bc Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Tue, 14 Jan 2025 22:50:52 -0500 Subject: [PATCH 03/26] Started renaming classes --- src/catalog_api.cpp | 32 +++++++-------- src/catalog_utils.cpp | 12 +++--- src/iceberg_extension.cpp | 18 ++++----- src/include/catalog_api.hpp | 24 +++++------ src/include/catalog_utils.hpp | 10 ++--- src/include/storage/ic_catalog.hpp | 8 ++-- src/include/storage/ic_catalog_set.hpp | 14 +++---- src/include/storage/ic_schema_entry.hpp | 12 +++--- src/include/storage/ic_schema_set.hpp | 2 +- src/include/storage/ic_table_entry.hpp | 2 +- src/include/storage/ic_table_set.hpp | 6 +-- src/include/storage/ic_transaction.hpp | 10 ++--- .../storage/ic_transaction_manager.hpp | 2 +- src/storage/ic_catalog.cpp | 4 +- src/storage/ic_catalog_set.cpp | 22 +++++----- src/storage/ic_schema_entry.cpp | 40 +++++++++---------- src/storage/ic_schema_set.cpp | 8 ++-- src/storage/ic_table_entry.cpp | 4 +- src/storage/ic_table_set.cpp | 14 +++---- src/storage/ic_transaction.cpp | 18 ++++----- src/storage/ic_transaction_manager.cpp | 8 ++-- 21 files changed, 135 insertions(+), 135 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index 78bc9ea..dd40f7b 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -8,7 +8,7 @@ namespace duckdb { -//! We use a global here to store the path that is selected on the UCAPI::InitializeCurl call +//! We use a global here to store the path that is selected on the IBAPI::InitializeCurl call static string SELECTED_CURL_CERT_PATH = ""; static size_t GetRequestWriteCallback(void *contents, size_t size, size_t nmemb, void *userp) { @@ -137,7 +137,7 @@ static string PostRequest(const string &url, const string &post_data, curl_slist } -static duckdb_yyjson::yyjson_val *GetTableMetadata(const string &internal, const string &schema, const string &table, UCCredentials credentials) { +static duckdb_yyjson::yyjson_val *GetTableMetadata(const string &internal, const string &schema, const string &table, IBCredentials credentials) { struct curl_slist *extra_headers = NULL; extra_headers = curl_slist_append(extra_headers, "X-Iceberg-Access-Delegation: vended-credentials"); auto api_result = GetRequest( @@ -176,16 +176,16 @@ static string TryGetStrFromObject(duckdb_yyjson::yyjson_val *obj, const string & fail_on_missing); } -void UCAPI::InitializeCurl() { +void IBAPI::InitializeCurl() { SelectCurlCertPath(); } -vector UCAPI::GetCatalogs(const string &catalog, UCCredentials credentials) { - throw NotImplementedException("UCAPI::GetCatalogs"); +vector IBAPI::GetCatalogs(const string &catalog, IBCredentials credentials) { + throw NotImplementedException("IBAPI::GetCatalogs"); } -static UCAPIColumnDefinition ParseColumnDefinition(duckdb_yyjson::yyjson_val *column_def) { - UCAPIColumnDefinition result; +static IBAPIColumnDefinition ParseColumnDefinition(duckdb_yyjson::yyjson_val *column_def) { + IBAPIColumnDefinition result; result.name = TryGetStrFromObject(column_def, "name"); result.type_text = TryGetStrFromObject(column_def, "type"); @@ -196,8 +196,8 @@ static UCAPIColumnDefinition ParseColumnDefinition(duckdb_yyjson::yyjson_val *co return result; } -UCAPITableCredentials UCAPI::GetTableCredentials(const string &internal, const string &schema, const string &table, UCCredentials credentials) { - UCAPITableCredentials result; +IBAPITableCredentials IBAPI::GetTableCredentials(const string &internal, const string &schema, const string &table, IBCredentials credentials) { + IBAPITableCredentials result; duckdb_yyjson::yyjson_val *root = GetTableMetadata(internal, schema, table, credentials); auto *aws_temp_credentials = yyjson_obj_get(root, "config"); @@ -211,7 +211,7 @@ UCAPITableCredentials UCAPI::GetTableCredentials(const string &internal, const s return result; } -string UCAPI::GetToken(string id, string secret, string endpoint) { +string IBAPI::GetToken(string id, string secret, string endpoint) { string post_data = "grant_type=client_credentials&client_id=" + id + "&client_secret=" + secret + "&scope=PRINCIPAL_ROLE:ALL"; string api_result = PostRequest(endpoint + "/v1/oauth/tokens", post_data); @@ -228,8 +228,8 @@ string UCAPI::GetToken(string id, string secret, string endpoint) { return TryGetStrFromObject(root, "access_token"); } -vector UCAPI::GetTables(const string &catalog, const string &internal, const string &schema, UCCredentials credentials) { - vector result; +vector IBAPI::GetTables(const string &catalog, const string &internal, const string &schema, IBCredentials credentials) { + vector result; auto api_result = GetRequest(credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables", credentials.token); @@ -242,7 +242,7 @@ vector UCAPI::GetTables(const string &catalog, const string &interna size_t idx, max; duckdb_yyjson::yyjson_val *table; yyjson_arr_foreach(tables, idx, max, table) { - UCAPITable table_result; + IBAPITable table_result; table_result.catalog_name = catalog; table_result.schema_name = schema; table_result.name = TryGetStrFromObject(table, "name"); @@ -285,8 +285,8 @@ vector UCAPI::GetTables(const string &catalog, const string &interna return result; } -vector UCAPI::GetSchemas(const string &catalog, const string &internal, UCCredentials credentials) { - vector result; +vector IBAPI::GetSchemas(const string &catalog, const string &internal, IBCredentials credentials) { + vector result; auto api_result = GetRequest(credentials.endpoint + "/v1/" + internal + "/namespaces", credentials.token); @@ -306,7 +306,7 @@ vector UCAPI::GetSchemas(const string &catalog, const string &inter size_t idx, max; duckdb_yyjson::yyjson_val *schema; yyjson_arr_foreach(schemas, idx, max, schema) { - UCAPISchema schema_result; + IBAPISchema schema_result; schema_result.catalog_name = catalog; duckdb_yyjson::yyjson_val *value = yyjson_arr_get(schema, 0); diff --git a/src/catalog_utils.cpp b/src/catalog_utils.cpp index 591f6ab..e6f3e52 100644 --- a/src/catalog_utils.cpp +++ b/src/catalog_utils.cpp @@ -7,7 +7,7 @@ namespace duckdb { -string UCUtils::TypeToString(const LogicalType &input) { +string IBUtils::TypeToString(const LogicalType &input) { switch (input.id()) { case LogicalType::VARCHAR: return "TEXT"; @@ -28,7 +28,7 @@ string UCUtils::TypeToString(const LogicalType &input) { } } -LogicalType UCUtils::TypeToLogicalType(ClientContext &context, const string &type_text) { +LogicalType IBUtils::TypeToLogicalType(ClientContext &context, const string &type_text) { if (type_text == "tinyint") { return LogicalType::TINYINT; } else if (type_text == "smallint") { @@ -71,7 +71,7 @@ LogicalType UCUtils::TypeToLogicalType(ClientContext &context, const string &typ size_t type_end = type_text.rfind('>'); // find last, to deal with nested if (type_end != string::npos) { auto child_type_str = type_text.substr(6, type_end - 6); - auto child_type = UCUtils::TypeToLogicalType(context, child_type_str); + auto child_type = IBUtils::TypeToLogicalType(context, child_type_str); return LogicalType::LIST(child_type); } } else if (type_text.find("map<") == 0) { @@ -96,7 +96,7 @@ LogicalType UCUtils::TypeToLogicalType(ClientContext &context, const string &typ } } auto child_str = type_text.substr(cur, next_sep - cur); - auto child_type = UCUtils::TypeToLogicalType(context, child_str); + auto child_type = IBUtils::TypeToLogicalType(context, child_str); key_val.push_back(child_type); if (next_sep == type_end) { break; @@ -134,7 +134,7 @@ LogicalType UCUtils::TypeToLogicalType(ClientContext &context, const string &typ throw NotImplementedException("Invalid struct child type specifier: %s", child_str); } auto child_name = child_str.substr(0, type_sep); - auto child_type = UCUtils::TypeToLogicalType(context, child_str.substr(type_sep + 1, string::npos)); + auto child_type = IBUtils::TypeToLogicalType(context, child_str.substr(type_sep + 1, string::npos)); children.push_back({child_name, child_type}); if (next_sep == type_end) { break; @@ -150,7 +150,7 @@ LogicalType UCUtils::TypeToLogicalType(ClientContext &context, const string &typ return LogicalType::VARCHAR; } -LogicalType UCUtils::ToUCType(const LogicalType &input) { +LogicalType IBUtils::ToUCType(const LogicalType &input) { // todo do we need this mapping? throw NotImplementedException("ToUCType not yet implemented"); switch (input.id()) { diff --git a/src/iceberg_extension.cpp b/src/iceberg_extension.cpp index 10937ff..5b5d9c5 100644 --- a/src/iceberg_extension.cpp +++ b/src/iceberg_extension.cpp @@ -23,7 +23,7 @@ namespace duckdb { -static unique_ptr CreatePolarisSecretFunction(ClientContext &, CreateSecretInput &input) { +static unique_ptr CreateCatalogSecretFunction(ClientContext &, CreateSecretInput &input) { // apply any overridden settings vector prefix_paths; auto result = make_uniq(prefix_paths, "iceberg", "config", input.name); @@ -42,7 +42,7 @@ static unique_ptr CreatePolarisSecretFunction(ClientContext &, Creat } // Get token from catalog - result->secret_map["token"] = UCAPI::GetToken( + result->secret_map["token"] = IBAPI::GetToken( result->secret_map["client_id"].ToString(), result->secret_map["client_secret"].ToString(), result->secret_map["endpoint"].ToString()); @@ -53,7 +53,7 @@ static unique_ptr CreatePolarisSecretFunction(ClientContext &, Creat return std::move(result); } -static void SetPolarisSecretParameters(CreateSecretFunction &function) { +static void SetCatalogSecretParameters(CreateSecretFunction &function) { function.named_parameters["client_id"] = LogicalType::VARCHAR; function.named_parameters["client_secret"] = LogicalType::VARCHAR; function.named_parameters["endpoint"] = LogicalType::VARCHAR; @@ -77,10 +77,10 @@ unique_ptr GetSecret(ClientContext &context, const string &secret_n return nullptr; } -static unique_ptr PolarisCatalogAttach(StorageExtensionInfo *storage_info, ClientContext &context, +static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_info, ClientContext &context, AttachedDatabase &db, const string &name, AttachInfo &info, AccessMode access_mode) { - UCCredentials credentials; + IBCredentials credentials; // check if we have a secret provided string secret_name; @@ -143,7 +143,7 @@ static unique_ptr CreateTransactionManager(StorageExtensionI class UCCatalogStorageExtension : public StorageExtension { public: UCCatalogStorageExtension() { - attach = PolarisCatalogAttach; + attach = IcebergCatalogAttach; create_transaction_manager = CreateTransactionManager; } }; @@ -168,7 +168,7 @@ static void LoadInternal(DatabaseInstance &instance) { ExtensionUtil::RegisterFunction(instance, fun); } - UCAPI::InitializeCurl(); + IBAPI::InitializeCurl(); SecretType secret_type; secret_type.name = "iceberg"; @@ -176,8 +176,8 @@ static void LoadInternal(DatabaseInstance &instance) { secret_type.default_provider = "config"; ExtensionUtil::RegisterSecretType(instance, secret_type); - CreateSecretFunction secret_function = {"iceberg", "config", CreatePolarisSecretFunction}; - SetPolarisSecretParameters(secret_function); + CreateSecretFunction secret_function = {"iceberg", "config", CreateCatalogSecretFunction}; + SetCatalogSecretParameters(secret_function); ExtensionUtil::RegisterFunction(instance, secret_function); config.storage_extensions["iceberg"] = make_uniq(); diff --git a/src/include/catalog_api.hpp b/src/include/catalog_api.hpp index ff25711..e8ef25b 100644 --- a/src/include/catalog_api.hpp +++ b/src/include/catalog_api.hpp @@ -4,9 +4,9 @@ #include "duckdb/common/types.hpp" namespace duckdb { -struct UCCredentials; +struct IBCredentials; -struct UCAPIColumnDefinition { +struct IBAPIColumnDefinition { string name; string type_text; idx_t precision; @@ -14,7 +14,7 @@ struct UCAPIColumnDefinition { idx_t position; }; -struct UCAPITable { +struct IBAPITable { string table_id; string name; @@ -25,30 +25,30 @@ struct UCAPITable { string storage_location; - vector columns; + vector columns; }; -struct UCAPISchema { +struct IBAPISchema { string schema_name; string catalog_name; }; -struct UCAPITableCredentials { +struct IBAPITableCredentials { string key_id; string secret; string session_token; }; -class UCAPI { +class IBAPI { public: //! WARNING: not thread-safe. To be called once on extension initialization static void InitializeCurl(); - static UCAPITableCredentials GetTableCredentials(const string &internal, const string &schema, const string &table, UCCredentials credentials); - static vector GetCatalogs(const string &catalog, UCCredentials credentials); - static vector GetTables(const string &catalog, const string &internal, const string &schema, UCCredentials credentials); - static vector GetSchemas(const string &catalog, const string &internal, UCCredentials credentials); - static vector GetTablesInSchema(const string &catalog, const string &schema, UCCredentials credentials); + static IBAPITableCredentials GetTableCredentials(const string &internal, const string &schema, const string &table, IBCredentials credentials); + static vector GetCatalogs(const string &catalog, IBCredentials credentials); + static vector GetTables(const string &catalog, const string &internal, const string &schema, IBCredentials credentials); + static vector GetSchemas(const string &catalog, const string &internal, IBCredentials credentials); + static vector GetTablesInSchema(const string &catalog, const string &schema, IBCredentials credentials); static string GetToken(string id, string secret, string endpoint); }; } // namespace duckdb diff --git a/src/include/catalog_utils.hpp b/src/include/catalog_utils.hpp index e7db342..f294299 100644 --- a/src/include/catalog_utils.hpp +++ b/src/include/catalog_utils.hpp @@ -5,18 +5,18 @@ #include "catalog_api.hpp" namespace duckdb { -class UCSchemaEntry; -class UCTransaction; +class IBSchemaEntry; +class IBTransaction; -enum class UCTypeAnnotation { STANDARD, CAST_TO_VARCHAR, NUMERIC_AS_DOUBLE, CTID, JSONB, FIXED_LENGTH_CHAR }; +enum class IBTypeAnnotation { STANDARD, CAST_TO_VARCHAR, NUMERIC_AS_DOUBLE, CTID, JSONB, FIXED_LENGTH_CHAR }; struct UCType { idx_t oid = 0; - UCTypeAnnotation info = UCTypeAnnotation::STANDARD; + IBTypeAnnotation info = IBTypeAnnotation::STANDARD; vector children; }; -class UCUtils { +class IBUtils { public: static LogicalType ToUCType(const LogicalType &input); static LogicalType TypeToLogicalType(ClientContext &context, const string &columnDefinition); diff --git a/src/include/storage/ic_catalog.hpp b/src/include/storage/ic_catalog.hpp index 42da0b4..901ee05 100644 --- a/src/include/storage/ic_catalog.hpp +++ b/src/include/storage/ic_catalog.hpp @@ -7,9 +7,9 @@ #include "storage/ic_schema_set.hpp" namespace duckdb { -class UCSchemaEntry; +class IBSchemaEntry; -struct UCCredentials { +struct IBCredentials { string endpoint; string client_id; string client_secret; @@ -29,12 +29,12 @@ class UCClearCacheFunction : public TableFunction { class UCCatalog : public Catalog { public: explicit UCCatalog(AttachedDatabase &db_p, const string &internal_name, AccessMode access_mode, - UCCredentials credentials); + IBCredentials credentials); ~UCCatalog(); string internal_name; AccessMode access_mode; - UCCredentials credentials; + IBCredentials credentials; public: void Initialize(bool load_builtin) override; diff --git a/src/include/storage/ic_catalog_set.hpp b/src/include/storage/ic_catalog_set.hpp index d21af26..d2a0099 100644 --- a/src/include/storage/ic_catalog_set.hpp +++ b/src/include/storage/ic_catalog_set.hpp @@ -7,12 +7,12 @@ namespace duckdb { struct DropInfo; -class UCSchemaEntry; -class UCTransaction; +class IBSchemaEntry; +class IBTransaction; -class UCCatalogSet { +class IBCatalogSet { public: - UCCatalogSet(Catalog &catalog); + IBCatalogSet(Catalog &catalog); optional_ptr GetEntry(ClientContext &context, const string &name); virtual void DropEntry(ClientContext &context, DropInfo &info); @@ -34,14 +34,14 @@ class UCCatalogSet { bool is_loaded; }; -class UCInSchemaSet : public UCCatalogSet { +class UCInSchemaSet : public IBCatalogSet { public: - UCInSchemaSet(UCSchemaEntry &schema); + UCInSchemaSet(IBSchemaEntry &schema); optional_ptr CreateEntry(unique_ptr entry) override; protected: - UCSchemaEntry &schema; + IBSchemaEntry &schema; }; } // namespace duckdb diff --git a/src/include/storage/ic_schema_entry.hpp b/src/include/storage/ic_schema_entry.hpp index 9cde3da..3579f59 100644 --- a/src/include/storage/ic_schema_entry.hpp +++ b/src/include/storage/ic_schema_entry.hpp @@ -6,14 +6,14 @@ #include "storage/ic_table_set.hpp" namespace duckdb { -class UCTransaction; +class IBTransaction; -class UCSchemaEntry : public SchemaCatalogEntry { +class IBSchemaEntry : public SchemaCatalogEntry { public: - UCSchemaEntry(Catalog &catalog, CreateSchemaInfo &info); - ~UCSchemaEntry() override; + IBSchemaEntry(Catalog &catalog, CreateSchemaInfo &info); + ~IBSchemaEntry() override; - unique_ptr schema_data; + unique_ptr schema_data; public: optional_ptr CreateTable(CatalogTransaction transaction, BoundCreateTableInfo &info) override; @@ -37,7 +37,7 @@ class UCSchemaEntry : public SchemaCatalogEntry { optional_ptr GetEntry(CatalogTransaction transaction, CatalogType type, const string &name) override; private: - UCCatalogSet &GetCatalogSet(CatalogType type); + IBCatalogSet &GetCatalogSet(CatalogType type); private: UCTableSet tables; diff --git a/src/include/storage/ic_schema_set.hpp b/src/include/storage/ic_schema_set.hpp index c019f6e..365bf0a 100644 --- a/src/include/storage/ic_schema_set.hpp +++ b/src/include/storage/ic_schema_set.hpp @@ -7,7 +7,7 @@ namespace duckdb { struct CreateSchemaInfo; -class UCSchemaSet : public UCCatalogSet { +class UCSchemaSet : public IBCatalogSet { public: explicit UCSchemaSet(Catalog &catalog); diff --git a/src/include/storage/ic_table_entry.hpp b/src/include/storage/ic_table_entry.hpp index 5b25e82..7480ac5 100644 --- a/src/include/storage/ic_table_entry.hpp +++ b/src/include/storage/ic_table_entry.hpp @@ -30,7 +30,7 @@ class UCTableEntry : public TableCatalogEntry { UCTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTableInfo &info); UCTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, UCTableInfo &info); - unique_ptr table_data; + unique_ptr table_data; public: unique_ptr GetStatistics(ClientContext &context, column_t column_id) override; diff --git a/src/include/storage/ic_table_set.hpp b/src/include/storage/ic_table_set.hpp index c73b1bf..e20381d 100644 --- a/src/include/storage/ic_table_set.hpp +++ b/src/include/storage/ic_table_set.hpp @@ -7,16 +7,16 @@ namespace duckdb { struct CreateTableInfo; class UCResult; -class UCSchemaEntry; +class IBSchemaEntry; class UCTableSet : public UCInSchemaSet { public: - explicit UCTableSet(UCSchemaEntry &schema); + explicit UCTableSet(IBSchemaEntry &schema); public: optional_ptr CreateTable(ClientContext &context, BoundCreateTableInfo &info); - static unique_ptr GetTableInfo(ClientContext &context, UCSchemaEntry &schema, + static unique_ptr GetTableInfo(ClientContext &context, IBSchemaEntry &schema, const string &table_name); optional_ptr RefreshTable(ClientContext &context, const string &table_name); diff --git a/src/include/storage/ic_transaction.hpp b/src/include/storage/ic_transaction.hpp index 16d6da8..3043741 100644 --- a/src/include/storage/ic_transaction.hpp +++ b/src/include/storage/ic_transaction.hpp @@ -5,15 +5,15 @@ namespace duckdb { class UCCatalog; -class UCSchemaEntry; +class IBSchemaEntry; class UCTableEntry; enum class UCTransactionState { TRANSACTION_NOT_YET_STARTED, TRANSACTION_STARTED, TRANSACTION_FINISHED }; -class UCTransaction : public Transaction { +class IBTransaction : public Transaction { public: - UCTransaction(UCCatalog &ic_catalog, TransactionManager &manager, ClientContext &context); - ~UCTransaction() override; + IBTransaction(UCCatalog &ic_catalog, TransactionManager &manager, ClientContext &context); + ~IBTransaction() override; void Start(); void Commit(); @@ -21,7 +21,7 @@ class UCTransaction : public Transaction { // UCConnection &GetConnection(); // unique_ptr Query(const string &query); - static UCTransaction &Get(ClientContext &context, Catalog &catalog); + static IBTransaction &Get(ClientContext &context, Catalog &catalog); AccessMode GetAccessMode() const { return access_mode; } diff --git a/src/include/storage/ic_transaction_manager.hpp b/src/include/storage/ic_transaction_manager.hpp index 7f9fb80..255f65f 100644 --- a/src/include/storage/ic_transaction_manager.hpp +++ b/src/include/storage/ic_transaction_manager.hpp @@ -20,7 +20,7 @@ class UCTransactionManager : public TransactionManager { private: UCCatalog &ic_catalog; mutex transaction_lock; - reference_map_t> transactions; + reference_map_t> transactions; }; } // namespace duckdb diff --git a/src/storage/ic_catalog.cpp b/src/storage/ic_catalog.cpp index f01387c..efb4e3e 100644 --- a/src/storage/ic_catalog.cpp +++ b/src/storage/ic_catalog.cpp @@ -9,7 +9,7 @@ namespace duckdb { UCCatalog::UCCatalog(AttachedDatabase &db_p, const string &internal_name, AccessMode access_mode, - UCCredentials credentials) + IBCredentials credentials) : Catalog(db_p), internal_name(internal_name), access_mode(access_mode), credentials(std::move(credentials)), schemas(*this) { } @@ -36,7 +36,7 @@ void UCCatalog::DropSchema(ClientContext &context, DropInfo &info) { } void UCCatalog::ScanSchemas(ClientContext &context, std::function callback) { - schemas.Scan(context, [&](CatalogEntry &schema) { callback(schema.Cast()); }); + schemas.Scan(context, [&](CatalogEntry &schema) { callback(schema.Cast()); }); } optional_ptr UCCatalog::GetSchema(CatalogTransaction transaction, const string &schema_name, diff --git a/src/storage/ic_catalog_set.cpp b/src/storage/ic_catalog_set.cpp index 7f7a26c..e8db5c6 100644 --- a/src/storage/ic_catalog_set.cpp +++ b/src/storage/ic_catalog_set.cpp @@ -5,10 +5,10 @@ namespace duckdb { -UCCatalogSet::UCCatalogSet(Catalog &catalog) : catalog(catalog), is_loaded(false) { +IBCatalogSet::IBCatalogSet(Catalog &catalog) : catalog(catalog), is_loaded(false) { } -optional_ptr UCCatalogSet::GetEntry(ClientContext &context, const string &name) { +optional_ptr IBCatalogSet::GetEntry(ClientContext &context, const string &name) { if (!is_loaded) { is_loaded = true; LoadEntries(context); @@ -21,16 +21,16 @@ optional_ptr UCCatalogSet::GetEntry(ClientContext &context, const return entry->second.get(); } -void UCCatalogSet::DropEntry(ClientContext &context, DropInfo &info) { - throw NotImplementedException("UCCatalogSet::DropEntry"); +void IBCatalogSet::DropEntry(ClientContext &context, DropInfo &info) { + throw NotImplementedException("IBCatalogSet::DropEntry"); } -void UCCatalogSet::EraseEntryInternal(const string &name) { +void IBCatalogSet::EraseEntryInternal(const string &name) { lock_guard l(entry_lock); entries.erase(name); } -void UCCatalogSet::Scan(ClientContext &context, const std::function &callback) { +void IBCatalogSet::Scan(ClientContext &context, const std::function &callback) { if (!is_loaded) { is_loaded = true; LoadEntries(context); @@ -41,29 +41,29 @@ void UCCatalogSet::Scan(ClientContext &context, const std::function UCCatalogSet::CreateEntry(unique_ptr entry) { +optional_ptr IBCatalogSet::CreateEntry(unique_ptr entry) { lock_guard l(entry_lock); auto result = entry.get(); if (result->name.empty()) { - throw InternalException("UCCatalogSet::CreateEntry called with empty name"); + throw InternalException("IBCatalogSet::CreateEntry called with empty name"); } entries.insert(make_pair(result->name, std::move(entry))); return result; } -void UCCatalogSet::ClearEntries() { +void IBCatalogSet::ClearEntries() { entries.clear(); is_loaded = false; } -UCInSchemaSet::UCInSchemaSet(UCSchemaEntry &schema) : UCCatalogSet(schema.ParentCatalog()), schema(schema) { +UCInSchemaSet::UCInSchemaSet(IBSchemaEntry &schema) : IBCatalogSet(schema.ParentCatalog()), schema(schema) { } optional_ptr UCInSchemaSet::CreateEntry(unique_ptr entry) { if (!entry->internal) { entry->internal = schema.internal; } - return UCCatalogSet::CreateEntry(std::move(entry)); + return IBCatalogSet::CreateEntry(std::move(entry)); } } // namespace duckdb diff --git a/src/storage/ic_schema_entry.cpp b/src/storage/ic_schema_entry.cpp index 57cd7d2..5fdc4c6 100644 --- a/src/storage/ic_schema_entry.cpp +++ b/src/storage/ic_schema_entry.cpp @@ -13,21 +13,21 @@ namespace duckdb { -UCSchemaEntry::UCSchemaEntry(Catalog &catalog, CreateSchemaInfo &info) +IBSchemaEntry::IBSchemaEntry(Catalog &catalog, CreateSchemaInfo &info) : SchemaCatalogEntry(catalog, info), tables(*this) { } -UCSchemaEntry::~UCSchemaEntry() { +IBSchemaEntry::~IBSchemaEntry() { } -UCTransaction &GetUCTransaction(CatalogTransaction transaction) { +IBTransaction &GetUCTransaction(CatalogTransaction transaction) { if (!transaction.transaction) { throw InternalException("No transaction!?"); } - return transaction.transaction->Cast(); + return transaction.transaction->Cast(); } -optional_ptr UCSchemaEntry::CreateTable(CatalogTransaction transaction, BoundCreateTableInfo &info) { +optional_ptr IBSchemaEntry::CreateTable(CatalogTransaction transaction, BoundCreateTableInfo &info) { auto &base_info = info.Base(); auto table_name = base_info.table; if (base_info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) { @@ -36,7 +36,7 @@ optional_ptr UCSchemaEntry::CreateTable(CatalogTransaction transac return tables.CreateTable(transaction.GetContext(), info); } -optional_ptr UCSchemaEntry::CreateFunction(CatalogTransaction transaction, CreateFunctionInfo &info) { +optional_ptr IBSchemaEntry::CreateFunction(CatalogTransaction transaction, CreateFunctionInfo &info) { throw BinderException("PC databases do not support creating functions"); } @@ -50,7 +50,7 @@ void UCUnqualifyColumnRef(ParsedExpression &expr) { ParsedExpressionIterator::EnumerateChildren(expr, UCUnqualifyColumnRef); } -optional_ptr UCSchemaEntry::CreateIndex(CatalogTransaction transaction, CreateIndexInfo &info, +optional_ptr IBSchemaEntry::CreateIndex(CatalogTransaction transaction, CreateIndexInfo &info, TableCatalogEntry &table) { throw NotImplementedException("CreateIndex"); } @@ -59,7 +59,7 @@ string GetUCCreateView(CreateViewInfo &info) { throw NotImplementedException("GetCreateView"); } -optional_ptr UCSchemaEntry::CreateView(CatalogTransaction transaction, CreateViewInfo &info) { +optional_ptr IBSchemaEntry::CreateView(CatalogTransaction transaction, CreateViewInfo &info) { if (info.sql.empty()) { throw BinderException("Cannot create view in PC that originated from an " "empty SQL statement"); @@ -79,34 +79,34 @@ optional_ptr UCSchemaEntry::CreateView(CatalogTransaction transact return tables.RefreshTable(transaction.GetContext(), info.view_name); } -optional_ptr UCSchemaEntry::CreateType(CatalogTransaction transaction, CreateTypeInfo &info) { +optional_ptr IBSchemaEntry::CreateType(CatalogTransaction transaction, CreateTypeInfo &info) { throw BinderException("PC databases do not support creating types"); } -optional_ptr UCSchemaEntry::CreateSequence(CatalogTransaction transaction, CreateSequenceInfo &info) { +optional_ptr IBSchemaEntry::CreateSequence(CatalogTransaction transaction, CreateSequenceInfo &info) { throw BinderException("PC databases do not support creating sequences"); } -optional_ptr UCSchemaEntry::CreateTableFunction(CatalogTransaction transaction, +optional_ptr IBSchemaEntry::CreateTableFunction(CatalogTransaction transaction, CreateTableFunctionInfo &info) { throw BinderException("PC databases do not support creating table functions"); } -optional_ptr UCSchemaEntry::CreateCopyFunction(CatalogTransaction transaction, +optional_ptr IBSchemaEntry::CreateCopyFunction(CatalogTransaction transaction, CreateCopyFunctionInfo &info) { throw BinderException("PC databases do not support creating copy functions"); } -optional_ptr UCSchemaEntry::CreatePragmaFunction(CatalogTransaction transaction, +optional_ptr IBSchemaEntry::CreatePragmaFunction(CatalogTransaction transaction, CreatePragmaFunctionInfo &info) { throw BinderException("PC databases do not support creating pragma functions"); } -optional_ptr UCSchemaEntry::CreateCollation(CatalogTransaction transaction, CreateCollationInfo &info) { +optional_ptr IBSchemaEntry::CreateCollation(CatalogTransaction transaction, CreateCollationInfo &info) { throw BinderException("PC databases do not support creating collations"); } -void UCSchemaEntry::Alter(CatalogTransaction transaction, AlterInfo &info) { +void IBSchemaEntry::Alter(CatalogTransaction transaction, AlterInfo &info) { if (info.type != AlterType::ALTER_TABLE) { throw BinderException("Only altering tables is supported for now"); } @@ -125,22 +125,22 @@ bool CatalogTypeIsSupported(CatalogType type) { } } -void UCSchemaEntry::Scan(ClientContext &context, CatalogType type, +void IBSchemaEntry::Scan(ClientContext &context, CatalogType type, const std::function &callback) { if (!CatalogTypeIsSupported(type)) { return; } GetCatalogSet(type).Scan(context, callback); } -void UCSchemaEntry::Scan(CatalogType type, const std::function &callback) { +void IBSchemaEntry::Scan(CatalogType type, const std::function &callback) { throw NotImplementedException("Scan without context not supported"); } -void UCSchemaEntry::DropEntry(ClientContext &context, DropInfo &info) { +void IBSchemaEntry::DropEntry(ClientContext &context, DropInfo &info) { GetCatalogSet(info.type).DropEntry(context, info); } -optional_ptr UCSchemaEntry::GetEntry(CatalogTransaction transaction, CatalogType type, +optional_ptr IBSchemaEntry::GetEntry(CatalogTransaction transaction, CatalogType type, const string &name) { if (!CatalogTypeIsSupported(type)) { return nullptr; @@ -148,7 +148,7 @@ optional_ptr UCSchemaEntry::GetEntry(CatalogTransaction transactio return GetCatalogSet(type).GetEntry(transaction.GetContext(), name); } -UCCatalogSet &UCSchemaEntry::GetCatalogSet(CatalogType type) { +IBCatalogSet &IBSchemaEntry::GetCatalogSet(CatalogType type) { switch (type) { case CatalogType::TABLE_ENTRY: case CatalogType::VIEW_ENTRY: diff --git a/src/storage/ic_schema_set.cpp b/src/storage/ic_schema_set.cpp index 3f85a09..77db3e8 100644 --- a/src/storage/ic_schema_set.cpp +++ b/src/storage/ic_schema_set.cpp @@ -7,7 +7,7 @@ namespace duckdb { -UCSchemaSet::UCSchemaSet(Catalog &catalog) : UCCatalogSet(catalog) { +UCSchemaSet::UCSchemaSet(Catalog &catalog) : IBCatalogSet(catalog) { } static bool IsInternalTable(const string &catalog, const string &schema) { @@ -19,14 +19,14 @@ static bool IsInternalTable(const string &catalog, const string &schema) { void UCSchemaSet::LoadEntries(ClientContext &context) { auto &ic_catalog = catalog.Cast(); - auto tables = UCAPI::GetSchemas(catalog.GetName(), ic_catalog.internal_name, ic_catalog.credentials); + auto tables = IBAPI::GetSchemas(catalog.GetName(), ic_catalog.internal_name, ic_catalog.credentials); for (const auto &schema : tables) { CreateSchemaInfo info; info.schema = schema.schema_name; info.internal = IsInternalTable(schema.catalog_name, schema.schema_name); - auto schema_entry = make_uniq(catalog, info); - schema_entry->schema_data = make_uniq(schema); + auto schema_entry = make_uniq(catalog, info); + schema_entry->schema_data = make_uniq(schema); CreateEntry(std::move(schema_entry)); } } diff --git a/src/storage/ic_table_entry.cpp b/src/storage/ic_table_entry.cpp index 2928870..d85f3bc 100644 --- a/src/storage/ic_table_entry.cpp +++ b/src/storage/ic_table_entry.cpp @@ -74,8 +74,8 @@ TableFunction UCTableEntry::GetScanFunction(ClientContext &context, unique_ptrstorage_location.find("file://") != 0) { auto &secret_manager = SecretManager::Get(context); - // Get Credentials from UCAPI - auto table_credentials = UCAPI::GetTableCredentials( + // Get Credentials from IBAPI + auto table_credentials = IBAPI::GetTableCredentials( ic_catalog.internal_name, table_data->schema_name, table_data->name, ic_catalog.credentials); // Inject secret into secret manager scoped to this path diff --git a/src/storage/ic_table_set.cpp b/src/storage/ic_table_set.cpp index 6ea097d..62b834c 100644 --- a/src/storage/ic_table_set.cpp +++ b/src/storage/ic_table_set.cpp @@ -17,20 +17,20 @@ namespace duckdb { -UCTableSet::UCTableSet(UCSchemaEntry &schema) : UCInSchemaSet(schema) { +UCTableSet::UCTableSet(IBSchemaEntry &schema) : UCInSchemaSet(schema) { } -static ColumnDefinition CreateColumnDefinition(ClientContext &context, UCAPIColumnDefinition &coldef) { - return {coldef.name, UCUtils::TypeToLogicalType(context, coldef.type_text)}; +static ColumnDefinition CreateColumnDefinition(ClientContext &context, IBAPIColumnDefinition &coldef) { + return {coldef.name, IBUtils::TypeToLogicalType(context, coldef.type_text)}; } void UCTableSet::LoadEntries(ClientContext &context) { - auto &transaction = UCTransaction::Get(context, catalog); + auto &transaction = IBTransaction::Get(context, catalog); auto &ic_catalog = catalog.Cast(); // TODO: handle out-of-order columns using position property - auto tables = UCAPI::GetTables(catalog.GetName(), catalog.GetDBPath(), schema.name, ic_catalog.credentials); + auto tables = IBAPI::GetTables(catalog.GetName(), catalog.GetDBPath(), schema.name, ic_catalog.credentials); for (auto &table : tables) { D_ASSERT(schema.name == table.schema_name); @@ -41,7 +41,7 @@ void UCTableSet::LoadEntries(ClientContext &context) { info.table = table.name; auto table_entry = make_uniq(catalog, schema, info); - table_entry->table_data = make_uniq(table); + table_entry->table_data = make_uniq(table); CreateEntry(std::move(table_entry)); } @@ -55,7 +55,7 @@ optional_ptr UCTableSet::RefreshTable(ClientContext &context, cons return table_ptr; } -unique_ptr UCTableSet::GetTableInfo(ClientContext &context, UCSchemaEntry &schema, +unique_ptr UCTableSet::GetTableInfo(ClientContext &context, IBSchemaEntry &schema, const string &table_name) { throw NotImplementedException("UCTableSet::CreateTable"); } diff --git a/src/storage/ic_transaction.cpp b/src/storage/ic_transaction.cpp index fbff966..1a971fb 100644 --- a/src/storage/ic_transaction.cpp +++ b/src/storage/ic_transaction.cpp @@ -6,30 +6,30 @@ namespace duckdb { -UCTransaction::UCTransaction(UCCatalog &ic_catalog, TransactionManager &manager, ClientContext &context) +IBTransaction::IBTransaction(UCCatalog &ic_catalog, TransactionManager &manager, ClientContext &context) : Transaction(manager, context), access_mode(ic_catalog.access_mode) { // connection = UCConnection::Open(ic_catalog.path); } -UCTransaction::~UCTransaction() = default; +IBTransaction::~IBTransaction() = default; -void UCTransaction::Start() { +void IBTransaction::Start() { transaction_state = UCTransactionState::TRANSACTION_NOT_YET_STARTED; } -void UCTransaction::Commit() { +void IBTransaction::Commit() { if (transaction_state == UCTransactionState::TRANSACTION_STARTED) { transaction_state = UCTransactionState::TRANSACTION_FINISHED; // connection.Execute("COMMIT"); } } -void UCTransaction::Rollback() { +void IBTransaction::Rollback() { if (transaction_state == UCTransactionState::TRANSACTION_STARTED) { transaction_state = UCTransactionState::TRANSACTION_FINISHED; // connection.Execute("ROLLBACK"); } } -// UCConnection &UCTransaction::GetConnection() { +// UCConnection &IBTransaction::GetConnection() { // if (transaction_state == UCTransactionState::TRANSACTION_NOT_YET_STARTED) { // transaction_state = UCTransactionState::TRANSACTION_STARTED; // string query = "START TRANSACTION"; @@ -41,7 +41,7 @@ void UCTransaction::Rollback() { // return connection; //} -// unique_ptr UCTransaction::Query(const string &query) { +// unique_ptr IBTransaction::Query(const string &query) { // if (transaction_state == UCTransactionState::TRANSACTION_NOT_YET_STARTED) { // transaction_state = UCTransactionState::TRANSACTION_STARTED; // string transaction_start = "START TRANSACTION"; @@ -54,8 +54,8 @@ void UCTransaction::Rollback() { // return connection.Query(query); //} -UCTransaction &UCTransaction::Get(ClientContext &context, Catalog &catalog) { - return Transaction::Get(context, catalog).Cast(); +IBTransaction &IBTransaction::Get(ClientContext &context, Catalog &catalog) { + return Transaction::Get(context, catalog).Cast(); } } // namespace duckdb diff --git a/src/storage/ic_transaction_manager.cpp b/src/storage/ic_transaction_manager.cpp index bf0f319..5e9f15f 100644 --- a/src/storage/ic_transaction_manager.cpp +++ b/src/storage/ic_transaction_manager.cpp @@ -8,7 +8,7 @@ UCTransactionManager::UCTransactionManager(AttachedDatabase &db_p, UCCatalog &ic } Transaction &UCTransactionManager::StartTransaction(ClientContext &context) { - auto transaction = make_uniq(ic_catalog, *this, context); + auto transaction = make_uniq(ic_catalog, *this, context); transaction->Start(); auto &result = *transaction; lock_guard l(transaction_lock); @@ -17,7 +17,7 @@ Transaction &UCTransactionManager::StartTransaction(ClientContext &context) { } ErrorData UCTransactionManager::CommitTransaction(ClientContext &context, Transaction &transaction) { - auto &ic_transaction = transaction.Cast(); + auto &ic_transaction = transaction.Cast(); ic_transaction.Commit(); lock_guard l(transaction_lock); transactions.erase(transaction); @@ -25,14 +25,14 @@ ErrorData UCTransactionManager::CommitTransaction(ClientContext &context, Transa } void UCTransactionManager::RollbackTransaction(Transaction &transaction) { - auto &ic_transaction = transaction.Cast(); + auto &ic_transaction = transaction.Cast(); ic_transaction.Rollback(); lock_guard l(transaction_lock); transactions.erase(transaction); } void UCTransactionManager::Checkpoint(ClientContext &context, bool force) { - auto &transaction = UCTransaction::Get(context, db.GetCatalog()); + auto &transaction = IBTransaction::Get(context, db.GetCatalog()); // auto &db = transaction.GetConnection(); // db.Execute("CHECKPOINT"); } From 63ef6e00baa277171d424a2173dbfb31d70b2c1e Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Tue, 14 Jan 2025 23:07:36 -0500 Subject: [PATCH 04/26] FInished renaming --- src/iceberg_extension.cpp | 12 +++--- src/include/catalog_utils.hpp | 4 +- src/include/storage/ic_catalog.hpp | 12 +++--- src/include/storage/ic_catalog_set.hpp | 4 +- src/include/storage/ic_schema_entry.hpp | 2 +- src/include/storage/ic_schema_set.hpp | 4 +- src/include/storage/ic_table_entry.hpp | 14 +++---- src/include/storage/ic_table_set.hpp | 10 ++--- src/include/storage/ic_transaction.hpp | 13 +++--- .../storage/ic_transaction_manager.hpp | 6 +-- src/storage/ic_catalog.cpp | 42 +++++++++---------- src/storage/ic_catalog_set.cpp | 4 +- src/storage/ic_clear_cache.cpp | 6 +-- src/storage/ic_schema_entry.cpp | 4 +- src/storage/ic_schema_set.cpp | 8 ++-- src/storage/ic_table_entry.cpp | 14 +++---- src/storage/ic_table_set.cpp | 40 +++++++++--------- src/storage/ic_transaction.cpp | 26 ++++++------ src/storage/ic_transaction_manager.cpp | 10 ++--- 19 files changed, 116 insertions(+), 119 deletions(-) diff --git a/src/iceberg_extension.cpp b/src/iceberg_extension.cpp index 5b5d9c5..ca5971c 100644 --- a/src/iceberg_extension.cpp +++ b/src/iceberg_extension.cpp @@ -131,18 +131,18 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in // TODO: Check catalog with name actually exists! - return make_uniq(db, info.path, access_mode, credentials); + return make_uniq(db, info.path, access_mode, credentials); } static unique_ptr CreateTransactionManager(StorageExtensionInfo *storage_info, AttachedDatabase &db, Catalog &catalog) { - auto &ic_catalog = catalog.Cast(); - return make_uniq(db, ic_catalog); + auto &ic_catalog = catalog.Cast(); + return make_uniq(db, ic_catalog); } -class UCCatalogStorageExtension : public StorageExtension { +class IBCatalogStorageExtension : public StorageExtension { public: - UCCatalogStorageExtension() { + IBCatalogStorageExtension() { attach = IcebergCatalogAttach; create_transaction_manager = CreateTransactionManager; } @@ -180,7 +180,7 @@ static void LoadInternal(DatabaseInstance &instance) { SetCatalogSecretParameters(secret_function); ExtensionUtil::RegisterFunction(instance, secret_function); - config.storage_extensions["iceberg"] = make_uniq(); + config.storage_extensions["iceberg"] = make_uniq(); } void IcebergExtension::Load(DuckDB &db) { diff --git a/src/include/catalog_utils.hpp b/src/include/catalog_utils.hpp index f294299..ee11eb8 100644 --- a/src/include/catalog_utils.hpp +++ b/src/include/catalog_utils.hpp @@ -10,10 +10,10 @@ class IBTransaction; enum class IBTypeAnnotation { STANDARD, CAST_TO_VARCHAR, NUMERIC_AS_DOUBLE, CTID, JSONB, FIXED_LENGTH_CHAR }; -struct UCType { +struct IBType { idx_t oid = 0; IBTypeAnnotation info = IBTypeAnnotation::STANDARD; - vector children; + vector children; }; class IBUtils { diff --git a/src/include/storage/ic_catalog.hpp b/src/include/storage/ic_catalog.hpp index 901ee05..0b97be4 100644 --- a/src/include/storage/ic_catalog.hpp +++ b/src/include/storage/ic_catalog.hpp @@ -19,18 +19,18 @@ struct IBCredentials { string token; }; -class UCClearCacheFunction : public TableFunction { +class IBClearCacheFunction : public TableFunction { public: - UCClearCacheFunction(); + IBClearCacheFunction(); static void ClearCacheOnSetting(ClientContext &context, SetScope scope, Value ¶meter); }; -class UCCatalog : public Catalog { +class IBCatalog : public Catalog { public: - explicit UCCatalog(AttachedDatabase &db_p, const string &internal_name, AccessMode access_mode, + explicit IBCatalog(AttachedDatabase &db_p, const string &internal_name, AccessMode access_mode, IBCredentials credentials); - ~UCCatalog(); + ~IBCatalog(); string internal_name; AccessMode access_mode; @@ -73,7 +73,7 @@ class UCCatalog : public Catalog { void DropSchema(ClientContext &context, DropInfo &info) override; private: - UCSchemaSet schemas; + IBSchemaSet schemas; string default_schema; }; diff --git a/src/include/storage/ic_catalog_set.hpp b/src/include/storage/ic_catalog_set.hpp index d2a0099..b4cae9b 100644 --- a/src/include/storage/ic_catalog_set.hpp +++ b/src/include/storage/ic_catalog_set.hpp @@ -34,9 +34,9 @@ class IBCatalogSet { bool is_loaded; }; -class UCInSchemaSet : public IBCatalogSet { +class IBInSchemaSet : public IBCatalogSet { public: - UCInSchemaSet(IBSchemaEntry &schema); + IBInSchemaSet(IBSchemaEntry &schema); optional_ptr CreateEntry(unique_ptr entry) override; diff --git a/src/include/storage/ic_schema_entry.hpp b/src/include/storage/ic_schema_entry.hpp index 3579f59..c8d5e1c 100644 --- a/src/include/storage/ic_schema_entry.hpp +++ b/src/include/storage/ic_schema_entry.hpp @@ -40,7 +40,7 @@ class IBSchemaEntry : public SchemaCatalogEntry { IBCatalogSet &GetCatalogSet(CatalogType type); private: - UCTableSet tables; + IBTableSet tables; }; } // namespace duckdb diff --git a/src/include/storage/ic_schema_set.hpp b/src/include/storage/ic_schema_set.hpp index 365bf0a..de9974a 100644 --- a/src/include/storage/ic_schema_set.hpp +++ b/src/include/storage/ic_schema_set.hpp @@ -7,9 +7,9 @@ namespace duckdb { struct CreateSchemaInfo; -class UCSchemaSet : public IBCatalogSet { +class IBSchemaSet : public IBCatalogSet { public: - explicit UCSchemaSet(Catalog &catalog); + explicit IBSchemaSet(Catalog &catalog); public: optional_ptr CreateSchema(ClientContext &context, CreateSchemaInfo &info); diff --git a/src/include/storage/ic_table_entry.hpp b/src/include/storage/ic_table_entry.hpp index 7480ac5..f648237 100644 --- a/src/include/storage/ic_table_entry.hpp +++ b/src/include/storage/ic_table_entry.hpp @@ -7,14 +7,14 @@ namespace duckdb { -struct UCTableInfo { - UCTableInfo() { +struct IBTableInfo { + IBTableInfo() { create_info = make_uniq(); } - UCTableInfo(const string &schema, const string &table) { + IBTableInfo(const string &schema, const string &table) { create_info = make_uniq(string(), schema, table); } - UCTableInfo(const SchemaCatalogEntry &schema, const string &table) { + IBTableInfo(const SchemaCatalogEntry &schema, const string &table) { create_info = make_uniq((SchemaCatalogEntry &)schema, table); } @@ -25,10 +25,10 @@ struct UCTableInfo { unique_ptr create_info; }; -class UCTableEntry : public TableCatalogEntry { +class IBTableEntry : public TableCatalogEntry { public: - UCTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTableInfo &info); - UCTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, UCTableInfo &info); + IBTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTableInfo &info); + IBTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, IBTableInfo &info); unique_ptr table_data; diff --git a/src/include/storage/ic_table_set.hpp b/src/include/storage/ic_table_set.hpp index e20381d..5bc9940 100644 --- a/src/include/storage/ic_table_set.hpp +++ b/src/include/storage/ic_table_set.hpp @@ -6,17 +6,17 @@ namespace duckdb { struct CreateTableInfo; -class UCResult; +class IBResult; class IBSchemaEntry; -class UCTableSet : public UCInSchemaSet { +class IBTableSet : public IBInSchemaSet { public: - explicit UCTableSet(IBSchemaEntry &schema); + explicit IBTableSet(IBSchemaEntry &schema); public: optional_ptr CreateTable(ClientContext &context, BoundCreateTableInfo &info); - static unique_ptr GetTableInfo(ClientContext &context, IBSchemaEntry &schema, + static unique_ptr GetTableInfo(ClientContext &context, IBSchemaEntry &schema, const string &table_name); optional_ptr RefreshTable(ClientContext &context, const string &table_name); @@ -30,7 +30,7 @@ class UCTableSet : public UCInSchemaSet { void AlterTable(ClientContext &context, AddColumnInfo &info); void AlterTable(ClientContext &context, RemoveColumnInfo &info); - static void AddColumn(ClientContext &context, UCResult &result, UCTableInfo &table_info, idx_t column_offset = 0); + static void AddColumn(ClientContext &context, IBResult &result, IBTableInfo &table_info, idx_t column_offset = 0); }; } // namespace duckdb diff --git a/src/include/storage/ic_transaction.hpp b/src/include/storage/ic_transaction.hpp index 3043741..7dba882 100644 --- a/src/include/storage/ic_transaction.hpp +++ b/src/include/storage/ic_transaction.hpp @@ -4,31 +4,28 @@ #include "duckdb/transaction/transaction.hpp" namespace duckdb { -class UCCatalog; +class IBCatalog; class IBSchemaEntry; -class UCTableEntry; +class IBTableEntry; -enum class UCTransactionState { TRANSACTION_NOT_YET_STARTED, TRANSACTION_STARTED, TRANSACTION_FINISHED }; +enum class IBTransactionState { TRANSACTION_NOT_YET_STARTED, TRANSACTION_STARTED, TRANSACTION_FINISHED }; class IBTransaction : public Transaction { public: - IBTransaction(UCCatalog &ic_catalog, TransactionManager &manager, ClientContext &context); + IBTransaction(IBCatalog &ic_catalog, TransactionManager &manager, ClientContext &context); ~IBTransaction() override; void Start(); void Commit(); void Rollback(); - // UCConnection &GetConnection(); - // unique_ptr Query(const string &query); static IBTransaction &Get(ClientContext &context, Catalog &catalog); AccessMode GetAccessMode() const { return access_mode; } private: - // UCConnection connection; - UCTransactionState transaction_state; + IBTransactionState transaction_state; AccessMode access_mode; }; diff --git a/src/include/storage/ic_transaction_manager.hpp b/src/include/storage/ic_transaction_manager.hpp index 255f65f..4d02af7 100644 --- a/src/include/storage/ic_transaction_manager.hpp +++ b/src/include/storage/ic_transaction_manager.hpp @@ -7,9 +7,9 @@ namespace duckdb { -class UCTransactionManager : public TransactionManager { +class IBTransactionManager : public TransactionManager { public: - UCTransactionManager(AttachedDatabase &db_p, UCCatalog &ic_catalog); + IBTransactionManager(AttachedDatabase &db_p, IBCatalog &ic_catalog); Transaction &StartTransaction(ClientContext &context) override; ErrorData CommitTransaction(ClientContext &context, Transaction &transaction) override; @@ -18,7 +18,7 @@ class UCTransactionManager : public TransactionManager { void Checkpoint(ClientContext &context, bool force = false) override; private: - UCCatalog &ic_catalog; + IBCatalog &ic_catalog; mutex transaction_lock; reference_map_t> transactions; }; diff --git a/src/storage/ic_catalog.cpp b/src/storage/ic_catalog.cpp index efb4e3e..9b1a447 100644 --- a/src/storage/ic_catalog.cpp +++ b/src/storage/ic_catalog.cpp @@ -8,18 +8,18 @@ namespace duckdb { -UCCatalog::UCCatalog(AttachedDatabase &db_p, const string &internal_name, AccessMode access_mode, +IBCatalog::IBCatalog(AttachedDatabase &db_p, const string &internal_name, AccessMode access_mode, IBCredentials credentials) : Catalog(db_p), internal_name(internal_name), access_mode(access_mode), credentials(std::move(credentials)), schemas(*this) { } -UCCatalog::~UCCatalog() = default; +IBCatalog::~IBCatalog() = default; -void UCCatalog::Initialize(bool load_builtin) { +void IBCatalog::Initialize(bool load_builtin) { } -optional_ptr UCCatalog::CreateSchema(CatalogTransaction transaction, CreateSchemaInfo &info) { +optional_ptr IBCatalog::CreateSchema(CatalogTransaction transaction, CreateSchemaInfo &info) { if (info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) { DropInfo try_drop; try_drop.type = CatalogType::SCHEMA_ENTRY; @@ -31,15 +31,15 @@ optional_ptr UCCatalog::CreateSchema(CatalogTransaction transactio return schemas.CreateSchema(transaction.GetContext(), info); } -void UCCatalog::DropSchema(ClientContext &context, DropInfo &info) { +void IBCatalog::DropSchema(ClientContext &context, DropInfo &info) { return schemas.DropEntry(context, info); } -void UCCatalog::ScanSchemas(ClientContext &context, std::function callback) { +void IBCatalog::ScanSchemas(ClientContext &context, std::function callback) { schemas.Scan(context, [&](CatalogEntry &schema) { callback(schema.Cast()); }); } -optional_ptr UCCatalog::GetSchema(CatalogTransaction transaction, const string &schema_name, +optional_ptr IBCatalog::GetSchema(CatalogTransaction transaction, const string &schema_name, OnEntryNotFound if_not_found, QueryErrorContext error_context) { if (schema_name == DEFAULT_SCHEMA) { if (default_schema.empty()) { @@ -55,17 +55,17 @@ optional_ptr UCCatalog::GetSchema(CatalogTransaction transac return reinterpret_cast(entry.get()); } -bool UCCatalog::InMemory() { +bool IBCatalog::InMemory() { return false; } -string UCCatalog::GetDBPath() { +string IBCatalog::GetDBPath() { return internal_name; } -DatabaseSize UCCatalog::GetDatabaseSize(ClientContext &context) { +DatabaseSize IBCatalog::GetDatabaseSize(ClientContext &context) { if (default_schema.empty()) { throw InvalidInputException("Attempting to fetch the database size - but no database was provided " "in the connection string"); @@ -74,29 +74,29 @@ DatabaseSize UCCatalog::GetDatabaseSize(ClientContext &context) { return size; } -void UCCatalog::ClearCache() { +void IBCatalog::ClearCache() { schemas.ClearEntries(); } -unique_ptr UCCatalog::PlanInsert(ClientContext &context, LogicalInsert &op, +unique_ptr IBCatalog::PlanInsert(ClientContext &context, LogicalInsert &op, unique_ptr plan) { - throw NotImplementedException("UCCatalog PlanInsert"); + throw NotImplementedException("IBCatalog PlanInsert"); } -unique_ptr UCCatalog::PlanCreateTableAs(ClientContext &context, LogicalCreateTable &op, +unique_ptr IBCatalog::PlanCreateTableAs(ClientContext &context, LogicalCreateTable &op, unique_ptr plan) { - throw NotImplementedException("UCCatalog PlanCreateTableAs"); + throw NotImplementedException("IBCatalog PlanCreateTableAs"); } -unique_ptr UCCatalog::PlanDelete(ClientContext &context, LogicalDelete &op, +unique_ptr IBCatalog::PlanDelete(ClientContext &context, LogicalDelete &op, unique_ptr plan) { - throw NotImplementedException("UCCatalog PlanDelete"); + throw NotImplementedException("IBCatalog PlanDelete"); } -unique_ptr UCCatalog::PlanUpdate(ClientContext &context, LogicalUpdate &op, +unique_ptr IBCatalog::PlanUpdate(ClientContext &context, LogicalUpdate &op, unique_ptr plan) { - throw NotImplementedException("UCCatalog PlanUpdate"); + throw NotImplementedException("IBCatalog PlanUpdate"); } -unique_ptr UCCatalog::BindCreateIndex(Binder &binder, CreateStatement &stmt, TableCatalogEntry &table, +unique_ptr IBCatalog::BindCreateIndex(Binder &binder, CreateStatement &stmt, TableCatalogEntry &table, unique_ptr plan) { - throw NotImplementedException("UCCatalog BindCreateIndex"); + throw NotImplementedException("IBCatalog BindCreateIndex"); } } // namespace duckdb diff --git a/src/storage/ic_catalog_set.cpp b/src/storage/ic_catalog_set.cpp index e8db5c6..4c0e1e0 100644 --- a/src/storage/ic_catalog_set.cpp +++ b/src/storage/ic_catalog_set.cpp @@ -56,10 +56,10 @@ void IBCatalogSet::ClearEntries() { is_loaded = false; } -UCInSchemaSet::UCInSchemaSet(IBSchemaEntry &schema) : IBCatalogSet(schema.ParentCatalog()), schema(schema) { +IBInSchemaSet::IBInSchemaSet(IBSchemaEntry &schema) : IBCatalogSet(schema.ParentCatalog()), schema(schema) { } -optional_ptr UCInSchemaSet::CreateEntry(unique_ptr entry) { +optional_ptr IBInSchemaSet::CreateEntry(unique_ptr entry) { if (!entry->internal) { entry->internal = schema.internal; } diff --git a/src/storage/ic_clear_cache.cpp b/src/storage/ic_clear_cache.cpp index d4346b8..99db62a 100644 --- a/src/storage/ic_clear_cache.cpp +++ b/src/storage/ic_clear_cache.cpp @@ -28,7 +28,7 @@ static void ClearUCCaches(ClientContext &context) { if (catalog.GetCatalogType() != "iceberg") { continue; } - catalog.Cast().ClearCache(); + catalog.Cast().ClearCache(); } } @@ -41,10 +41,10 @@ static void ClearCacheFunction(ClientContext &context, TableFunctionInput &data_ data.finished = true; } -void UCClearCacheFunction::ClearCacheOnSetting(ClientContext &context, SetScope scope, Value ¶meter) { +void IBClearCacheFunction::ClearCacheOnSetting(ClientContext &context, SetScope scope, Value ¶meter) { ClearUCCaches(context); } -UCClearCacheFunction::UCClearCacheFunction() : TableFunction("pc_clear_cache", {}, ClearCacheFunction, ClearCacheBind) { +IBClearCacheFunction::IBClearCacheFunction() : TableFunction("pc_clear_cache", {}, ClearCacheFunction, ClearCacheBind) { } } // namespace duckdb diff --git a/src/storage/ic_schema_entry.cpp b/src/storage/ic_schema_entry.cpp index 5fdc4c6..41cecaf 100644 --- a/src/storage/ic_schema_entry.cpp +++ b/src/storage/ic_schema_entry.cpp @@ -40,14 +40,14 @@ optional_ptr IBSchemaEntry::CreateFunction(CatalogTransaction tran throw BinderException("PC databases do not support creating functions"); } -void UCUnqualifyColumnRef(ParsedExpression &expr) { +void IBUnqualifyColumnRef(ParsedExpression &expr) { if (expr.type == ExpressionType::COLUMN_REF) { auto &colref = expr.Cast(); auto name = std::move(colref.column_names.back()); colref.column_names = {std::move(name)}; return; } - ParsedExpressionIterator::EnumerateChildren(expr, UCUnqualifyColumnRef); + ParsedExpressionIterator::EnumerateChildren(expr, IBUnqualifyColumnRef); } optional_ptr IBSchemaEntry::CreateIndex(CatalogTransaction transaction, CreateIndexInfo &info, diff --git a/src/storage/ic_schema_set.cpp b/src/storage/ic_schema_set.cpp index 77db3e8..e117633 100644 --- a/src/storage/ic_schema_set.cpp +++ b/src/storage/ic_schema_set.cpp @@ -7,7 +7,7 @@ namespace duckdb { -UCSchemaSet::UCSchemaSet(Catalog &catalog) : IBCatalogSet(catalog) { +IBSchemaSet::IBSchemaSet(Catalog &catalog) : IBCatalogSet(catalog) { } static bool IsInternalTable(const string &catalog, const string &schema) { @@ -16,9 +16,9 @@ static bool IsInternalTable(const string &catalog, const string &schema) { } return false; } -void UCSchemaSet::LoadEntries(ClientContext &context) { +void IBSchemaSet::LoadEntries(ClientContext &context) { - auto &ic_catalog = catalog.Cast(); + auto &ic_catalog = catalog.Cast(); auto tables = IBAPI::GetSchemas(catalog.GetName(), ic_catalog.internal_name, ic_catalog.credentials); for (const auto &schema : tables) { @@ -31,7 +31,7 @@ void UCSchemaSet::LoadEntries(ClientContext &context) { } } -optional_ptr UCSchemaSet::CreateSchema(ClientContext &context, CreateSchemaInfo &info) { +optional_ptr IBSchemaSet::CreateSchema(ClientContext &context, CreateSchemaInfo &info) { throw NotImplementedException("Schema creation"); } diff --git a/src/storage/ic_table_entry.cpp b/src/storage/ic_table_entry.cpp index d85f3bc..e625ee8 100644 --- a/src/storage/ic_table_entry.cpp +++ b/src/storage/ic_table_entry.cpp @@ -19,21 +19,21 @@ namespace duckdb { -UCTableEntry::UCTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTableInfo &info) +IBTableEntry::IBTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTableInfo &info) : TableCatalogEntry(catalog, schema, info) { this->internal = false; } -UCTableEntry::UCTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, UCTableInfo &info) +IBTableEntry::IBTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, IBTableInfo &info) : TableCatalogEntry(catalog, schema, *info.create_info) { this->internal = false; } -unique_ptr UCTableEntry::GetStatistics(ClientContext &context, column_t column_id) { +unique_ptr IBTableEntry::GetStatistics(ClientContext &context, column_t column_id) { return nullptr; } -void UCTableEntry::BindUpdateConstraints(Binder &binder, LogicalGet &, LogicalProjection &, LogicalUpdate &, +void IBTableEntry::BindUpdateConstraints(Binder &binder, LogicalGet &, LogicalProjection &, LogicalUpdate &, ClientContext &) { throw NotImplementedException("BindUpdateConstraints"); } @@ -55,9 +55,9 @@ struct MyIcebergFunctionData : public FunctionData { } }; -TableFunction UCTableEntry::GetScanFunction(ClientContext &context, unique_ptr &bind_data) { +TableFunction IBTableEntry::GetScanFunction(ClientContext &context, unique_ptr &bind_data) { auto &db = DatabaseInstance::GetDatabase(context); - auto &ic_catalog = catalog.Cast(); + auto &ic_catalog = catalog.Cast(); auto &parquet_function_set = ExtensionUtil::GetTableFunction(db, "parquet_scan"); auto parquet_scan_function = parquet_function_set.functions.GetFunctionByArguments(context, {LogicalType::VARCHAR}); @@ -138,7 +138,7 @@ TableFunction UCTableEntry::GetScanFunction(ClientContext &context, unique_ptr(); + auto &ic_catalog = catalog.Cast(); // TODO: handle out-of-order columns using position property auto tables = IBAPI::GetTables(catalog.GetName(), catalog.GetDBPath(), schema.name, ic_catalog.credentials); @@ -40,48 +40,48 @@ void UCTableSet::LoadEntries(ClientContext &context) { } info.table = table.name; - auto table_entry = make_uniq(catalog, schema, info); + auto table_entry = make_uniq(catalog, schema, info); table_entry->table_data = make_uniq(table); CreateEntry(std::move(table_entry)); } } -optional_ptr UCTableSet::RefreshTable(ClientContext &context, const string &table_name) { +optional_ptr IBTableSet::RefreshTable(ClientContext &context, const string &table_name) { auto table_info = GetTableInfo(context, schema, table_name); - auto table_entry = make_uniq(catalog, schema, *table_info); + auto table_entry = make_uniq(catalog, schema, *table_info); auto table_ptr = table_entry.get(); CreateEntry(std::move(table_entry)); return table_ptr; } -unique_ptr UCTableSet::GetTableInfo(ClientContext &context, IBSchemaEntry &schema, +unique_ptr IBTableSet::GetTableInfo(ClientContext &context, IBSchemaEntry &schema, const string &table_name) { - throw NotImplementedException("UCTableSet::CreateTable"); + throw NotImplementedException("IBTableSet::CreateTable"); } -optional_ptr UCTableSet::CreateTable(ClientContext &context, BoundCreateTableInfo &info) { - throw NotImplementedException("UCTableSet::CreateTable"); +optional_ptr IBTableSet::CreateTable(ClientContext &context, BoundCreateTableInfo &info) { + throw NotImplementedException("IBTableSet::CreateTable"); } -void UCTableSet::AlterTable(ClientContext &context, RenameTableInfo &info) { - throw NotImplementedException("UCTableSet::AlterTable"); +void IBTableSet::AlterTable(ClientContext &context, RenameTableInfo &info) { + throw NotImplementedException("IBTableSet::AlterTable"); } -void UCTableSet::AlterTable(ClientContext &context, RenameColumnInfo &info) { - throw NotImplementedException("UCTableSet::AlterTable"); +void IBTableSet::AlterTable(ClientContext &context, RenameColumnInfo &info) { + throw NotImplementedException("IBTableSet::AlterTable"); } -void UCTableSet::AlterTable(ClientContext &context, AddColumnInfo &info) { - throw NotImplementedException("UCTableSet::AlterTable"); +void IBTableSet::AlterTable(ClientContext &context, AddColumnInfo &info) { + throw NotImplementedException("IBTableSet::AlterTable"); } -void UCTableSet::AlterTable(ClientContext &context, RemoveColumnInfo &info) { - throw NotImplementedException("UCTableSet::AlterTable"); +void IBTableSet::AlterTable(ClientContext &context, RemoveColumnInfo &info) { + throw NotImplementedException("IBTableSet::AlterTable"); } -void UCTableSet::AlterTable(ClientContext &context, AlterTableInfo &alter) { - throw NotImplementedException("UCTableSet::AlterTable"); +void IBTableSet::AlterTable(ClientContext &context, AlterTableInfo &alter) { + throw NotImplementedException("IBTableSet::AlterTable"); } } // namespace duckdb diff --git a/src/storage/ic_transaction.cpp b/src/storage/ic_transaction.cpp index 1a971fb..57c2d2b 100644 --- a/src/storage/ic_transaction.cpp +++ b/src/storage/ic_transaction.cpp @@ -6,32 +6,32 @@ namespace duckdb { -IBTransaction::IBTransaction(UCCatalog &ic_catalog, TransactionManager &manager, ClientContext &context) +IBTransaction::IBTransaction(IBCatalog &ic_catalog, TransactionManager &manager, ClientContext &context) : Transaction(manager, context), access_mode(ic_catalog.access_mode) { - // connection = UCConnection::Open(ic_catalog.path); + // connection = IBConnection::Open(ic_catalog.path); } IBTransaction::~IBTransaction() = default; void IBTransaction::Start() { - transaction_state = UCTransactionState::TRANSACTION_NOT_YET_STARTED; + transaction_state = IBTransactionState::TRANSACTION_NOT_YET_STARTED; } void IBTransaction::Commit() { - if (transaction_state == UCTransactionState::TRANSACTION_STARTED) { - transaction_state = UCTransactionState::TRANSACTION_FINISHED; + if (transaction_state == IBTransactionState::TRANSACTION_STARTED) { + transaction_state = IBTransactionState::TRANSACTION_FINISHED; // connection.Execute("COMMIT"); } } void IBTransaction::Rollback() { - if (transaction_state == UCTransactionState::TRANSACTION_STARTED) { - transaction_state = UCTransactionState::TRANSACTION_FINISHED; + if (transaction_state == IBTransactionState::TRANSACTION_STARTED) { + transaction_state = IBTransactionState::TRANSACTION_FINISHED; // connection.Execute("ROLLBACK"); } } -// UCConnection &IBTransaction::GetConnection() { -// if (transaction_state == UCTransactionState::TRANSACTION_NOT_YET_STARTED) { -// transaction_state = UCTransactionState::TRANSACTION_STARTED; +// IBConnection &IBTransaction::GetConnection() { +// if (transaction_state == IBTransactionState::TRANSACTION_NOT_YET_STARTED) { +// transaction_state = IBTransactionState::TRANSACTION_STARTED; // string query = "START TRANSACTION"; // if (access_mode == AccessMode::READ_ONLY) { // query += " READ ONLY"; @@ -41,9 +41,9 @@ void IBTransaction::Rollback() { // return connection; //} -// unique_ptr IBTransaction::Query(const string &query) { -// if (transaction_state == UCTransactionState::TRANSACTION_NOT_YET_STARTED) { -// transaction_state = UCTransactionState::TRANSACTION_STARTED; +// unique_ptr IBTransaction::Query(const string &query) { +// if (transaction_state == IBTransactionState::TRANSACTION_NOT_YET_STARTED) { +// transaction_state = IBTransactionState::TRANSACTION_STARTED; // string transaction_start = "START TRANSACTION"; // if (access_mode == AccessMode::READ_ONLY) { // transaction_start += " READ ONLY"; diff --git a/src/storage/ic_transaction_manager.cpp b/src/storage/ic_transaction_manager.cpp index 5e9f15f..6b2d3b9 100644 --- a/src/storage/ic_transaction_manager.cpp +++ b/src/storage/ic_transaction_manager.cpp @@ -3,11 +3,11 @@ namespace duckdb { -UCTransactionManager::UCTransactionManager(AttachedDatabase &db_p, UCCatalog &ic_catalog) +IBTransactionManager::IBTransactionManager(AttachedDatabase &db_p, IBCatalog &ic_catalog) : TransactionManager(db_p), ic_catalog(ic_catalog) { } -Transaction &UCTransactionManager::StartTransaction(ClientContext &context) { +Transaction &IBTransactionManager::StartTransaction(ClientContext &context) { auto transaction = make_uniq(ic_catalog, *this, context); transaction->Start(); auto &result = *transaction; @@ -16,7 +16,7 @@ Transaction &UCTransactionManager::StartTransaction(ClientContext &context) { return result; } -ErrorData UCTransactionManager::CommitTransaction(ClientContext &context, Transaction &transaction) { +ErrorData IBTransactionManager::CommitTransaction(ClientContext &context, Transaction &transaction) { auto &ic_transaction = transaction.Cast(); ic_transaction.Commit(); lock_guard l(transaction_lock); @@ -24,14 +24,14 @@ ErrorData UCTransactionManager::CommitTransaction(ClientContext &context, Transa return ErrorData(); } -void UCTransactionManager::RollbackTransaction(Transaction &transaction) { +void IBTransactionManager::RollbackTransaction(Transaction &transaction) { auto &ic_transaction = transaction.Cast(); ic_transaction.Rollback(); lock_guard l(transaction_lock); transactions.erase(transaction); } -void UCTransactionManager::Checkpoint(ClientContext &context, bool force) { +void IBTransactionManager::Checkpoint(ClientContext &context, bool force) { auto &transaction = IBTransaction::Get(context, db.GetCatalog()); // auto &db = transaction.GetConnection(); // db.Execute("CHECKPOINT"); From e6f671ad366c071162dee7bba008928a350e77a8 Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Fri, 17 Jan 2025 22:08:34 -0500 Subject: [PATCH 05/26] Delete duplicate --- src/catalog_utils.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/catalog_utils.cpp b/src/catalog_utils.cpp index e6f3e52..87777df 100644 --- a/src/catalog_utils.cpp +++ b/src/catalog_utils.cpp @@ -55,8 +55,6 @@ LogicalType IBUtils::TypeToLogicalType(ClientContext &context, const string &typ return LogicalType::BLOB; } else if (type_text == "date") { return LogicalType::DATE; - } else if (type_text == "timestamp") { - return LogicalType::TIMESTAMP; // TODO: Is this the right timestamp } else if (type_text.find("decimal(") == 0) { size_t spec_end = type_text.find(')'); if (spec_end != string::npos) { From 46a994699e847644e58ffb365ec4de3346581e38 Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Tue, 21 Jan 2025 12:19:13 -0500 Subject: [PATCH 06/26] Lazy load metadata for tables --- CMakeLists.txt | 2 +- src/catalog_api.cpp | 135 ++++++++++++++----------- src/include/catalog_api.hpp | 1 + src/include/storage/ic_catalog_set.hpp | 2 +- src/include/storage/ic_schema_set.hpp | 5 + src/include/storage/ic_table_set.hpp | 6 ++ src/storage/ic_catalog.cpp | 4 +- src/storage/ic_catalog_set.cpp | 18 ++-- src/storage/ic_schema_set.cpp | 29 +++++- src/storage/ic_table_set.cpp | 49 ++++++--- 10 files changed, 161 insertions(+), 90 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0f57eb2..cd7bd25 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,7 +4,7 @@ cmake_minimum_required(VERSION 2.8.12) set(TARGET_NAME iceberg) project(${TARGET_NAME}) -set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED True) set(EXTENSION_NAME ${TARGET_NAME}_extension) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index dd40f7b..d41660f 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -5,6 +5,7 @@ #include #include +#include namespace duckdb { @@ -60,6 +61,34 @@ static void InitializeCurlObject(CURL * curl, const string &token) { SetCurlCAFileInfo(curl); } +template +static TYPE TemplatedTryGetYYJson(duckdb_yyjson::yyjson_val *obj, const string &field, TYPE default_val, + bool fail_on_missing = true) { + auto val = yyjson_obj_get(obj, field.c_str()); + if (val && yyjson_get_type(val) == TYPE_NUM) { + return get_function(val); + } else if (!fail_on_missing) { + return default_val; + } + throw IOException("Invalid field found while parsing field: " + field); +} + +static uint64_t TryGetNumFromObject(duckdb_yyjson::yyjson_val *obj, const string &field, bool fail_on_missing = true, + uint64_t default_val = 0) { + return TemplatedTryGetYYJson(obj, field, default_val, + fail_on_missing); +} +static bool TryGetBoolFromObject(duckdb_yyjson::yyjson_val *obj, const string &field, bool fail_on_missing = false, + bool default_val = false) { + return TemplatedTryGetYYJson(obj, field, default_val, + fail_on_missing); +} +static string TryGetStrFromObject(duckdb_yyjson::yyjson_val *obj, const string &field, bool fail_on_missing = true, + const char *default_val = "") { + return TemplatedTryGetYYJson(obj, field, default_val, + fail_on_missing); +} + static string GetRequest(const string &url, const string &token = "", curl_slist *extra_headers = NULL) { CURL *curl; CURLcode res; @@ -145,35 +174,15 @@ static duckdb_yyjson::yyjson_val *GetTableMetadata(const string &internal, const credentials.token, extra_headers); duckdb_yyjson::yyjson_doc *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); - return yyjson_doc_get_root(doc); -} - -template -static TYPE TemplatedTryGetYYJson(duckdb_yyjson::yyjson_val *obj, const string &field, TYPE default_val, - bool fail_on_missing = true) { - auto val = yyjson_obj_get(obj, field.c_str()); - if (val && yyjson_get_type(val) == TYPE_NUM) { - return get_function(val); - } else if (!fail_on_missing) { - return default_val; + + auto *root = yyjson_doc_get_root(doc); + auto *error = yyjson_obj_get(root, "error"); + if (error != NULL) { + string err_msg = TryGetStrFromObject(error, "message"); + throw std::runtime_error(err_msg); } - throw IOException("Invalid field found while parsing field: " + field); -} -static uint64_t TryGetNumFromObject(duckdb_yyjson::yyjson_val *obj, const string &field, bool fail_on_missing = true, - uint64_t default_val = 0) { - return TemplatedTryGetYYJson(obj, field, default_val, - fail_on_missing); -} -static bool TryGetBoolFromObject(duckdb_yyjson::yyjson_val *obj, const string &field, bool fail_on_missing = false, - bool default_val = false) { - return TemplatedTryGetYYJson(obj, field, default_val, - fail_on_missing); -} -static string TryGetStrFromObject(duckdb_yyjson::yyjson_val *obj, const string &field, bool fail_on_missing = true, - const char *default_val = "") { - return TemplatedTryGetYYJson(obj, field, default_val, - fail_on_missing); + return yyjson_doc_get_root(doc); } void IBAPI::InitializeCurl() { @@ -228,30 +237,22 @@ string IBAPI::GetToken(string id, string secret, string endpoint) { return TryGetStrFromObject(root, "access_token"); } -vector IBAPI::GetTables(const string &catalog, const string &internal, const string &schema, IBCredentials credentials) { - vector result; - - auto api_result = GetRequest(credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables", credentials.token); - - // Read JSON and get root - auto *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); - auto *root = yyjson_doc_get_root(doc); - - // Get root["hits"], iterate over the array - auto *tables = yyjson_obj_get(root, "identifiers"); - size_t idx, max; - duckdb_yyjson::yyjson_val *table; - yyjson_arr_foreach(tables, idx, max, table) { - IBAPITable table_result; - table_result.catalog_name = catalog; - table_result.schema_name = schema; - table_result.name = TryGetStrFromObject(table, "name"); - - auto *metadata_root = GetTableMetadata(internal, schema, table_result.name, credentials); - table_result.data_source_format = "ICEBERG"; +IBAPITable IBAPI::GetTable( + const string &catalog, const string &internal, const string &schema, const string &table, std::optional credentials) { + + IBAPITable table_result; + table_result.catalog_name = catalog; + table_result.schema_name = schema; + table_result.name = table; + table_result.data_source_format = "ICEBERG"; + table_result.table_id = "uuid-" + schema + "-" + "table"; + std::replace(table_result.table_id.begin(), table_result.table_id.end(), '_', '-'); + + if (credentials) { + auto *metadata_root = GetTableMetadata(internal, schema, table_result.name, *credentials); table_result.storage_location = TryGetStrFromObject(metadata_root, "metadata-location"); auto *metadata = yyjson_obj_get(metadata_root, "metadata"); - table_result.table_id = TryGetStrFromObject(metadata, "table-uuid"); + //table_result.table_id = TryGetStrFromObject(metadata, "table-uuid"); uint64_t current_schema_id = TryGetNumFromObject(metadata, "current-schema-id"); auto *schemas = yyjson_obj_get(metadata, "schemas"); @@ -269,17 +270,39 @@ vector IBAPI::GetTables(const string &catalog, const string &interna auto column_definition = ParseColumnDefinition(col); table_result.columns.push_back(column_definition); } - result.push_back(table_result); - - // This could take a while so display found tables to indicate progress - // TODO: Is there a way to show progress some other way? - std::cout << table_result.schema_name << " | " << table_result.name << std::endl; } } if (!found) { throw InternalException("Current schema not found"); } + } else { + // Skip fetching metadata, we'll do it later when we access the table + IBAPIColumnDefinition col; + col.name = "__"; + col.type_text = "int"; + col.precision = -1; //TODO: TryGetNumFromObject(column_def, "type_precision"); + col.scale = -1; //TODO: TryGetNumFromObject(column_def, "type_scale"); + col.position = 0; + table_result.columns.push_back(col); + } + + return table_result; +} + +// TODO: handle out-of-order columns using position property +vector IBAPI::GetTables(const string &catalog, const string &internal, const string &schema, IBCredentials credentials) { + vector result; + + auto api_result = GetRequest(credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables", credentials.token); + auto *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); + auto *root = yyjson_doc_get_root(doc); + auto *tables = yyjson_obj_get(root, "identifiers"); + size_t idx, max; + duckdb_yyjson::yyjson_val *table; + yyjson_arr_foreach(tables, idx, max, table) { + auto table_result = GetTable(catalog, internal, schema, TryGetStrFromObject(table, "name"), std::nullopt); + result.push_back(table_result); } return result; @@ -291,23 +314,19 @@ vector IBAPI::GetSchemas(const string &catalog, const string &inter auto api_result = GetRequest(credentials.endpoint + "/v1/" + internal + "/namespaces", credentials.token); - // Read JSON and get root duckdb_yyjson::yyjson_doc *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); duckdb_yyjson::yyjson_val *root = yyjson_doc_get_root(doc); - auto *error = yyjson_obj_get(root, "error"); if (error != NULL) { string err_msg = TryGetStrFromObject(error, "message"); throw std::runtime_error(err_msg); } - // Get root["hits"], iterate over the array auto *schemas = yyjson_obj_get(root, "namespaces"); size_t idx, max; duckdb_yyjson::yyjson_val *schema; yyjson_arr_foreach(schemas, idx, max, schema) { IBAPISchema schema_result; - schema_result.catalog_name = catalog; duckdb_yyjson::yyjson_val *value = yyjson_arr_get(schema, 0); schema_result.schema_name = yyjson_get_str(value); diff --git a/src/include/catalog_api.hpp b/src/include/catalog_api.hpp index e8ef25b..89de9a2 100644 --- a/src/include/catalog_api.hpp +++ b/src/include/catalog_api.hpp @@ -47,6 +47,7 @@ class IBAPI { static IBAPITableCredentials GetTableCredentials(const string &internal, const string &schema, const string &table, IBCredentials credentials); static vector GetCatalogs(const string &catalog, IBCredentials credentials); static vector GetTables(const string &catalog, const string &internal, const string &schema, IBCredentials credentials); + static IBAPITable GetTable(const string &catalog, const string &internal, const string &schema, const string &table, std::optional credentials); static vector GetSchemas(const string &catalog, const string &internal, IBCredentials credentials); static vector GetTablesInSchema(const string &catalog, const string &schema, IBCredentials credentials); static string GetToken(string id, string secret, string endpoint); diff --git a/src/include/storage/ic_catalog_set.hpp b/src/include/storage/ic_catalog_set.hpp index b4cae9b..d31578f 100644 --- a/src/include/storage/ic_catalog_set.hpp +++ b/src/include/storage/ic_catalog_set.hpp @@ -22,6 +22,7 @@ class IBCatalogSet { protected: virtual void LoadEntries(ClientContext &context) = 0; + virtual void FillEntry(ClientContext &context, unique_ptr &entry) = 0; void EraseEntryInternal(const string &name); @@ -31,7 +32,6 @@ class IBCatalogSet { private: mutex entry_lock; case_insensitive_map_t> entries; - bool is_loaded; }; class IBInSchemaSet : public IBCatalogSet { diff --git a/src/include/storage/ic_schema_set.hpp b/src/include/storage/ic_schema_set.hpp index de9974a..80aca8e 100644 --- a/src/include/storage/ic_schema_set.hpp +++ b/src/include/storage/ic_schema_set.hpp @@ -13,9 +13,14 @@ class IBSchemaSet : public IBCatalogSet { public: optional_ptr CreateSchema(ClientContext &context, CreateSchemaInfo &info); + void DropSchema(ClientContext &context, DropInfo &info); protected: void LoadEntries(ClientContext &context) override; + void FillEntry(ClientContext &context, unique_ptr &entry) override; + +private: + bool is_loaded; }; } // namespace duckdb diff --git a/src/include/storage/ic_table_set.hpp b/src/include/storage/ic_table_set.hpp index 5bc9940..4207bdd 100644 --- a/src/include/storage/ic_table_set.hpp +++ b/src/include/storage/ic_table_set.hpp @@ -24,6 +24,7 @@ class IBTableSet : public IBInSchemaSet { protected: void LoadEntries(ClientContext &context) override; + void FillEntry(ClientContext &context, unique_ptr &entry) override; void AlterTable(ClientContext &context, RenameTableInfo &info); void AlterTable(ClientContext &context, RenameColumnInfo &info); @@ -31,6 +32,11 @@ class IBTableSet : public IBInSchemaSet { void AlterTable(ClientContext &context, RemoveColumnInfo &info); static void AddColumn(ClientContext &context, IBResult &result, IBTableInfo &table_info, idx_t column_offset = 0); + +private: + unique_ptr _CreateCatalogEntry(ClientContext &context, IBAPITable table); + + bool is_loaded; }; } // namespace duckdb diff --git a/src/storage/ic_catalog.cpp b/src/storage/ic_catalog.cpp index 9b1a447..be7276a 100644 --- a/src/storage/ic_catalog.cpp +++ b/src/storage/ic_catalog.cpp @@ -26,13 +26,13 @@ optional_ptr IBCatalog::CreateSchema(CatalogTransaction transactio try_drop.name = info.schema; try_drop.if_not_found = OnEntryNotFound::RETURN_NULL; try_drop.cascade = false; - schemas.DropEntry(transaction.GetContext(), try_drop); + schemas.DropSchema(transaction.GetContext(), try_drop); } return schemas.CreateSchema(transaction.GetContext(), info); } void IBCatalog::DropSchema(ClientContext &context, DropInfo &info) { - return schemas.DropEntry(context, info); + return schemas.DropSchema(context, info); } void IBCatalog::ScanSchemas(ClientContext &context, std::function callback) { diff --git a/src/storage/ic_catalog_set.cpp b/src/storage/ic_catalog_set.cpp index 4c0e1e0..903f357 100644 --- a/src/storage/ic_catalog_set.cpp +++ b/src/storage/ic_catalog_set.cpp @@ -5,24 +5,22 @@ namespace duckdb { -IBCatalogSet::IBCatalogSet(Catalog &catalog) : catalog(catalog), is_loaded(false) { +IBCatalogSet::IBCatalogSet(Catalog &catalog) : catalog(catalog) { } optional_ptr IBCatalogSet::GetEntry(ClientContext &context, const string &name) { - if (!is_loaded) { - is_loaded = true; - LoadEntries(context); - } + LoadEntries(context); lock_guard l(entry_lock); auto entry = entries.find(name); if (entry == entries.end()) { return nullptr; } + FillEntry(context, entry->second); return entry->second.get(); } void IBCatalogSet::DropEntry(ClientContext &context, DropInfo &info) { - throw NotImplementedException("IBCatalogSet::DropEntry"); + EraseEntryInternal(info.name); } void IBCatalogSet::EraseEntryInternal(const string &name) { @@ -31,10 +29,8 @@ void IBCatalogSet::EraseEntryInternal(const string &name) { } void IBCatalogSet::Scan(ClientContext &context, const std::function &callback) { - if (!is_loaded) { - is_loaded = true; - LoadEntries(context); - } + LoadEntries(context); + lock_guard l(entry_lock); for (auto &entry : entries) { callback(*entry.second); @@ -53,7 +49,7 @@ optional_ptr IBCatalogSet::CreateEntry(unique_ptr en void IBCatalogSet::ClearEntries() { entries.clear(); - is_loaded = false; + // TODO: is_loaded = false; } IBInSchemaSet::IBInSchemaSet(IBSchemaEntry &schema) : IBCatalogSet(schema.ParentCatalog()), schema(schema) { diff --git a/src/storage/ic_schema_set.cpp b/src/storage/ic_schema_set.cpp index e117633..948774b 100644 --- a/src/storage/ic_schema_set.cpp +++ b/src/storage/ic_schema_set.cpp @@ -5,9 +5,11 @@ #include "duckdb/parser/parsed_data/create_schema_info.hpp" #include "duckdb/catalog/catalog.hpp" +#include + namespace duckdb { -IBSchemaSet::IBSchemaSet(Catalog &catalog) : IBCatalogSet(catalog) { +IBSchemaSet::IBSchemaSet(Catalog &catalog) : IBCatalogSet(catalog), is_loaded(false) { } static bool IsInternalTable(const string &catalog, const string &schema) { @@ -16,12 +18,17 @@ static bool IsInternalTable(const string &catalog, const string &schema) { } return false; } + void IBSchemaSet::LoadEntries(ClientContext &context) { + if (is_loaded) { + return; + } + is_loaded = true; auto &ic_catalog = catalog.Cast(); - auto tables = IBAPI::GetSchemas(catalog.GetName(), ic_catalog.internal_name, ic_catalog.credentials); + auto schemas = IBAPI::GetSchemas(catalog.GetName(), ic_catalog.internal_name, ic_catalog.credentials); - for (const auto &schema : tables) { + for (const auto &schema : schemas) { CreateSchemaInfo info; info.schema = schema.schema_name; info.internal = IsInternalTable(schema.catalog_name, schema.schema_name); @@ -31,8 +38,22 @@ void IBSchemaSet::LoadEntries(ClientContext &context) { } } +void IBSchemaSet::FillEntry(ClientContext &context, unique_ptr &entry) { + // Nothing to do +} + optional_ptr IBSchemaSet::CreateSchema(ClientContext &context, CreateSchemaInfo &info) { - throw NotImplementedException("Schema creation"); + // TODO: throw NotImplementedException("Schema creation"); + std::cout << " >> Create schema" << std::endl; + // CreateEntry(...) } +void IBSchemaSet::DropSchema(ClientContext &context, DropInfo &info) { + // TODO + std::cout << " >> Drop schema" << std::endl; + + DropEntry(context, info); +} + + } // namespace duckdb diff --git a/src/storage/ic_table_set.cpp b/src/storage/ic_table_set.cpp index ec88263..67d1cb2 100644 --- a/src/storage/ic_table_set.cpp +++ b/src/storage/ic_table_set.cpp @@ -15,35 +15,58 @@ #include "storage/ic_schema_entry.hpp" #include "duckdb/parser/parser.hpp" +#include + namespace duckdb { -IBTableSet::IBTableSet(IBSchemaEntry &schema) : IBInSchemaSet(schema) { +IBTableSet::IBTableSet(IBSchemaEntry &schema) : IBInSchemaSet(schema), is_loaded(false) { } static ColumnDefinition CreateColumnDefinition(ClientContext &context, IBAPIColumnDefinition &coldef) { return {coldef.name, IBUtils::TypeToLogicalType(context, coldef.type_text)}; } +unique_ptr IBTableSet::_CreateCatalogEntry(ClientContext &context, IBAPITable table) { + D_ASSERT(schema.name == table.schema_name); + CreateTableInfo info; + info.table = table.name; + + for (auto &col : table.columns) { + info.columns.AddColumn(CreateColumnDefinition(context, col)); + } + + auto table_entry = make_uniq(catalog, schema, info); + table_entry->table_data = make_uniq(table); + return table_entry; +} + +void IBTableSet::FillEntry(ClientContext &context, unique_ptr &entry) { + auto* derived = static_cast(entry.get()); + if (!derived->table_data->storage_location.empty()) { + std::cout << "HERE: " << derived->table_data->storage_location << std::endl; + return; + } + + auto &ic_catalog = catalog.Cast(); + auto table = IBAPI::GetTable(catalog.GetName(), catalog.GetDBPath(), schema.name, entry->name, ic_catalog.credentials); + entry = _CreateCatalogEntry(context, table); +} + void IBTableSet::LoadEntries(ClientContext &context) { - auto &transaction = IBTransaction::Get(context, catalog); + if (is_loaded) { + return; + } + is_loaded = true; + auto &transaction = IBTransaction::Get(context, catalog); auto &ic_catalog = catalog.Cast(); // TODO: handle out-of-order columns using position property auto tables = IBAPI::GetTables(catalog.GetName(), catalog.GetDBPath(), schema.name, ic_catalog.credentials); for (auto &table : tables) { - D_ASSERT(schema.name == table.schema_name); - CreateTableInfo info; - for (auto &col : table.columns) { - info.columns.AddColumn(CreateColumnDefinition(context, col)); - } - - info.table = table.name; - auto table_entry = make_uniq(catalog, schema, info); - table_entry->table_data = make_uniq(table); - - CreateEntry(std::move(table_entry)); + auto entry = _CreateCatalogEntry(context, table); + CreateEntry(std::move(entry)); } } From 26d9f1bf4e5acc4df45634c3819ec35df0ae91c0 Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Tue, 21 Jan 2025 12:27:28 -0500 Subject: [PATCH 07/26] Extract precision and scale --- src/catalog_api.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index d41660f..bcdf7dc 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -195,13 +195,11 @@ vector IBAPI::GetCatalogs(const string &catalog, IBCredentials credentia static IBAPIColumnDefinition ParseColumnDefinition(duckdb_yyjson::yyjson_val *column_def) { IBAPIColumnDefinition result; - result.name = TryGetStrFromObject(column_def, "name"); result.type_text = TryGetStrFromObject(column_def, "type"); - result.precision = -1; //TODO: TryGetNumFromObject(column_def, "type_precision"); - result.scale = -1; //TODO: TryGetNumFromObject(column_def, "type_scale"); + result.precision = (result.type_text == "decimal") ? TryGetNumFromObject(column_def, "type_precision") : -1; + result.scale = (result.type_text == "decimal") ? TryGetNumFromObject(column_def, "type_scale") : -1; result.position = TryGetNumFromObject(column_def, "id") - 1; - return result; } @@ -281,8 +279,8 @@ IBAPITable IBAPI::GetTable( IBAPIColumnDefinition col; col.name = "__"; col.type_text = "int"; - col.precision = -1; //TODO: TryGetNumFromObject(column_def, "type_precision"); - col.scale = -1; //TODO: TryGetNumFromObject(column_def, "type_scale"); + col.precision = -1; + col.scale = -1; col.position = 0; table_result.columns.push_back(col); } From 645e83c6488424b58852aa72232434e59e63cf13 Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Tue, 21 Jan 2025 13:28:52 -0500 Subject: [PATCH 08/26] Get rid of is_loaded --- src/include/storage/ic_catalog_set.hpp | 2 +- src/include/storage/ic_schema_set.hpp | 3 --- src/include/storage/ic_table_set.hpp | 2 -- src/storage/ic_catalog_set.cpp | 1 - src/storage/ic_schema_set.cpp | 6 ++---- src/storage/ic_table_set.cpp | 8 ++------ 6 files changed, 5 insertions(+), 17 deletions(-) diff --git a/src/include/storage/ic_catalog_set.hpp b/src/include/storage/ic_catalog_set.hpp index d31578f..f486842 100644 --- a/src/include/storage/ic_catalog_set.hpp +++ b/src/include/storage/ic_catalog_set.hpp @@ -28,10 +28,10 @@ class IBCatalogSet { protected: Catalog &catalog; + case_insensitive_map_t> entries; private: mutex entry_lock; - case_insensitive_map_t> entries; }; class IBInSchemaSet : public IBCatalogSet { diff --git a/src/include/storage/ic_schema_set.hpp b/src/include/storage/ic_schema_set.hpp index 80aca8e..6e8332f 100644 --- a/src/include/storage/ic_schema_set.hpp +++ b/src/include/storage/ic_schema_set.hpp @@ -18,9 +18,6 @@ class IBSchemaSet : public IBCatalogSet { protected: void LoadEntries(ClientContext &context) override; void FillEntry(ClientContext &context, unique_ptr &entry) override; - -private: - bool is_loaded; }; } // namespace duckdb diff --git a/src/include/storage/ic_table_set.hpp b/src/include/storage/ic_table_set.hpp index 4207bdd..5803086 100644 --- a/src/include/storage/ic_table_set.hpp +++ b/src/include/storage/ic_table_set.hpp @@ -35,8 +35,6 @@ class IBTableSet : public IBInSchemaSet { private: unique_ptr _CreateCatalogEntry(ClientContext &context, IBAPITable table); - - bool is_loaded; }; } // namespace duckdb diff --git a/src/storage/ic_catalog_set.cpp b/src/storage/ic_catalog_set.cpp index 903f357..cc00d54 100644 --- a/src/storage/ic_catalog_set.cpp +++ b/src/storage/ic_catalog_set.cpp @@ -49,7 +49,6 @@ optional_ptr IBCatalogSet::CreateEntry(unique_ptr en void IBCatalogSet::ClearEntries() { entries.clear(); - // TODO: is_loaded = false; } IBInSchemaSet::IBInSchemaSet(IBSchemaEntry &schema) : IBCatalogSet(schema.ParentCatalog()), schema(schema) { diff --git a/src/storage/ic_schema_set.cpp b/src/storage/ic_schema_set.cpp index 948774b..2b3fd1f 100644 --- a/src/storage/ic_schema_set.cpp +++ b/src/storage/ic_schema_set.cpp @@ -9,7 +9,7 @@ namespace duckdb { -IBSchemaSet::IBSchemaSet(Catalog &catalog) : IBCatalogSet(catalog), is_loaded(false) { +IBSchemaSet::IBSchemaSet(Catalog &catalog) : IBCatalogSet(catalog) { } static bool IsInternalTable(const string &catalog, const string &schema) { @@ -20,14 +20,12 @@ static bool IsInternalTable(const string &catalog, const string &schema) { } void IBSchemaSet::LoadEntries(ClientContext &context) { - if (is_loaded) { + if (!entries.empty()) { return; } - is_loaded = true; auto &ic_catalog = catalog.Cast(); auto schemas = IBAPI::GetSchemas(catalog.GetName(), ic_catalog.internal_name, ic_catalog.credentials); - for (const auto &schema : schemas) { CreateSchemaInfo info; info.schema = schema.schema_name; diff --git a/src/storage/ic_table_set.cpp b/src/storage/ic_table_set.cpp index 67d1cb2..5c211cb 100644 --- a/src/storage/ic_table_set.cpp +++ b/src/storage/ic_table_set.cpp @@ -15,11 +15,9 @@ #include "storage/ic_schema_entry.hpp" #include "duckdb/parser/parser.hpp" -#include - namespace duckdb { -IBTableSet::IBTableSet(IBSchemaEntry &schema) : IBInSchemaSet(schema), is_loaded(false) { +IBTableSet::IBTableSet(IBSchemaEntry &schema) : IBInSchemaSet(schema) { } static ColumnDefinition CreateColumnDefinition(ClientContext &context, IBAPIColumnDefinition &coldef) { @@ -43,7 +41,6 @@ unique_ptr IBTableSet::_CreateCatalogEntry(ClientContext &context, void IBTableSet::FillEntry(ClientContext &context, unique_ptr &entry) { auto* derived = static_cast(entry.get()); if (!derived->table_data->storage_location.empty()) { - std::cout << "HERE: " << derived->table_data->storage_location << std::endl; return; } @@ -53,11 +50,10 @@ void IBTableSet::FillEntry(ClientContext &context, unique_ptr &ent } void IBTableSet::LoadEntries(ClientContext &context) { - if (is_loaded) { + if (!entries.empty()) { return; } - is_loaded = true; auto &transaction = IBTransaction::Get(context, catalog); auto &ic_catalog = catalog.Cast(); From 3a63af6fc73706d0ffeb58f07eb8e178c867790e Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Wed, 22 Jan 2025 02:59:23 -0500 Subject: [PATCH 09/26] Finish implementing create/drop schema --- src/catalog_api.cpp | 127 +++++++++++++++++++++------------- src/include/catalog_api.hpp | 2 + src/storage/ic_schema_set.cpp | 23 +++--- 3 files changed, 91 insertions(+), 61 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index bcdf7dc..d0790af 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -12,7 +12,7 @@ namespace duckdb { //! We use a global here to store the path that is selected on the IBAPI::InitializeCurl call static string SELECTED_CURL_CERT_PATH = ""; -static size_t GetRequestWriteCallback(void *contents, size_t size, size_t nmemb, void *userp) { +static size_t RequestWriteCallback(void *contents, size_t size, size_t nmemb, void *userp) { ((std::string *)userp)->append((char *)contents, size * nmemb); return size * nmemb; } @@ -89,6 +89,36 @@ static string TryGetStrFromObject(duckdb_yyjson::yyjson_val *obj, const string & fail_on_missing); } +static string DeleteRequest(const string &url, const string &token = "", curl_slist *extra_headers = NULL) { + CURL *curl; + CURLcode res; + string readBuffer; + + curl = curl_easy_init(); + if (curl) { + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, RequestWriteCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer); + + if(extra_headers) { + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, extra_headers); + } + + InitializeCurlObject(curl, token); + res = curl_easy_perform(curl); + curl_easy_cleanup(curl); + + if (res != CURLcode::CURLE_OK) { + string error = curl_easy_strerror(res); + throw IOException("Curl DELETE Request to '%s' failed with error: '%s'", url, error); + } + + return readBuffer; + } + throw InternalException("Failed to initialize curl"); +} + static string GetRequest(const string &url, const string &token = "", curl_slist *extra_headers = NULL) { CURL *curl; CURLcode res; @@ -97,11 +127,13 @@ static string GetRequest(const string &url, const string &token = "", curl_slist curl = curl_easy_init(); if (curl) { curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, GetRequestWriteCallback); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, RequestWriteCallback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer); + if(extra_headers) { curl_easy_setopt(curl, CURLOPT_HTTPHEADER, extra_headers); } + InitializeCurlObject(curl, token); res = curl_easy_perform(curl); curl_easy_cleanup(curl); @@ -110,34 +142,34 @@ static string GetRequest(const string &url, const string &token = "", curl_slist string error = curl_easy_strerror(res); throw IOException("Curl Request to '%s' failed with error: '%s'", url, error); } + return readBuffer; } throw InternalException("Failed to initialize curl"); } -static string PostRequest(const string &url, const string &post_data, curl_slist *extra_headers = NULL) { +static string PostRequest( + const string &url, + const string &post_data, + const string &content_type = "x-www-form-urlencoded", + const string &token = "", + curl_slist *extra_headers = NULL) { string readBuffer; CURL *curl = curl_easy_init(); if (!curl) { throw InternalException("Failed to initialize curl"); } - // Set the URL curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - - // Set POST method curl_easy_setopt(curl, CURLOPT_POST, 1L); - - // Set POST data curl_easy_setopt(curl, CURLOPT_POSTFIELDS, post_data.c_str()); - - // Set up response handling - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, GetRequestWriteCallback); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, RequestWriteCallback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer); // Create default headers for content type struct curl_slist *headers = NULL; - headers = curl_slist_append(headers, "Content-Type: application/x-www-form-urlencoded"); + const string content_type_str = "Content-Type: application/" + content_type; + headers = curl_slist_append(headers, content_type_str.c_str()); // Append any extra headers if (extra_headers) { @@ -149,7 +181,7 @@ static string PostRequest(const string &url, const string &post_data, curl_slist } curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - InitializeCurlObject(curl, ""); + InitializeCurlObject(curl, token); // Perform the request CURLcode res = curl_easy_perform(curl); @@ -163,26 +195,27 @@ static string PostRequest(const string &url, const string &post_data, curl_slist throw IOException("Curl Request to '%s' failed with error: '%s'", url, error); } return readBuffer; - } -static duckdb_yyjson::yyjson_val *GetTableMetadata(const string &internal, const string &schema, const string &table, IBCredentials credentials) { - struct curl_slist *extra_headers = NULL; - extra_headers = curl_slist_append(extra_headers, "X-Iceberg-Access-Delegation: vended-credentials"); - auto api_result = GetRequest( - credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables/" + table, - credentials.token, - extra_headers); - duckdb_yyjson::yyjson_doc *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); - +static duckdb_yyjson::yyjson_val *api_result_to_doc(const string &api_result) { + auto *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); auto *root = yyjson_doc_get_root(doc); auto *error = yyjson_obj_get(root, "error"); if (error != NULL) { string err_msg = TryGetStrFromObject(error, "message"); throw std::runtime_error(err_msg); } + return root; +} - return yyjson_doc_get_root(doc); +static duckdb_yyjson::yyjson_val *GetTableMetadata(const string &internal, const string &schema, const string &table, IBCredentials credentials) { + struct curl_slist *extra_headers = NULL; + extra_headers = curl_slist_append(extra_headers, "X-Iceberg-Access-Delegation: vended-credentials"); + auto api_result = GetRequest( + credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables/" + table, + credentials.token, + extra_headers); + return api_result_to_doc(api_result); } void IBAPI::InitializeCurl() { @@ -205,7 +238,6 @@ static IBAPIColumnDefinition ParseColumnDefinition(duckdb_yyjson::yyjson_val *co IBAPITableCredentials IBAPI::GetTableCredentials(const string &internal, const string &schema, const string &table, IBCredentials credentials) { IBAPITableCredentials result; - duckdb_yyjson::yyjson_val *root = GetTableMetadata(internal, schema, table, credentials); auto *aws_temp_credentials = yyjson_obj_get(root, "config"); @@ -221,17 +253,7 @@ IBAPITableCredentials IBAPI::GetTableCredentials(const string &internal, const s string IBAPI::GetToken(string id, string secret, string endpoint) { string post_data = "grant_type=client_credentials&client_id=" + id + "&client_secret=" + secret + "&scope=PRINCIPAL_ROLE:ALL"; string api_result = PostRequest(endpoint + "/v1/oauth/tokens", post_data); - - // Read JSON and get root - auto *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); - auto *root = yyjson_doc_get_root(doc); - - auto *error = yyjson_obj_get(root, "error"); - if (error != NULL) { - string err_msg = TryGetStrFromObject(error, "message"); - throw std::runtime_error(err_msg); - } - + auto *root = api_result_to_doc(api_result); return TryGetStrFromObject(root, "access_token"); } @@ -291,10 +313,8 @@ IBAPITable IBAPI::GetTable( // TODO: handle out-of-order columns using position property vector IBAPI::GetTables(const string &catalog, const string &internal, const string &schema, IBCredentials credentials) { vector result; - auto api_result = GetRequest(credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables", credentials.token); - auto *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); - auto *root = yyjson_doc_get_root(doc); + auto *root = api_result_to_doc(api_result); auto *tables = yyjson_obj_get(root, "identifiers"); size_t idx, max; duckdb_yyjson::yyjson_val *table; @@ -308,18 +328,9 @@ vector IBAPI::GetTables(const string &catalog, const string &interna vector IBAPI::GetSchemas(const string &catalog, const string &internal, IBCredentials credentials) { vector result; - auto api_result = GetRequest(credentials.endpoint + "/v1/" + internal + "/namespaces", credentials.token); - - duckdb_yyjson::yyjson_doc *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); - duckdb_yyjson::yyjson_val *root = yyjson_doc_get_root(doc); - auto *error = yyjson_obj_get(root, "error"); - if (error != NULL) { - string err_msg = TryGetStrFromObject(error, "message"); - throw std::runtime_error(err_msg); - } - + auto *root = api_result_to_doc(api_result); auto *schemas = yyjson_obj_get(root, "namespaces"); size_t idx, max; duckdb_yyjson::yyjson_val *schema; @@ -334,4 +345,22 @@ vector IBAPI::GetSchemas(const string &catalog, const string &inter return result; } +IBAPISchema IBAPI::CreateSchema(const string &catalog, const string &internal, const string &schema, IBCredentials credentials) { + string post_data = "{\"namespace\":[\"" + schema + "\"]}"; + string api_result = PostRequest( + credentials.endpoint + "/v1/" + internal + "/namespaces", post_data, "json", credentials.token); + api_result_to_doc(api_result); // if the method returns, request was successful + + IBAPISchema schema_result; + schema_result.catalog_name = catalog; + schema_result.schema_name = schema; //yyjson_get_str(value); + return schema_result; +} + +void IBAPI::DropSchema(const string &internal, const string &schema, IBCredentials credentials) { + string api_result = DeleteRequest( + credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema, credentials.token); + api_result_to_doc(api_result); // if the method returns, request was successful +} + } // namespace duckdb diff --git a/src/include/catalog_api.hpp b/src/include/catalog_api.hpp index 89de9a2..65cea98 100644 --- a/src/include/catalog_api.hpp +++ b/src/include/catalog_api.hpp @@ -51,5 +51,7 @@ class IBAPI { static vector GetSchemas(const string &catalog, const string &internal, IBCredentials credentials); static vector GetTablesInSchema(const string &catalog, const string &schema, IBCredentials credentials); static string GetToken(string id, string secret, string endpoint); + static IBAPISchema CreateSchema(const string &catalog, const string &internal, const string &schema, IBCredentials credentials); + static void DropSchema(const string &internal, const string &schema, IBCredentials credentials); }; } // namespace duckdb diff --git a/src/storage/ic_schema_set.cpp b/src/storage/ic_schema_set.cpp index 2b3fd1f..fc7b1f1 100644 --- a/src/storage/ic_schema_set.cpp +++ b/src/storage/ic_schema_set.cpp @@ -1,11 +1,10 @@ -#include "storage/ic_schema_set.hpp" -#include "storage/ic_catalog.hpp" #include "catalog_api.hpp" -#include "storage/ic_transaction.hpp" #include "duckdb/parser/parsed_data/create_schema_info.hpp" +#include "duckdb/parser/parsed_data/drop_info.hpp" #include "duckdb/catalog/catalog.hpp" - -#include +#include "storage/ic_catalog.hpp" +#include "storage/ic_schema_set.hpp" +#include "storage/ic_transaction.hpp" namespace duckdb { @@ -41,17 +40,17 @@ void IBSchemaSet::FillEntry(ClientContext &context, unique_ptr &en } optional_ptr IBSchemaSet::CreateSchema(ClientContext &context, CreateSchemaInfo &info) { - // TODO: throw NotImplementedException("Schema creation"); - std::cout << " >> Create schema" << std::endl; - // CreateEntry(...) + auto &ic_catalog = catalog.Cast(); + auto schema = IBAPI::CreateSchema(catalog.GetName(), ic_catalog.internal_name, info.schema, ic_catalog.credentials); + auto schema_entry = make_uniq(catalog, info); + schema_entry->schema_data = make_uniq(schema); + return CreateEntry(std::move(schema_entry)); } void IBSchemaSet::DropSchema(ClientContext &context, DropInfo &info) { - // TODO - std::cout << " >> Drop schema" << std::endl; - + auto &ic_catalog = catalog.Cast(); + IBAPI::DropSchema(ic_catalog.internal_name, info.name, ic_catalog.credentials); DropEntry(context, info); } - } // namespace duckdb From f24fc3a3e55cfa477ddaea1a57d2d7e0a7beb858 Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Thu, 23 Jan 2025 00:53:01 -0500 Subject: [PATCH 10/26] Reorg classes for clarity --- src/include/storage/ic_catalog.hpp | 1 + src/include/storage/ic_catalog_set.hpp | 10 ---------- src/include/storage/ic_table_set.hpp | 18 ++++++++++++++---- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/include/storage/ic_catalog.hpp b/src/include/storage/ic_catalog.hpp index 0b97be4..4957618 100644 --- a/src/include/storage/ic_catalog.hpp +++ b/src/include/storage/ic_catalog.hpp @@ -26,6 +26,7 @@ class IBClearCacheFunction : public TableFunction { static void ClearCacheOnSetting(ClientContext &context, SetScope scope, Value ¶meter); }; + class IBCatalog : public Catalog { public: explicit IBCatalog(AttachedDatabase &db_p, const string &internal_name, AccessMode access_mode, diff --git a/src/include/storage/ic_catalog_set.hpp b/src/include/storage/ic_catalog_set.hpp index f486842..9f9befb 100644 --- a/src/include/storage/ic_catalog_set.hpp +++ b/src/include/storage/ic_catalog_set.hpp @@ -34,14 +34,4 @@ class IBCatalogSet { mutex entry_lock; }; -class IBInSchemaSet : public IBCatalogSet { -public: - IBInSchemaSet(IBSchemaEntry &schema); - - optional_ptr CreateEntry(unique_ptr entry) override; - -protected: - IBSchemaEntry &schema; -}; - } // namespace duckdb diff --git a/src/include/storage/ic_table_set.hpp b/src/include/storage/ic_table_set.hpp index 5803086..25b7c8e 100644 --- a/src/include/storage/ic_table_set.hpp +++ b/src/include/storage/ic_table_set.hpp @@ -9,17 +9,26 @@ struct CreateTableInfo; class IBResult; class IBSchemaEntry; + +class IBInSchemaSet : public IBCatalogSet { +public: + IBInSchemaSet(IBSchemaEntry &schema); + + optional_ptr CreateEntry(unique_ptr entry) override; + +protected: + IBSchemaEntry &schema; +}; + + class IBTableSet : public IBInSchemaSet { public: explicit IBTableSet(IBSchemaEntry &schema); public: optional_ptr CreateTable(ClientContext &context, BoundCreateTableInfo &info); - - static unique_ptr GetTableInfo(ClientContext &context, IBSchemaEntry &schema, - const string &table_name); + static unique_ptr GetTableInfo(ClientContext &context, IBSchemaEntry &schema, const string &table_name); optional_ptr RefreshTable(ClientContext &context, const string &table_name); - void AlterTable(ClientContext &context, AlterTableInfo &info); protected: @@ -37,4 +46,5 @@ class IBTableSet : public IBInSchemaSet { unique_ptr _CreateCatalogEntry(ClientContext &context, IBAPITable table); }; + } // namespace duckdb From 56e4c764db42333596bc59341b56fb8a53f0a45c Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Thu, 23 Jan 2025 00:54:13 -0500 Subject: [PATCH 11/26] Minor fixes --- src/storage/ic_catalog.cpp | 2 -- src/storage/ic_schema_entry.cpp | 2 +- src/storage/ic_table_set.cpp | 2 -- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/storage/ic_catalog.cpp b/src/storage/ic_catalog.cpp index be7276a..4238651 100644 --- a/src/storage/ic_catalog.cpp +++ b/src/storage/ic_catalog.cpp @@ -63,8 +63,6 @@ string IBCatalog::GetDBPath() { return internal_name; } - - DatabaseSize IBCatalog::GetDatabaseSize(ClientContext &context) { if (default_schema.empty()) { throw InvalidInputException("Attempting to fetch the database size - but no database was provided " diff --git a/src/storage/ic_schema_entry.cpp b/src/storage/ic_schema_entry.cpp index 41cecaf..b137d1c 100644 --- a/src/storage/ic_schema_entry.cpp +++ b/src/storage/ic_schema_entry.cpp @@ -61,7 +61,7 @@ string GetUCCreateView(CreateViewInfo &info) { optional_ptr IBSchemaEntry::CreateView(CatalogTransaction transaction, CreateViewInfo &info) { if (info.sql.empty()) { - throw BinderException("Cannot create view in PC that originated from an " + throw BinderException("Cannot create view that originated from an " "empty SQL statement"); } if (info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT || diff --git a/src/storage/ic_table_set.cpp b/src/storage/ic_table_set.cpp index 5c211cb..bc65c5a 100644 --- a/src/storage/ic_table_set.cpp +++ b/src/storage/ic_table_set.cpp @@ -54,9 +54,7 @@ void IBTableSet::LoadEntries(ClientContext &context) { return; } - auto &transaction = IBTransaction::Get(context, catalog); auto &ic_catalog = catalog.Cast(); - // TODO: handle out-of-order columns using position property auto tables = IBAPI::GetTables(catalog.GetName(), catalog.GetDBPath(), schema.name, ic_catalog.credentials); From aee6c1d42645de6ec5cc501b710d9c8fb5ad47dd Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Thu, 23 Jan 2025 01:06:32 -0500 Subject: [PATCH 12/26] Rename classes for clarity --- src/catalog_api.cpp | 44 +++++++------- src/catalog_utils.cpp | 12 ++-- src/iceberg_extension.cpp | 18 +++--- src/include/catalog_api.hpp | 30 +++++----- src/include/catalog_utils.hpp | 14 ++--- src/include/storage/ic_catalog.hpp | 20 +++---- src/include/storage/ic_catalog_set.hpp | 8 +-- src/include/storage/ic_schema_entry.hpp | 14 ++--- src/include/storage/ic_schema_set.hpp | 4 +- src/include/storage/ic_table_entry.hpp | 16 ++--- src/include/storage/ic_table_set.hpp | 20 +++---- src/include/storage/ic_transaction.hpp | 18 +++--- .../storage/ic_transaction_manager.hpp | 8 +-- src/storage/ic_catalog.cpp | 46 +++++++-------- src/storage/ic_catalog_set.cpp | 24 ++++---- src/storage/ic_clear_cache.cpp | 6 +- src/storage/ic_schema_entry.cpp | 44 +++++++------- src/storage/ic_schema_set.cpp | 30 +++++----- src/storage/ic_table_entry.cpp | 18 +++--- src/storage/ic_table_set.cpp | 58 +++++++++---------- src/storage/ic_transaction.cpp | 38 ++++++------ src/storage/ic_transaction_manager.cpp | 18 +++--- 22 files changed, 255 insertions(+), 253 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index d0790af..d38d9a4 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -9,7 +9,7 @@ namespace duckdb { -//! We use a global here to store the path that is selected on the IBAPI::InitializeCurl call +//! We use a global here to store the path that is selected on the ICAPI::InitializeCurl call static string SELECTED_CURL_CERT_PATH = ""; static size_t RequestWriteCallback(void *contents, size_t size, size_t nmemb, void *userp) { @@ -208,7 +208,7 @@ static duckdb_yyjson::yyjson_val *api_result_to_doc(const string &api_result) { return root; } -static duckdb_yyjson::yyjson_val *GetTableMetadata(const string &internal, const string &schema, const string &table, IBCredentials credentials) { +static duckdb_yyjson::yyjson_val *GetTableMetadata(const string &internal, const string &schema, const string &table, ICCredentials credentials) { struct curl_slist *extra_headers = NULL; extra_headers = curl_slist_append(extra_headers, "X-Iceberg-Access-Delegation: vended-credentials"); auto api_result = GetRequest( @@ -218,16 +218,16 @@ static duckdb_yyjson::yyjson_val *GetTableMetadata(const string &internal, const return api_result_to_doc(api_result); } -void IBAPI::InitializeCurl() { +void ICAPI::InitializeCurl() { SelectCurlCertPath(); } -vector IBAPI::GetCatalogs(const string &catalog, IBCredentials credentials) { - throw NotImplementedException("IBAPI::GetCatalogs"); +vector ICAPI::GetCatalogs(const string &catalog, ICCredentials credentials) { + throw NotImplementedException("ICAPI::GetCatalogs"); } -static IBAPIColumnDefinition ParseColumnDefinition(duckdb_yyjson::yyjson_val *column_def) { - IBAPIColumnDefinition result; +static ICAPIColumnDefinition ParseColumnDefinition(duckdb_yyjson::yyjson_val *column_def) { + ICAPIColumnDefinition result; result.name = TryGetStrFromObject(column_def, "name"); result.type_text = TryGetStrFromObject(column_def, "type"); result.precision = (result.type_text == "decimal") ? TryGetNumFromObject(column_def, "type_precision") : -1; @@ -236,8 +236,8 @@ static IBAPIColumnDefinition ParseColumnDefinition(duckdb_yyjson::yyjson_val *co return result; } -IBAPITableCredentials IBAPI::GetTableCredentials(const string &internal, const string &schema, const string &table, IBCredentials credentials) { - IBAPITableCredentials result; +ICAPITableCredentials ICAPI::GetTableCredentials(const string &internal, const string &schema, const string &table, ICCredentials credentials) { + ICAPITableCredentials result; duckdb_yyjson::yyjson_val *root = GetTableMetadata(internal, schema, table, credentials); auto *aws_temp_credentials = yyjson_obj_get(root, "config"); @@ -250,17 +250,17 @@ IBAPITableCredentials IBAPI::GetTableCredentials(const string &internal, const s return result; } -string IBAPI::GetToken(string id, string secret, string endpoint) { +string ICAPI::GetToken(string id, string secret, string endpoint) { string post_data = "grant_type=client_credentials&client_id=" + id + "&client_secret=" + secret + "&scope=PRINCIPAL_ROLE:ALL"; string api_result = PostRequest(endpoint + "/v1/oauth/tokens", post_data); auto *root = api_result_to_doc(api_result); return TryGetStrFromObject(root, "access_token"); } -IBAPITable IBAPI::GetTable( - const string &catalog, const string &internal, const string &schema, const string &table, std::optional credentials) { +ICAPITable ICAPI::GetTable( + const string &catalog, const string &internal, const string &schema, const string &table, std::optional credentials) { - IBAPITable table_result; + ICAPITable table_result; table_result.catalog_name = catalog; table_result.schema_name = schema; table_result.name = table; @@ -298,7 +298,7 @@ IBAPITable IBAPI::GetTable( } } else { // Skip fetching metadata, we'll do it later when we access the table - IBAPIColumnDefinition col; + ICAPIColumnDefinition col; col.name = "__"; col.type_text = "int"; col.precision = -1; @@ -311,8 +311,8 @@ IBAPITable IBAPI::GetTable( } // TODO: handle out-of-order columns using position property -vector IBAPI::GetTables(const string &catalog, const string &internal, const string &schema, IBCredentials credentials) { - vector result; +vector ICAPI::GetTables(const string &catalog, const string &internal, const string &schema, ICCredentials credentials) { + vector result; auto api_result = GetRequest(credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables", credentials.token); auto *root = api_result_to_doc(api_result); auto *tables = yyjson_obj_get(root, "identifiers"); @@ -326,8 +326,8 @@ vector IBAPI::GetTables(const string &catalog, const string &interna return result; } -vector IBAPI::GetSchemas(const string &catalog, const string &internal, IBCredentials credentials) { - vector result; +vector ICAPI::GetSchemas(const string &catalog, const string &internal, ICCredentials credentials) { + vector result; auto api_result = GetRequest(credentials.endpoint + "/v1/" + internal + "/namespaces", credentials.token); auto *root = api_result_to_doc(api_result); @@ -335,7 +335,7 @@ vector IBAPI::GetSchemas(const string &catalog, const string &inter size_t idx, max; duckdb_yyjson::yyjson_val *schema; yyjson_arr_foreach(schemas, idx, max, schema) { - IBAPISchema schema_result; + ICAPISchema schema_result; schema_result.catalog_name = catalog; duckdb_yyjson::yyjson_val *value = yyjson_arr_get(schema, 0); schema_result.schema_name = yyjson_get_str(value); @@ -345,19 +345,19 @@ vector IBAPI::GetSchemas(const string &catalog, const string &inter return result; } -IBAPISchema IBAPI::CreateSchema(const string &catalog, const string &internal, const string &schema, IBCredentials credentials) { +ICAPISchema ICAPI::CreateSchema(const string &catalog, const string &internal, const string &schema, ICCredentials credentials) { string post_data = "{\"namespace\":[\"" + schema + "\"]}"; string api_result = PostRequest( credentials.endpoint + "/v1/" + internal + "/namespaces", post_data, "json", credentials.token); api_result_to_doc(api_result); // if the method returns, request was successful - IBAPISchema schema_result; + ICAPISchema schema_result; schema_result.catalog_name = catalog; schema_result.schema_name = schema; //yyjson_get_str(value); return schema_result; } -void IBAPI::DropSchema(const string &internal, const string &schema, IBCredentials credentials) { +void ICAPI::DropSchema(const string &internal, const string &schema, ICCredentials credentials) { string api_result = DeleteRequest( credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema, credentials.token); api_result_to_doc(api_result); // if the method returns, request was successful diff --git a/src/catalog_utils.cpp b/src/catalog_utils.cpp index 87777df..f71b41e 100644 --- a/src/catalog_utils.cpp +++ b/src/catalog_utils.cpp @@ -7,7 +7,7 @@ namespace duckdb { -string IBUtils::TypeToString(const LogicalType &input) { +string ICUtils::TypeToString(const LogicalType &input) { switch (input.id()) { case LogicalType::VARCHAR: return "TEXT"; @@ -28,7 +28,7 @@ string IBUtils::TypeToString(const LogicalType &input) { } } -LogicalType IBUtils::TypeToLogicalType(ClientContext &context, const string &type_text) { +LogicalType ICUtils::TypeToLogicalType(ClientContext &context, const string &type_text) { if (type_text == "tinyint") { return LogicalType::TINYINT; } else if (type_text == "smallint") { @@ -69,7 +69,7 @@ LogicalType IBUtils::TypeToLogicalType(ClientContext &context, const string &typ size_t type_end = type_text.rfind('>'); // find last, to deal with nested if (type_end != string::npos) { auto child_type_str = type_text.substr(6, type_end - 6); - auto child_type = IBUtils::TypeToLogicalType(context, child_type_str); + auto child_type = ICUtils::TypeToLogicalType(context, child_type_str); return LogicalType::LIST(child_type); } } else if (type_text.find("map<") == 0) { @@ -94,7 +94,7 @@ LogicalType IBUtils::TypeToLogicalType(ClientContext &context, const string &typ } } auto child_str = type_text.substr(cur, next_sep - cur); - auto child_type = IBUtils::TypeToLogicalType(context, child_str); + auto child_type = ICUtils::TypeToLogicalType(context, child_str); key_val.push_back(child_type); if (next_sep == type_end) { break; @@ -132,7 +132,7 @@ LogicalType IBUtils::TypeToLogicalType(ClientContext &context, const string &typ throw NotImplementedException("Invalid struct child type specifier: %s", child_str); } auto child_name = child_str.substr(0, type_sep); - auto child_type = IBUtils::TypeToLogicalType(context, child_str.substr(type_sep + 1, string::npos)); + auto child_type = ICUtils::TypeToLogicalType(context, child_str.substr(type_sep + 1, string::npos)); children.push_back({child_name, child_type}); if (next_sep == type_end) { break; @@ -148,7 +148,7 @@ LogicalType IBUtils::TypeToLogicalType(ClientContext &context, const string &typ return LogicalType::VARCHAR; } -LogicalType IBUtils::ToUCType(const LogicalType &input) { +LogicalType ICUtils::ToUCType(const LogicalType &input) { // todo do we need this mapping? throw NotImplementedException("ToUCType not yet implemented"); switch (input.id()) { diff --git a/src/iceberg_extension.cpp b/src/iceberg_extension.cpp index ca5971c..114b3ff 100644 --- a/src/iceberg_extension.cpp +++ b/src/iceberg_extension.cpp @@ -42,7 +42,7 @@ static unique_ptr CreateCatalogSecretFunction(ClientContext &, Creat } // Get token from catalog - result->secret_map["token"] = IBAPI::GetToken( + result->secret_map["token"] = ICAPI::GetToken( result->secret_map["client_id"].ToString(), result->secret_map["client_secret"].ToString(), result->secret_map["endpoint"].ToString()); @@ -80,7 +80,7 @@ unique_ptr GetSecret(ClientContext &context, const string &secret_n static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_info, ClientContext &context, AttachedDatabase &db, const string &name, AttachInfo &info, AccessMode access_mode) { - IBCredentials credentials; + ICCredentials credentials; // check if we have a secret provided string secret_name; @@ -131,18 +131,18 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in // TODO: Check catalog with name actually exists! - return make_uniq(db, info.path, access_mode, credentials); + return make_uniq(db, info.path, access_mode, credentials); } static unique_ptr CreateTransactionManager(StorageExtensionInfo *storage_info, AttachedDatabase &db, Catalog &catalog) { - auto &ic_catalog = catalog.Cast(); - return make_uniq(db, ic_catalog); + auto &ic_catalog = catalog.Cast(); + return make_uniq(db, ic_catalog); } -class IBCatalogStorageExtension : public StorageExtension { +class ICCatalogStorageExtension : public StorageExtension { public: - IBCatalogStorageExtension() { + ICCatalogStorageExtension() { attach = IcebergCatalogAttach; create_transaction_manager = CreateTransactionManager; } @@ -168,7 +168,7 @@ static void LoadInternal(DatabaseInstance &instance) { ExtensionUtil::RegisterFunction(instance, fun); } - IBAPI::InitializeCurl(); + ICAPI::InitializeCurl(); SecretType secret_type; secret_type.name = "iceberg"; @@ -180,7 +180,7 @@ static void LoadInternal(DatabaseInstance &instance) { SetCatalogSecretParameters(secret_function); ExtensionUtil::RegisterFunction(instance, secret_function); - config.storage_extensions["iceberg"] = make_uniq(); + config.storage_extensions["iceberg"] = make_uniq(); } void IcebergExtension::Load(DuckDB &db) { diff --git a/src/include/catalog_api.hpp b/src/include/catalog_api.hpp index 65cea98..b2d0683 100644 --- a/src/include/catalog_api.hpp +++ b/src/include/catalog_api.hpp @@ -4,9 +4,9 @@ #include "duckdb/common/types.hpp" namespace duckdb { -struct IBCredentials; +struct ICCredentials; -struct IBAPIColumnDefinition { +struct ICAPIColumnDefinition { string name; string type_text; idx_t precision; @@ -14,7 +14,7 @@ struct IBAPIColumnDefinition { idx_t position; }; -struct IBAPITable { +struct ICAPITable { string table_id; string name; @@ -25,33 +25,33 @@ struct IBAPITable { string storage_location; - vector columns; + vector columns; }; -struct IBAPISchema { +struct ICAPISchema { string schema_name; string catalog_name; }; -struct IBAPITableCredentials { +struct ICAPITableCredentials { string key_id; string secret; string session_token; }; -class IBAPI { +class ICAPI { public: //! WARNING: not thread-safe. To be called once on extension initialization static void InitializeCurl(); - static IBAPITableCredentials GetTableCredentials(const string &internal, const string &schema, const string &table, IBCredentials credentials); - static vector GetCatalogs(const string &catalog, IBCredentials credentials); - static vector GetTables(const string &catalog, const string &internal, const string &schema, IBCredentials credentials); - static IBAPITable GetTable(const string &catalog, const string &internal, const string &schema, const string &table, std::optional credentials); - static vector GetSchemas(const string &catalog, const string &internal, IBCredentials credentials); - static vector GetTablesInSchema(const string &catalog, const string &schema, IBCredentials credentials); + static ICAPITableCredentials GetTableCredentials(const string &internal, const string &schema, const string &table, ICCredentials credentials); + static vector GetCatalogs(const string &catalog, ICCredentials credentials); + static vector GetTables(const string &catalog, const string &internal, const string &schema, ICCredentials credentials); + static ICAPITable GetTable(const string &catalog, const string &internal, const string &schema, const string &table, std::optional credentials); + static vector GetSchemas(const string &catalog, const string &internal, ICCredentials credentials); + static vector GetTablesInSchema(const string &catalog, const string &schema, ICCredentials credentials); static string GetToken(string id, string secret, string endpoint); - static IBAPISchema CreateSchema(const string &catalog, const string &internal, const string &schema, IBCredentials credentials); - static void DropSchema(const string &internal, const string &schema, IBCredentials credentials); + static ICAPISchema CreateSchema(const string &catalog, const string &internal, const string &schema, ICCredentials credentials); + static void DropSchema(const string &internal, const string &schema, ICCredentials credentials); }; } // namespace duckdb diff --git a/src/include/catalog_utils.hpp b/src/include/catalog_utils.hpp index ee11eb8..240e63b 100644 --- a/src/include/catalog_utils.hpp +++ b/src/include/catalog_utils.hpp @@ -5,18 +5,18 @@ #include "catalog_api.hpp" namespace duckdb { -class IBSchemaEntry; -class IBTransaction; +class ICSchemaEntry; +class ICTransaction; -enum class IBTypeAnnotation { STANDARD, CAST_TO_VARCHAR, NUMERIC_AS_DOUBLE, CTID, JSONB, FIXED_LENGTH_CHAR }; +enum class ICTypeAnnotation { STANDARD, CAST_TO_VARCHAR, NUMERIC_AS_DOUBLE, CTID, JSONB, FIXED_LENGTH_CHAR }; -struct IBType { +struct ICType { idx_t oid = 0; - IBTypeAnnotation info = IBTypeAnnotation::STANDARD; - vector children; + ICTypeAnnotation info = ICTypeAnnotation::STANDARD; + vector children; }; -class IBUtils { +class ICUtils { public: static LogicalType ToUCType(const LogicalType &input); static LogicalType TypeToLogicalType(ClientContext &context, const string &columnDefinition); diff --git a/src/include/storage/ic_catalog.hpp b/src/include/storage/ic_catalog.hpp index 4957618..1c0abee 100644 --- a/src/include/storage/ic_catalog.hpp +++ b/src/include/storage/ic_catalog.hpp @@ -7,9 +7,9 @@ #include "storage/ic_schema_set.hpp" namespace duckdb { -class IBSchemaEntry; +class ICSchemaEntry; -struct IBCredentials { +struct ICCredentials { string endpoint; string client_id; string client_secret; @@ -19,23 +19,23 @@ struct IBCredentials { string token; }; -class IBClearCacheFunction : public TableFunction { +class ICClearCacheFunction : public TableFunction { public: - IBClearCacheFunction(); + ICClearCacheFunction(); static void ClearCacheOnSetting(ClientContext &context, SetScope scope, Value ¶meter); }; -class IBCatalog : public Catalog { +class ICCatalog : public Catalog { public: - explicit IBCatalog(AttachedDatabase &db_p, const string &internal_name, AccessMode access_mode, - IBCredentials credentials); - ~IBCatalog(); + explicit ICCatalog(AttachedDatabase &db_p, const string &internal_name, AccessMode access_mode, + ICCredentials credentials); + ~ICCatalog(); string internal_name; AccessMode access_mode; - IBCredentials credentials; + ICCredentials credentials; public: void Initialize(bool load_builtin) override; @@ -74,7 +74,7 @@ class IBCatalog : public Catalog { void DropSchema(ClientContext &context, DropInfo &info) override; private: - IBSchemaSet schemas; + ICSchemaSet schemas; string default_schema; }; diff --git a/src/include/storage/ic_catalog_set.hpp b/src/include/storage/ic_catalog_set.hpp index 9f9befb..927c6a2 100644 --- a/src/include/storage/ic_catalog_set.hpp +++ b/src/include/storage/ic_catalog_set.hpp @@ -7,12 +7,12 @@ namespace duckdb { struct DropInfo; -class IBSchemaEntry; -class IBTransaction; +class ICSchemaEntry; +class ICTransaction; -class IBCatalogSet { +class ICCatalogSet { public: - IBCatalogSet(Catalog &catalog); + ICCatalogSet(Catalog &catalog); optional_ptr GetEntry(ClientContext &context, const string &name); virtual void DropEntry(ClientContext &context, DropInfo &info); diff --git a/src/include/storage/ic_schema_entry.hpp b/src/include/storage/ic_schema_entry.hpp index c8d5e1c..344066b 100644 --- a/src/include/storage/ic_schema_entry.hpp +++ b/src/include/storage/ic_schema_entry.hpp @@ -6,14 +6,14 @@ #include "storage/ic_table_set.hpp" namespace duckdb { -class IBTransaction; +class ICTransaction; -class IBSchemaEntry : public SchemaCatalogEntry { +class ICSchemaEntry : public SchemaCatalogEntry { public: - IBSchemaEntry(Catalog &catalog, CreateSchemaInfo &info); - ~IBSchemaEntry() override; + ICSchemaEntry(Catalog &catalog, CreateSchemaInfo &info); + ~ICSchemaEntry() override; - unique_ptr schema_data; + unique_ptr schema_data; public: optional_ptr CreateTable(CatalogTransaction transaction, BoundCreateTableInfo &info) override; @@ -37,10 +37,10 @@ class IBSchemaEntry : public SchemaCatalogEntry { optional_ptr GetEntry(CatalogTransaction transaction, CatalogType type, const string &name) override; private: - IBCatalogSet &GetCatalogSet(CatalogType type); + ICCatalogSet &GetCatalogSet(CatalogType type); private: - IBTableSet tables; + ICTableSet tables; }; } // namespace duckdb diff --git a/src/include/storage/ic_schema_set.hpp b/src/include/storage/ic_schema_set.hpp index 6e8332f..6a5068c 100644 --- a/src/include/storage/ic_schema_set.hpp +++ b/src/include/storage/ic_schema_set.hpp @@ -7,9 +7,9 @@ namespace duckdb { struct CreateSchemaInfo; -class IBSchemaSet : public IBCatalogSet { +class ICSchemaSet : public ICCatalogSet { public: - explicit IBSchemaSet(Catalog &catalog); + explicit ICSchemaSet(Catalog &catalog); public: optional_ptr CreateSchema(ClientContext &context, CreateSchemaInfo &info); diff --git a/src/include/storage/ic_table_entry.hpp b/src/include/storage/ic_table_entry.hpp index f648237..e415c1c 100644 --- a/src/include/storage/ic_table_entry.hpp +++ b/src/include/storage/ic_table_entry.hpp @@ -7,14 +7,14 @@ namespace duckdb { -struct IBTableInfo { - IBTableInfo() { +struct ICTableInfo { + ICTableInfo() { create_info = make_uniq(); } - IBTableInfo(const string &schema, const string &table) { + ICTableInfo(const string &schema, const string &table) { create_info = make_uniq(string(), schema, table); } - IBTableInfo(const SchemaCatalogEntry &schema, const string &table) { + ICTableInfo(const SchemaCatalogEntry &schema, const string &table) { create_info = make_uniq((SchemaCatalogEntry &)schema, table); } @@ -25,12 +25,12 @@ struct IBTableInfo { unique_ptr create_info; }; -class IBTableEntry : public TableCatalogEntry { +class ICTableEntry : public TableCatalogEntry { public: - IBTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTableInfo &info); - IBTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, IBTableInfo &info); + ICTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTableInfo &info); + ICTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, ICTableInfo &info); - unique_ptr table_data; + unique_ptr table_data; public: unique_ptr GetStatistics(ClientContext &context, column_t column_id) override; diff --git a/src/include/storage/ic_table_set.hpp b/src/include/storage/ic_table_set.hpp index 25b7c8e..7e41593 100644 --- a/src/include/storage/ic_table_set.hpp +++ b/src/include/storage/ic_table_set.hpp @@ -6,28 +6,28 @@ namespace duckdb { struct CreateTableInfo; -class IBResult; -class IBSchemaEntry; +class ICResult; +class ICSchemaEntry; -class IBInSchemaSet : public IBCatalogSet { +class ICInSchemaSet : public ICCatalogSet { public: - IBInSchemaSet(IBSchemaEntry &schema); + ICInSchemaSet(ICSchemaEntry &schema); optional_ptr CreateEntry(unique_ptr entry) override; protected: - IBSchemaEntry &schema; + ICSchemaEntry &schema; }; -class IBTableSet : public IBInSchemaSet { +class ICTableSet : public ICInSchemaSet { public: - explicit IBTableSet(IBSchemaEntry &schema); + explicit ICTableSet(ICSchemaEntry &schema); public: optional_ptr CreateTable(ClientContext &context, BoundCreateTableInfo &info); - static unique_ptr GetTableInfo(ClientContext &context, IBSchemaEntry &schema, const string &table_name); + static unique_ptr GetTableInfo(ClientContext &context, ICSchemaEntry &schema, const string &table_name); optional_ptr RefreshTable(ClientContext &context, const string &table_name); void AlterTable(ClientContext &context, AlterTableInfo &info); @@ -40,10 +40,10 @@ class IBTableSet : public IBInSchemaSet { void AlterTable(ClientContext &context, AddColumnInfo &info); void AlterTable(ClientContext &context, RemoveColumnInfo &info); - static void AddColumn(ClientContext &context, IBResult &result, IBTableInfo &table_info, idx_t column_offset = 0); + static void AddColumn(ClientContext &context, ICResult &result, ICTableInfo &table_info, idx_t column_offset = 0); private: - unique_ptr _CreateCatalogEntry(ClientContext &context, IBAPITable table); + unique_ptr _CreateCatalogEntry(ClientContext &context, ICAPITable table); }; diff --git a/src/include/storage/ic_transaction.hpp b/src/include/storage/ic_transaction.hpp index 7dba882..79baf62 100644 --- a/src/include/storage/ic_transaction.hpp +++ b/src/include/storage/ic_transaction.hpp @@ -4,28 +4,28 @@ #include "duckdb/transaction/transaction.hpp" namespace duckdb { -class IBCatalog; -class IBSchemaEntry; -class IBTableEntry; +class ICCatalog; +class ICSchemaEntry; +class ICTableEntry; -enum class IBTransactionState { TRANSACTION_NOT_YET_STARTED, TRANSACTION_STARTED, TRANSACTION_FINISHED }; +enum class ICTransactionState { TRANSACTION_NOT_YET_STARTED, TRANSACTION_STARTED, TRANSACTION_FINISHED }; -class IBTransaction : public Transaction { +class ICTransaction : public Transaction { public: - IBTransaction(IBCatalog &ic_catalog, TransactionManager &manager, ClientContext &context); - ~IBTransaction() override; + ICTransaction(ICCatalog &ic_catalog, TransactionManager &manager, ClientContext &context); + ~ICTransaction() override; void Start(); void Commit(); void Rollback(); - static IBTransaction &Get(ClientContext &context, Catalog &catalog); + static ICTransaction &Get(ClientContext &context, Catalog &catalog); AccessMode GetAccessMode() const { return access_mode; } private: - IBTransactionState transaction_state; + ICTransactionState transaction_state; AccessMode access_mode; }; diff --git a/src/include/storage/ic_transaction_manager.hpp b/src/include/storage/ic_transaction_manager.hpp index 4d02af7..5337234 100644 --- a/src/include/storage/ic_transaction_manager.hpp +++ b/src/include/storage/ic_transaction_manager.hpp @@ -7,9 +7,9 @@ namespace duckdb { -class IBTransactionManager : public TransactionManager { +class ICTransactionManager : public TransactionManager { public: - IBTransactionManager(AttachedDatabase &db_p, IBCatalog &ic_catalog); + ICTransactionManager(AttachedDatabase &db_p, ICCatalog &ic_catalog); Transaction &StartTransaction(ClientContext &context) override; ErrorData CommitTransaction(ClientContext &context, Transaction &transaction) override; @@ -18,9 +18,9 @@ class IBTransactionManager : public TransactionManager { void Checkpoint(ClientContext &context, bool force = false) override; private: - IBCatalog &ic_catalog; + ICCatalog &ic_catalog; mutex transaction_lock; - reference_map_t> transactions; + reference_map_t> transactions; }; } // namespace duckdb diff --git a/src/storage/ic_catalog.cpp b/src/storage/ic_catalog.cpp index 4238651..a920c29 100644 --- a/src/storage/ic_catalog.cpp +++ b/src/storage/ic_catalog.cpp @@ -8,18 +8,18 @@ namespace duckdb { -IBCatalog::IBCatalog(AttachedDatabase &db_p, const string &internal_name, AccessMode access_mode, - IBCredentials credentials) +ICCatalog::ICCatalog(AttachedDatabase &db_p, const string &internal_name, AccessMode access_mode, + ICCredentials credentials) : Catalog(db_p), internal_name(internal_name), access_mode(access_mode), credentials(std::move(credentials)), schemas(*this) { } -IBCatalog::~IBCatalog() = default; +ICCatalog::~ICCatalog() = default; -void IBCatalog::Initialize(bool load_builtin) { +void ICCatalog::Initialize(bool load_builtin) { } -optional_ptr IBCatalog::CreateSchema(CatalogTransaction transaction, CreateSchemaInfo &info) { +optional_ptr ICCatalog::CreateSchema(CatalogTransaction transaction, CreateSchemaInfo &info) { if (info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) { DropInfo try_drop; try_drop.type = CatalogType::SCHEMA_ENTRY; @@ -31,15 +31,15 @@ optional_ptr IBCatalog::CreateSchema(CatalogTransaction transactio return schemas.CreateSchema(transaction.GetContext(), info); } -void IBCatalog::DropSchema(ClientContext &context, DropInfo &info) { +void ICCatalog::DropSchema(ClientContext &context, DropInfo &info) { return schemas.DropSchema(context, info); } -void IBCatalog::ScanSchemas(ClientContext &context, std::function callback) { - schemas.Scan(context, [&](CatalogEntry &schema) { callback(schema.Cast()); }); +void ICCatalog::ScanSchemas(ClientContext &context, std::function callback) { + schemas.Scan(context, [&](CatalogEntry &schema) { callback(schema.Cast()); }); } -optional_ptr IBCatalog::GetSchema(CatalogTransaction transaction, const string &schema_name, +optional_ptr ICCatalog::GetSchema(CatalogTransaction transaction, const string &schema_name, OnEntryNotFound if_not_found, QueryErrorContext error_context) { if (schema_name == DEFAULT_SCHEMA) { if (default_schema.empty()) { @@ -55,15 +55,15 @@ optional_ptr IBCatalog::GetSchema(CatalogTransaction transac return reinterpret_cast(entry.get()); } -bool IBCatalog::InMemory() { +bool ICCatalog::InMemory() { return false; } -string IBCatalog::GetDBPath() { +string ICCatalog::GetDBPath() { return internal_name; } -DatabaseSize IBCatalog::GetDatabaseSize(ClientContext &context) { +DatabaseSize ICCatalog::GetDatabaseSize(ClientContext &context) { if (default_schema.empty()) { throw InvalidInputException("Attempting to fetch the database size - but no database was provided " "in the connection string"); @@ -72,29 +72,29 @@ DatabaseSize IBCatalog::GetDatabaseSize(ClientContext &context) { return size; } -void IBCatalog::ClearCache() { +void ICCatalog::ClearCache() { schemas.ClearEntries(); } -unique_ptr IBCatalog::PlanInsert(ClientContext &context, LogicalInsert &op, +unique_ptr ICCatalog::PlanInsert(ClientContext &context, LogicalInsert &op, unique_ptr plan) { - throw NotImplementedException("IBCatalog PlanInsert"); + throw NotImplementedException("ICCatalog PlanInsert"); } -unique_ptr IBCatalog::PlanCreateTableAs(ClientContext &context, LogicalCreateTable &op, +unique_ptr ICCatalog::PlanCreateTableAs(ClientContext &context, LogicalCreateTable &op, unique_ptr plan) { - throw NotImplementedException("IBCatalog PlanCreateTableAs"); + throw NotImplementedException("ICCatalog PlanCreateTableAs"); } -unique_ptr IBCatalog::PlanDelete(ClientContext &context, LogicalDelete &op, +unique_ptr ICCatalog::PlanDelete(ClientContext &context, LogicalDelete &op, unique_ptr plan) { - throw NotImplementedException("IBCatalog PlanDelete"); + throw NotImplementedException("ICCatalog PlanDelete"); } -unique_ptr IBCatalog::PlanUpdate(ClientContext &context, LogicalUpdate &op, +unique_ptr ICCatalog::PlanUpdate(ClientContext &context, LogicalUpdate &op, unique_ptr plan) { - throw NotImplementedException("IBCatalog PlanUpdate"); + throw NotImplementedException("ICCatalog PlanUpdate"); } -unique_ptr IBCatalog::BindCreateIndex(Binder &binder, CreateStatement &stmt, TableCatalogEntry &table, +unique_ptr ICCatalog::BindCreateIndex(Binder &binder, CreateStatement &stmt, TableCatalogEntry &table, unique_ptr plan) { - throw NotImplementedException("IBCatalog BindCreateIndex"); + throw NotImplementedException("ICCatalog BindCreateIndex"); } } // namespace duckdb diff --git a/src/storage/ic_catalog_set.cpp b/src/storage/ic_catalog_set.cpp index cc00d54..c3b01a1 100644 --- a/src/storage/ic_catalog_set.cpp +++ b/src/storage/ic_catalog_set.cpp @@ -5,10 +5,10 @@ namespace duckdb { -IBCatalogSet::IBCatalogSet(Catalog &catalog) : catalog(catalog) { +ICCatalogSet::ICCatalogSet(Catalog &catalog) : catalog(catalog) { } -optional_ptr IBCatalogSet::GetEntry(ClientContext &context, const string &name) { +optional_ptr ICCatalogSet::GetEntry(ClientContext &context, const string &name) { LoadEntries(context); lock_guard l(entry_lock); auto entry = entries.find(name); @@ -19,16 +19,18 @@ optional_ptr IBCatalogSet::GetEntry(ClientContext &context, const return entry->second.get(); } -void IBCatalogSet::DropEntry(ClientContext &context, DropInfo &info) { +void ICCatalogSet::DropEntry(ClientContext &context, DropInfo &info) { + std::cout << "ICCatalogSet::DropEntry" << info.type << std::endl; + EraseEntryInternal(info.name); } -void IBCatalogSet::EraseEntryInternal(const string &name) { +void ICCatalogSet::EraseEntryInternal(const string &name) { lock_guard l(entry_lock); entries.erase(name); } -void IBCatalogSet::Scan(ClientContext &context, const std::function &callback) { +void ICCatalogSet::Scan(ClientContext &context, const std::function &callback) { LoadEntries(context); lock_guard l(entry_lock); @@ -37,28 +39,28 @@ void IBCatalogSet::Scan(ClientContext &context, const std::function IBCatalogSet::CreateEntry(unique_ptr entry) { +optional_ptr ICCatalogSet::CreateEntry(unique_ptr entry) { lock_guard l(entry_lock); auto result = entry.get(); if (result->name.empty()) { - throw InternalException("IBCatalogSet::CreateEntry called with empty name"); + throw InternalException("ICCatalogSet::CreateEntry called with empty name"); } entries.insert(make_pair(result->name, std::move(entry))); return result; } -void IBCatalogSet::ClearEntries() { +void ICCatalogSet::ClearEntries() { entries.clear(); } -IBInSchemaSet::IBInSchemaSet(IBSchemaEntry &schema) : IBCatalogSet(schema.ParentCatalog()), schema(schema) { +ICInSchemaSet::ICInSchemaSet(ICSchemaEntry &schema) : ICCatalogSet(schema.ParentCatalog()), schema(schema) { } -optional_ptr IBInSchemaSet::CreateEntry(unique_ptr entry) { +optional_ptr ICInSchemaSet::CreateEntry(unique_ptr entry) { if (!entry->internal) { entry->internal = schema.internal; } - return IBCatalogSet::CreateEntry(std::move(entry)); + return ICCatalogSet::CreateEntry(std::move(entry)); } } // namespace duckdb diff --git a/src/storage/ic_clear_cache.cpp b/src/storage/ic_clear_cache.cpp index 99db62a..0e07887 100644 --- a/src/storage/ic_clear_cache.cpp +++ b/src/storage/ic_clear_cache.cpp @@ -28,7 +28,7 @@ static void ClearUCCaches(ClientContext &context) { if (catalog.GetCatalogType() != "iceberg") { continue; } - catalog.Cast().ClearCache(); + catalog.Cast().ClearCache(); } } @@ -41,10 +41,10 @@ static void ClearCacheFunction(ClientContext &context, TableFunctionInput &data_ data.finished = true; } -void IBClearCacheFunction::ClearCacheOnSetting(ClientContext &context, SetScope scope, Value ¶meter) { +void ICClearCacheFunction::ClearCacheOnSetting(ClientContext &context, SetScope scope, Value ¶meter) { ClearUCCaches(context); } -IBClearCacheFunction::IBClearCacheFunction() : TableFunction("pc_clear_cache", {}, ClearCacheFunction, ClearCacheBind) { +ICClearCacheFunction::ICClearCacheFunction() : TableFunction("pc_clear_cache", {}, ClearCacheFunction, ClearCacheBind) { } } // namespace duckdb diff --git a/src/storage/ic_schema_entry.cpp b/src/storage/ic_schema_entry.cpp index b137d1c..691c9b0 100644 --- a/src/storage/ic_schema_entry.cpp +++ b/src/storage/ic_schema_entry.cpp @@ -13,21 +13,21 @@ namespace duckdb { -IBSchemaEntry::IBSchemaEntry(Catalog &catalog, CreateSchemaInfo &info) +ICSchemaEntry::ICSchemaEntry(Catalog &catalog, CreateSchemaInfo &info) : SchemaCatalogEntry(catalog, info), tables(*this) { } -IBSchemaEntry::~IBSchemaEntry() { +ICSchemaEntry::~ICSchemaEntry() { } -IBTransaction &GetUCTransaction(CatalogTransaction transaction) { +ICTransaction &GetUCTransaction(CatalogTransaction transaction) { if (!transaction.transaction) { throw InternalException("No transaction!?"); } - return transaction.transaction->Cast(); + return transaction.transaction->Cast(); } -optional_ptr IBSchemaEntry::CreateTable(CatalogTransaction transaction, BoundCreateTableInfo &info) { +optional_ptr ICSchemaEntry::CreateTable(CatalogTransaction transaction, BoundCreateTableInfo &info) { auto &base_info = info.Base(); auto table_name = base_info.table; if (base_info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) { @@ -36,21 +36,21 @@ optional_ptr IBSchemaEntry::CreateTable(CatalogTransaction transac return tables.CreateTable(transaction.GetContext(), info); } -optional_ptr IBSchemaEntry::CreateFunction(CatalogTransaction transaction, CreateFunctionInfo &info) { +optional_ptr ICSchemaEntry::CreateFunction(CatalogTransaction transaction, CreateFunctionInfo &info) { throw BinderException("PC databases do not support creating functions"); } -void IBUnqualifyColumnRef(ParsedExpression &expr) { +void ICUnqualifyColumnRef(ParsedExpression &expr) { if (expr.type == ExpressionType::COLUMN_REF) { auto &colref = expr.Cast(); auto name = std::move(colref.column_names.back()); colref.column_names = {std::move(name)}; return; } - ParsedExpressionIterator::EnumerateChildren(expr, IBUnqualifyColumnRef); + ParsedExpressionIterator::EnumerateChildren(expr, ICUnqualifyColumnRef); } -optional_ptr IBSchemaEntry::CreateIndex(CatalogTransaction transaction, CreateIndexInfo &info, +optional_ptr ICSchemaEntry::CreateIndex(CatalogTransaction transaction, CreateIndexInfo &info, TableCatalogEntry &table) { throw NotImplementedException("CreateIndex"); } @@ -59,7 +59,7 @@ string GetUCCreateView(CreateViewInfo &info) { throw NotImplementedException("GetCreateView"); } -optional_ptr IBSchemaEntry::CreateView(CatalogTransaction transaction, CreateViewInfo &info) { +optional_ptr ICSchemaEntry::CreateView(CatalogTransaction transaction, CreateViewInfo &info) { if (info.sql.empty()) { throw BinderException("Cannot create view that originated from an " "empty SQL statement"); @@ -79,34 +79,34 @@ optional_ptr IBSchemaEntry::CreateView(CatalogTransaction transact return tables.RefreshTable(transaction.GetContext(), info.view_name); } -optional_ptr IBSchemaEntry::CreateType(CatalogTransaction transaction, CreateTypeInfo &info) { +optional_ptr ICSchemaEntry::CreateType(CatalogTransaction transaction, CreateTypeInfo &info) { throw BinderException("PC databases do not support creating types"); } -optional_ptr IBSchemaEntry::CreateSequence(CatalogTransaction transaction, CreateSequenceInfo &info) { +optional_ptr ICSchemaEntry::CreateSequence(CatalogTransaction transaction, CreateSequenceInfo &info) { throw BinderException("PC databases do not support creating sequences"); } -optional_ptr IBSchemaEntry::CreateTableFunction(CatalogTransaction transaction, +optional_ptr ICSchemaEntry::CreateTableFunction(CatalogTransaction transaction, CreateTableFunctionInfo &info) { throw BinderException("PC databases do not support creating table functions"); } -optional_ptr IBSchemaEntry::CreateCopyFunction(CatalogTransaction transaction, +optional_ptr ICSchemaEntry::CreateCopyFunction(CatalogTransaction transaction, CreateCopyFunctionInfo &info) { throw BinderException("PC databases do not support creating copy functions"); } -optional_ptr IBSchemaEntry::CreatePragmaFunction(CatalogTransaction transaction, +optional_ptr ICSchemaEntry::CreatePragmaFunction(CatalogTransaction transaction, CreatePragmaFunctionInfo &info) { throw BinderException("PC databases do not support creating pragma functions"); } -optional_ptr IBSchemaEntry::CreateCollation(CatalogTransaction transaction, CreateCollationInfo &info) { +optional_ptr ICSchemaEntry::CreateCollation(CatalogTransaction transaction, CreateCollationInfo &info) { throw BinderException("PC databases do not support creating collations"); } -void IBSchemaEntry::Alter(CatalogTransaction transaction, AlterInfo &info) { +void ICSchemaEntry::Alter(CatalogTransaction transaction, AlterInfo &info) { if (info.type != AlterType::ALTER_TABLE) { throw BinderException("Only altering tables is supported for now"); } @@ -125,22 +125,22 @@ bool CatalogTypeIsSupported(CatalogType type) { } } -void IBSchemaEntry::Scan(ClientContext &context, CatalogType type, +void ICSchemaEntry::Scan(ClientContext &context, CatalogType type, const std::function &callback) { if (!CatalogTypeIsSupported(type)) { return; } GetCatalogSet(type).Scan(context, callback); } -void IBSchemaEntry::Scan(CatalogType type, const std::function &callback) { +void ICSchemaEntry::Scan(CatalogType type, const std::function &callback) { throw NotImplementedException("Scan without context not supported"); } -void IBSchemaEntry::DropEntry(ClientContext &context, DropInfo &info) { +void ICSchemaEntry::DropEntry(ClientContext &context, DropInfo &info) { GetCatalogSet(info.type).DropEntry(context, info); } -optional_ptr IBSchemaEntry::GetEntry(CatalogTransaction transaction, CatalogType type, +optional_ptr ICSchemaEntry::GetEntry(CatalogTransaction transaction, CatalogType type, const string &name) { if (!CatalogTypeIsSupported(type)) { return nullptr; @@ -148,7 +148,7 @@ optional_ptr IBSchemaEntry::GetEntry(CatalogTransaction transactio return GetCatalogSet(type).GetEntry(transaction.GetContext(), name); } -IBCatalogSet &IBSchemaEntry::GetCatalogSet(CatalogType type) { +ICCatalogSet &ICSchemaEntry::GetCatalogSet(CatalogType type) { switch (type) { case CatalogType::TABLE_ENTRY: case CatalogType::VIEW_ENTRY: diff --git a/src/storage/ic_schema_set.cpp b/src/storage/ic_schema_set.cpp index fc7b1f1..b35b624 100644 --- a/src/storage/ic_schema_set.cpp +++ b/src/storage/ic_schema_set.cpp @@ -8,7 +8,7 @@ namespace duckdb { -IBSchemaSet::IBSchemaSet(Catalog &catalog) : IBCatalogSet(catalog) { +ICSchemaSet::ICSchemaSet(Catalog &catalog) : ICCatalogSet(catalog) { } static bool IsInternalTable(const string &catalog, const string &schema) { @@ -18,38 +18,38 @@ static bool IsInternalTable(const string &catalog, const string &schema) { return false; } -void IBSchemaSet::LoadEntries(ClientContext &context) { +void ICSchemaSet::LoadEntries(ClientContext &context) { if (!entries.empty()) { return; } - auto &ic_catalog = catalog.Cast(); - auto schemas = IBAPI::GetSchemas(catalog.GetName(), ic_catalog.internal_name, ic_catalog.credentials); + auto &ic_catalog = catalog.Cast(); + auto schemas = ICAPI::GetSchemas(catalog.GetName(), ic_catalog.internal_name, ic_catalog.credentials); for (const auto &schema : schemas) { CreateSchemaInfo info; info.schema = schema.schema_name; info.internal = IsInternalTable(schema.catalog_name, schema.schema_name); - auto schema_entry = make_uniq(catalog, info); - schema_entry->schema_data = make_uniq(schema); + auto schema_entry = make_uniq(catalog, info); + schema_entry->schema_data = make_uniq(schema); CreateEntry(std::move(schema_entry)); } } -void IBSchemaSet::FillEntry(ClientContext &context, unique_ptr &entry) { +void ICSchemaSet::FillEntry(ClientContext &context, unique_ptr &entry) { // Nothing to do } -optional_ptr IBSchemaSet::CreateSchema(ClientContext &context, CreateSchemaInfo &info) { - auto &ic_catalog = catalog.Cast(); - auto schema = IBAPI::CreateSchema(catalog.GetName(), ic_catalog.internal_name, info.schema, ic_catalog.credentials); - auto schema_entry = make_uniq(catalog, info); - schema_entry->schema_data = make_uniq(schema); +optional_ptr ICSchemaSet::CreateSchema(ClientContext &context, CreateSchemaInfo &info) { + auto &ic_catalog = catalog.Cast(); + auto schema = ICAPI::CreateSchema(catalog.GetName(), ic_catalog.internal_name, info.schema, ic_catalog.credentials); + auto schema_entry = make_uniq(catalog, info); + schema_entry->schema_data = make_uniq(schema); return CreateEntry(std::move(schema_entry)); } -void IBSchemaSet::DropSchema(ClientContext &context, DropInfo &info) { - auto &ic_catalog = catalog.Cast(); - IBAPI::DropSchema(ic_catalog.internal_name, info.name, ic_catalog.credentials); +void ICSchemaSet::DropSchema(ClientContext &context, DropInfo &info) { + auto &ic_catalog = catalog.Cast(); + ICAPI::DropSchema(ic_catalog.internal_name, info.name, ic_catalog.credentials); DropEntry(context, info); } diff --git a/src/storage/ic_table_entry.cpp b/src/storage/ic_table_entry.cpp index e625ee8..18b9501 100644 --- a/src/storage/ic_table_entry.cpp +++ b/src/storage/ic_table_entry.cpp @@ -19,21 +19,21 @@ namespace duckdb { -IBTableEntry::IBTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTableInfo &info) +ICTableEntry::ICTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTableInfo &info) : TableCatalogEntry(catalog, schema, info) { this->internal = false; } -IBTableEntry::IBTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, IBTableInfo &info) +ICTableEntry::ICTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, ICTableInfo &info) : TableCatalogEntry(catalog, schema, *info.create_info) { this->internal = false; } -unique_ptr IBTableEntry::GetStatistics(ClientContext &context, column_t column_id) { +unique_ptr ICTableEntry::GetStatistics(ClientContext &context, column_t column_id) { return nullptr; } -void IBTableEntry::BindUpdateConstraints(Binder &binder, LogicalGet &, LogicalProjection &, LogicalUpdate &, +void ICTableEntry::BindUpdateConstraints(Binder &binder, LogicalGet &, LogicalProjection &, LogicalUpdate &, ClientContext &) { throw NotImplementedException("BindUpdateConstraints"); } @@ -55,9 +55,9 @@ struct MyIcebergFunctionData : public FunctionData { } }; -TableFunction IBTableEntry::GetScanFunction(ClientContext &context, unique_ptr &bind_data) { +TableFunction ICTableEntry::GetScanFunction(ClientContext &context, unique_ptr &bind_data) { auto &db = DatabaseInstance::GetDatabase(context); - auto &ic_catalog = catalog.Cast(); + auto &ic_catalog = catalog.Cast(); auto &parquet_function_set = ExtensionUtil::GetTableFunction(db, "parquet_scan"); auto parquet_scan_function = parquet_function_set.functions.GetFunctionByArguments(context, {LogicalType::VARCHAR}); @@ -74,8 +74,8 @@ TableFunction IBTableEntry::GetScanFunction(ClientContext &context, unique_ptrstorage_location.find("file://") != 0) { auto &secret_manager = SecretManager::Get(context); - // Get Credentials from IBAPI - auto table_credentials = IBAPI::GetTableCredentials( + // Get Credentials from ICAPI + auto table_credentials = ICAPI::GetTableCredentials( ic_catalog.internal_name, table_data->schema_name, table_data->name, ic_catalog.credentials); // Inject secret into secret manager scoped to this path @@ -138,7 +138,7 @@ TableFunction IBTableEntry::GetScanFunction(ClientContext &context, unique_ptr IBTableSet::_CreateCatalogEntry(ClientContext &context, IBAPITable table) { +unique_ptr ICTableSet::_CreateCatalogEntry(ClientContext &context, ICAPITable table) { D_ASSERT(schema.name == table.schema_name); CreateTableInfo info; info.table = table.name; @@ -33,30 +33,30 @@ unique_ptr IBTableSet::_CreateCatalogEntry(ClientContext &context, info.columns.AddColumn(CreateColumnDefinition(context, col)); } - auto table_entry = make_uniq(catalog, schema, info); - table_entry->table_data = make_uniq(table); + auto table_entry = make_uniq(catalog, schema, info); + table_entry->table_data = make_uniq(table); return table_entry; } -void IBTableSet::FillEntry(ClientContext &context, unique_ptr &entry) { - auto* derived = static_cast(entry.get()); +void ICTableSet::FillEntry(ClientContext &context, unique_ptr &entry) { + auto* derived = static_cast(entry.get()); if (!derived->table_data->storage_location.empty()) { return; } - auto &ic_catalog = catalog.Cast(); - auto table = IBAPI::GetTable(catalog.GetName(), catalog.GetDBPath(), schema.name, entry->name, ic_catalog.credentials); + auto &ic_catalog = catalog.Cast(); + auto table = ICAPI::GetTable(catalog.GetName(), catalog.GetDBPath(), schema.name, entry->name, ic_catalog.credentials); entry = _CreateCatalogEntry(context, table); } -void IBTableSet::LoadEntries(ClientContext &context) { +void ICTableSet::LoadEntries(ClientContext &context) { if (!entries.empty()) { return; } - auto &ic_catalog = catalog.Cast(); + auto &ic_catalog = catalog.Cast(); // TODO: handle out-of-order columns using position property - auto tables = IBAPI::GetTables(catalog.GetName(), catalog.GetDBPath(), schema.name, ic_catalog.credentials); + auto tables = ICAPI::GetTables(catalog.GetName(), catalog.GetDBPath(), schema.name, ic_catalog.credentials); for (auto &table : tables) { auto entry = _CreateCatalogEntry(context, table); @@ -64,41 +64,41 @@ void IBTableSet::LoadEntries(ClientContext &context) { } } -optional_ptr IBTableSet::RefreshTable(ClientContext &context, const string &table_name) { +optional_ptr ICTableSet::RefreshTable(ClientContext &context, const string &table_name) { auto table_info = GetTableInfo(context, schema, table_name); - auto table_entry = make_uniq(catalog, schema, *table_info); + auto table_entry = make_uniq(catalog, schema, *table_info); auto table_ptr = table_entry.get(); CreateEntry(std::move(table_entry)); return table_ptr; } -unique_ptr IBTableSet::GetTableInfo(ClientContext &context, IBSchemaEntry &schema, +unique_ptr ICTableSet::GetTableInfo(ClientContext &context, ICSchemaEntry &schema, const string &table_name) { - throw NotImplementedException("IBTableSet::CreateTable"); + throw NotImplementedException("ICTableSet::CreateTable"); } -optional_ptr IBTableSet::CreateTable(ClientContext &context, BoundCreateTableInfo &info) { - throw NotImplementedException("IBTableSet::CreateTable"); +optional_ptr ICTableSet::CreateTable(ClientContext &context, BoundCreateTableInfo &info) { + throw NotImplementedException("ICTableSet::CreateTable"); } -void IBTableSet::AlterTable(ClientContext &context, RenameTableInfo &info) { - throw NotImplementedException("IBTableSet::AlterTable"); +void ICTableSet::AlterTable(ClientContext &context, RenameTableInfo &info) { + throw NotImplementedException("ICTableSet::AlterTable"); } -void IBTableSet::AlterTable(ClientContext &context, RenameColumnInfo &info) { - throw NotImplementedException("IBTableSet::AlterTable"); +void ICTableSet::AlterTable(ClientContext &context, RenameColumnInfo &info) { + throw NotImplementedException("ICTableSet::AlterTable"); } -void IBTableSet::AlterTable(ClientContext &context, AddColumnInfo &info) { - throw NotImplementedException("IBTableSet::AlterTable"); +void ICTableSet::AlterTable(ClientContext &context, AddColumnInfo &info) { + throw NotImplementedException("ICTableSet::AlterTable"); } -void IBTableSet::AlterTable(ClientContext &context, RemoveColumnInfo &info) { - throw NotImplementedException("IBTableSet::AlterTable"); +void ICTableSet::AlterTable(ClientContext &context, RemoveColumnInfo &info) { + throw NotImplementedException("ICTableSet::AlterTable"); } -void IBTableSet::AlterTable(ClientContext &context, AlterTableInfo &alter) { - throw NotImplementedException("IBTableSet::AlterTable"); +void ICTableSet::AlterTable(ClientContext &context, AlterTableInfo &alter) { + throw NotImplementedException("ICTableSet::AlterTable"); } } // namespace duckdb diff --git a/src/storage/ic_transaction.cpp b/src/storage/ic_transaction.cpp index 57c2d2b..70c29c7 100644 --- a/src/storage/ic_transaction.cpp +++ b/src/storage/ic_transaction.cpp @@ -6,32 +6,32 @@ namespace duckdb { -IBTransaction::IBTransaction(IBCatalog &ic_catalog, TransactionManager &manager, ClientContext &context) +ICTransaction::ICTransaction(ICCatalog &ic_catalog, TransactionManager &manager, ClientContext &context) : Transaction(manager, context), access_mode(ic_catalog.access_mode) { - // connection = IBConnection::Open(ic_catalog.path); + // connection = ICConnection::Open(ic_catalog.path); } -IBTransaction::~IBTransaction() = default; +ICTransaction::~ICTransaction() = default; -void IBTransaction::Start() { - transaction_state = IBTransactionState::TRANSACTION_NOT_YET_STARTED; +void ICTransaction::Start() { + transaction_state = ICTransactionState::TRANSACTION_NOT_YET_STARTED; } -void IBTransaction::Commit() { - if (transaction_state == IBTransactionState::TRANSACTION_STARTED) { - transaction_state = IBTransactionState::TRANSACTION_FINISHED; +void ICTransaction::Commit() { + if (transaction_state == ICTransactionState::TRANSACTION_STARTED) { + transaction_state = ICTransactionState::TRANSACTION_FINISHED; // connection.Execute("COMMIT"); } } -void IBTransaction::Rollback() { - if (transaction_state == IBTransactionState::TRANSACTION_STARTED) { - transaction_state = IBTransactionState::TRANSACTION_FINISHED; +void ICTransaction::Rollback() { + if (transaction_state == ICTransactionState::TRANSACTION_STARTED) { + transaction_state = ICTransactionState::TRANSACTION_FINISHED; // connection.Execute("ROLLBACK"); } } -// IBConnection &IBTransaction::GetConnection() { -// if (transaction_state == IBTransactionState::TRANSACTION_NOT_YET_STARTED) { -// transaction_state = IBTransactionState::TRANSACTION_STARTED; +// ICConnection &ICTransaction::GetConnection() { +// if (transaction_state == ICTransactionState::TRANSACTION_NOT_YET_STARTED) { +// transaction_state = ICTransactionState::TRANSACTION_STARTED; // string query = "START TRANSACTION"; // if (access_mode == AccessMode::READ_ONLY) { // query += " READ ONLY"; @@ -41,9 +41,9 @@ void IBTransaction::Rollback() { // return connection; //} -// unique_ptr IBTransaction::Query(const string &query) { -// if (transaction_state == IBTransactionState::TRANSACTION_NOT_YET_STARTED) { -// transaction_state = IBTransactionState::TRANSACTION_STARTED; +// unique_ptr ICTransaction::Query(const string &query) { +// if (transaction_state == ICTransactionState::TRANSACTION_NOT_YET_STARTED) { +// transaction_state = ICTransactionState::TRANSACTION_STARTED; // string transaction_start = "START TRANSACTION"; // if (access_mode == AccessMode::READ_ONLY) { // transaction_start += " READ ONLY"; @@ -54,8 +54,8 @@ void IBTransaction::Rollback() { // return connection.Query(query); //} -IBTransaction &IBTransaction::Get(ClientContext &context, Catalog &catalog) { - return Transaction::Get(context, catalog).Cast(); +ICTransaction &ICTransaction::Get(ClientContext &context, Catalog &catalog) { + return Transaction::Get(context, catalog).Cast(); } } // namespace duckdb diff --git a/src/storage/ic_transaction_manager.cpp b/src/storage/ic_transaction_manager.cpp index 6b2d3b9..f034de9 100644 --- a/src/storage/ic_transaction_manager.cpp +++ b/src/storage/ic_transaction_manager.cpp @@ -3,12 +3,12 @@ namespace duckdb { -IBTransactionManager::IBTransactionManager(AttachedDatabase &db_p, IBCatalog &ic_catalog) +ICTransactionManager::ICTransactionManager(AttachedDatabase &db_p, ICCatalog &ic_catalog) : TransactionManager(db_p), ic_catalog(ic_catalog) { } -Transaction &IBTransactionManager::StartTransaction(ClientContext &context) { - auto transaction = make_uniq(ic_catalog, *this, context); +Transaction &ICTransactionManager::StartTransaction(ClientContext &context) { + auto transaction = make_uniq(ic_catalog, *this, context); transaction->Start(); auto &result = *transaction; lock_guard l(transaction_lock); @@ -16,23 +16,23 @@ Transaction &IBTransactionManager::StartTransaction(ClientContext &context) { return result; } -ErrorData IBTransactionManager::CommitTransaction(ClientContext &context, Transaction &transaction) { - auto &ic_transaction = transaction.Cast(); +ErrorData ICTransactionManager::CommitTransaction(ClientContext &context, Transaction &transaction) { + auto &ic_transaction = transaction.Cast(); ic_transaction.Commit(); lock_guard l(transaction_lock); transactions.erase(transaction); return ErrorData(); } -void IBTransactionManager::RollbackTransaction(Transaction &transaction) { - auto &ic_transaction = transaction.Cast(); +void ICTransactionManager::RollbackTransaction(Transaction &transaction) { + auto &ic_transaction = transaction.Cast(); ic_transaction.Rollback(); lock_guard l(transaction_lock); transactions.erase(transaction); } -void IBTransactionManager::Checkpoint(ClientContext &context, bool force) { - auto &transaction = IBTransaction::Get(context, db.GetCatalog()); +void ICTransactionManager::Checkpoint(ClientContext &context, bool force) { + auto &transaction = ICTransaction::Get(context, db.GetCatalog()); // auto &db = transaction.GetConnection(); // db.Execute("CHECKPOINT"); } From dc0ebb36f3ab3c738e32313f1983544569060367 Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Fri, 24 Jan 2025 03:05:02 -0500 Subject: [PATCH 13/26] Complete table create & drop --- src/catalog_api.cpp | 202 +++++++++++++++++++-------- src/catalog_utils.cpp | 43 ++++++ src/include/catalog_api.hpp | 7 +- src/include/catalog_utils.hpp | 1 + src/include/storage/ic_table_set.hpp | 1 + src/storage/ic_catalog_set.cpp | 2 - src/storage/ic_schema_entry.cpp | 12 +- src/storage/ic_table_entry.cpp | 1 - src/storage/ic_table_set.cpp | 14 +- 9 files changed, 214 insertions(+), 69 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index d38d9a4..9beafb5 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -1,12 +1,14 @@ #include "catalog_api.hpp" +#include "catalog_utils.hpp" #include "storage/ic_catalog.hpp" #include "yyjson.hpp" + #include #include - #include #include +using namespace duckdb_yyjson; namespace duckdb { //! We use a global here to store the path that is selected on the ICAPI::InitializeCurl call @@ -33,6 +35,15 @@ static string certFileLocations[] = { "/etc/ssl/cert.pem" }; +struct YyjsonDocDeleter { + void operator()(yyjson_doc* doc) { + yyjson_doc_free(doc); + } + void operator()(yyjson_mut_doc* doc) { + yyjson_mut_doc_free(doc); + } +}; + // Look through the the above locations and if one of the files exists, set that as the location curl should use. static bool SelectCurlCertPath() { for (string& caFile : certFileLocations) { @@ -61,8 +72,8 @@ static void InitializeCurlObject(CURL * curl, const string &token) { SetCurlCAFileInfo(curl); } -template -static TYPE TemplatedTryGetYYJson(duckdb_yyjson::yyjson_val *obj, const string &field, TYPE default_val, +template +static TYPE TemplatedTryGetYYJson(yyjson_val *obj, const string &field, TYPE default_val, bool fail_on_missing = true) { auto val = yyjson_obj_get(obj, field.c_str()); if (val && yyjson_get_type(val) == TYPE_NUM) { @@ -73,19 +84,19 @@ static TYPE TemplatedTryGetYYJson(duckdb_yyjson::yyjson_val *obj, const string & throw IOException("Invalid field found while parsing field: " + field); } -static uint64_t TryGetNumFromObject(duckdb_yyjson::yyjson_val *obj, const string &field, bool fail_on_missing = true, +static uint64_t TryGetNumFromObject(yyjson_val *obj, const string &field, bool fail_on_missing = true, uint64_t default_val = 0) { - return TemplatedTryGetYYJson(obj, field, default_val, + return TemplatedTryGetYYJson(obj, field, default_val, fail_on_missing); } -static bool TryGetBoolFromObject(duckdb_yyjson::yyjson_val *obj, const string &field, bool fail_on_missing = false, +static bool TryGetBoolFromObject(yyjson_val *obj, const string &field, bool fail_on_missing = false, bool default_val = false) { - return TemplatedTryGetYYJson(obj, field, default_val, + return TemplatedTryGetYYJson(obj, field, default_val, fail_on_missing); } -static string TryGetStrFromObject(duckdb_yyjson::yyjson_val *obj, const string &field, bool fail_on_missing = true, +static string TryGetStrFromObject(yyjson_val *obj, const string &field, bool fail_on_missing = true, const char *default_val = "") { - return TemplatedTryGetYYJson(obj, field, default_val, + return TemplatedTryGetYYJson(obj, field, default_val, fail_on_missing); } @@ -197,25 +208,26 @@ static string PostRequest( return readBuffer; } -static duckdb_yyjson::yyjson_val *api_result_to_doc(const string &api_result) { - auto *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); +static yyjson_doc *api_result_to_doc(const string &api_result) { + auto *doc = yyjson_read(api_result.c_str(), api_result.size(), 0); auto *root = yyjson_doc_get_root(doc); auto *error = yyjson_obj_get(root, "error"); if (error != NULL) { string err_msg = TryGetStrFromObject(error, "message"); throw std::runtime_error(err_msg); } - return root; + return doc; } -static duckdb_yyjson::yyjson_val *GetTableMetadata(const string &internal, const string &schema, const string &table, ICCredentials credentials) { +static string GetTableMetadata(const string &internal, const string &schema, const string &table, ICCredentials credentials) { struct curl_slist *extra_headers = NULL; extra_headers = curl_slist_append(extra_headers, "X-Iceberg-Access-Delegation: vended-credentials"); - auto api_result = GetRequest( + string api_result = GetRequest( credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables/" + table, credentials.token, extra_headers); - return api_result_to_doc(api_result); + curl_slist_free_all(extra_headers); + return api_result; } void ICAPI::InitializeCurl() { @@ -226,7 +238,7 @@ vector ICAPI::GetCatalogs(const string &catalog, ICCredentials credentia throw NotImplementedException("ICAPI::GetCatalogs"); } -static ICAPIColumnDefinition ParseColumnDefinition(duckdb_yyjson::yyjson_val *column_def) { +static ICAPIColumnDefinition ParseColumnDefinition(yyjson_val *column_def) { ICAPIColumnDefinition result; result.name = TryGetStrFromObject(column_def, "name"); result.type_text = TryGetStrFromObject(column_def, "type"); @@ -238,64 +250,77 @@ static ICAPIColumnDefinition ParseColumnDefinition(duckdb_yyjson::yyjson_val *co ICAPITableCredentials ICAPI::GetTableCredentials(const string &internal, const string &schema, const string &table, ICCredentials credentials) { ICAPITableCredentials result; - duckdb_yyjson::yyjson_val *root = GetTableMetadata(internal, schema, table, credentials); + string api_result = GetTableMetadata(internal, schema, table, credentials); + std::unique_ptr doc(api_result_to_doc(api_result)); + auto *root = yyjson_doc_get_root(doc.get()); auto *aws_temp_credentials = yyjson_obj_get(root, "config"); - if (aws_temp_credentials) { result.key_id = TryGetStrFromObject(aws_temp_credentials, "s3.access-key-id"); result.secret = TryGetStrFromObject(aws_temp_credentials, "s3.secret-access-key"); result.session_token = TryGetStrFromObject(aws_temp_credentials, "s3.session-token"); + return result; } - return result; + throw std::runtime_error("No AWS credentials found for table"); } string ICAPI::GetToken(string id, string secret, string endpoint) { string post_data = "grant_type=client_credentials&client_id=" + id + "&client_secret=" + secret + "&scope=PRINCIPAL_ROLE:ALL"; string api_result = PostRequest(endpoint + "/v1/oauth/tokens", post_data); - auto *root = api_result_to_doc(api_result); + std::unique_ptr doc(api_result_to_doc(api_result)); + auto *root = yyjson_doc_get_root(doc.get()); return TryGetStrFromObject(root, "access_token"); } -ICAPITable ICAPI::GetTable( - const string &catalog, const string &internal, const string &schema, const string &table, std::optional credentials) { - +static void populateTableMetadata(ICAPITable &table, yyjson_val *metadata_root) { + table.storage_location = TryGetStrFromObject(metadata_root, "metadata-location"); + auto *metadata = yyjson_obj_get(metadata_root, "metadata"); + //table_result.table_id = TryGetStrFromObject(metadata, "table-uuid"); + + uint64_t current_schema_id = TryGetNumFromObject(metadata, "current-schema-id"); + auto *schemas = yyjson_obj_get(metadata, "schemas"); + yyjson_val *schema; + size_t schema_idx, schema_max; + bool found = false; + yyjson_arr_foreach(schemas, schema_idx, schema_max, schema) { + uint64_t schema_id = TryGetNumFromObject(schema, "schema-id"); + if (schema_id == current_schema_id) { + found = true; + auto *columns = yyjson_obj_get(schema, "fields"); + yyjson_val *col; + size_t col_idx, col_max; + yyjson_arr_foreach(columns, col_idx, col_max, col) { + auto column_definition = ParseColumnDefinition(col); + table.columns.push_back(column_definition); + } + } + } + + if (!found) { + throw InternalException("Current schema not found"); + } +} + +static ICAPITable createTable(const string &catalog, const string &schema, const string &table_name) { ICAPITable table_result; table_result.catalog_name = catalog; table_result.schema_name = schema; - table_result.name = table; + table_result.name = table_name; table_result.data_source_format = "ICEBERG"; table_result.table_id = "uuid-" + schema + "-" + "table"; std::replace(table_result.table_id.begin(), table_result.table_id.end(), '_', '-'); + return table_result; +} +ICAPITable ICAPI::GetTable( + const string &catalog, const string &internal, const string &schema, const string &table_name, std::optional credentials) { + + ICAPITable table_result = createTable(catalog, schema, table_name); if (credentials) { - auto *metadata_root = GetTableMetadata(internal, schema, table_result.name, *credentials); - table_result.storage_location = TryGetStrFromObject(metadata_root, "metadata-location"); - auto *metadata = yyjson_obj_get(metadata_root, "metadata"); - //table_result.table_id = TryGetStrFromObject(metadata, "table-uuid"); - - uint64_t current_schema_id = TryGetNumFromObject(metadata, "current-schema-id"); - auto *schemas = yyjson_obj_get(metadata, "schemas"); - duckdb_yyjson::yyjson_val *schema; - size_t schema_idx, schema_max; - bool found = false; - yyjson_arr_foreach(schemas, schema_idx, schema_max, schema) { - uint64_t schema_id = TryGetNumFromObject(schema, "schema-id"); - if (schema_id == current_schema_id) { - found = true; - auto *columns = yyjson_obj_get(schema, "fields"); - duckdb_yyjson::yyjson_val *col; - size_t col_idx, col_max; - yyjson_arr_foreach(columns, col_idx, col_max, col) { - auto column_definition = ParseColumnDefinition(col); - table_result.columns.push_back(column_definition); - } - } - } - - if (!found) { - throw InternalException("Current schema not found"); - } + string result = GetTableMetadata(internal, schema, table_result.name, *credentials); + std::unique_ptr doc(api_result_to_doc(result)); + auto *metadata_root = yyjson_doc_get_root(doc.get()); + populateTableMetadata(table_result, metadata_root); } else { // Skip fetching metadata, we'll do it later when we access the table ICAPIColumnDefinition col; @@ -313,11 +338,12 @@ ICAPITable ICAPI::GetTable( // TODO: handle out-of-order columns using position property vector ICAPI::GetTables(const string &catalog, const string &internal, const string &schema, ICCredentials credentials) { vector result; - auto api_result = GetRequest(credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables", credentials.token); - auto *root = api_result_to_doc(api_result); + string api_result = GetRequest(credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables", credentials.token); + std::unique_ptr doc(api_result_to_doc(api_result)); + auto *root = yyjson_doc_get_root(doc.get()); auto *tables = yyjson_obj_get(root, "identifiers"); size_t idx, max; - duckdb_yyjson::yyjson_val *table; + yyjson_val *table; yyjson_arr_foreach(tables, idx, max, table) { auto table_result = GetTable(catalog, internal, schema, TryGetStrFromObject(table, "name"), std::nullopt); result.push_back(table_result); @@ -328,16 +354,17 @@ vector ICAPI::GetTables(const string &catalog, const string &interna vector ICAPI::GetSchemas(const string &catalog, const string &internal, ICCredentials credentials) { vector result; - auto api_result = + string api_result = GetRequest(credentials.endpoint + "/v1/" + internal + "/namespaces", credentials.token); - auto *root = api_result_to_doc(api_result); + std::unique_ptr doc(api_result_to_doc(api_result)); + auto *root = yyjson_doc_get_root(doc.get()); auto *schemas = yyjson_obj_get(root, "namespaces"); size_t idx, max; - duckdb_yyjson::yyjson_val *schema; + yyjson_val *schema; yyjson_arr_foreach(schemas, idx, max, schema) { ICAPISchema schema_result; schema_result.catalog_name = catalog; - duckdb_yyjson::yyjson_val *value = yyjson_arr_get(schema, 0); + yyjson_val *value = yyjson_arr_get(schema, 0); schema_result.schema_name = yyjson_get_str(value); result.push_back(schema_result); } @@ -363,4 +390,63 @@ void ICAPI::DropSchema(const string &internal, const string &schema, ICCredentia api_result_to_doc(api_result); // if the method returns, request was successful } +void ICAPI::DropTable(const string &catalog, const string &internal, const string &schema, string &table_name, ICCredentials credentials) { + string api_result = DeleteRequest( + credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables/" + table_name + "?purgeRequested=true", + credentials.token); + api_result_to_doc(api_result); // if the method returns, request was successful +} + +static std::string json_to_string(yyjson_mut_doc *doc, yyjson_write_flag flags = YYJSON_WRITE_PRETTY) { + char *json_chars = yyjson_mut_write(doc, flags, NULL); + std::string json_str(json_chars); + free(json_chars); + return json_str; +} + +ICAPITable ICAPI::CreateTable(const string &catalog, const string &internal, const string &schema, ICCredentials credentials, CreateTableInfo *table_info) { + std::unique_ptr dd(yyjson_mut_doc_new(NULL)); + yyjson_mut_val *rr = yyjson_mut_obj(dd.get()); + yyjson_mut_doc_set_root(dd.get(), rr); + yyjson_mut_obj_add_str(dd.get(), rr, "name", table_info->table.c_str()); + + yyjson_mut_val *sch = yyjson_mut_obj(dd.get()); + yyjson_mut_obj_add_val(dd.get(), rr, "schema", sch); + yyjson_mut_obj_add_str(dd.get(), sch, "type", "struct"); + + yyjson_mut_val *fields = yyjson_mut_arr(dd.get()); + yyjson_mut_obj_add_val(dd.get(), sch, "fields", fields); + + std::vector column_names; + std::vector column_types; + for (auto &col : table_info->columns.Logical()) { + // Store column name and type in vectors + column_names.push_back(col.GetName()); + column_types.push_back(ICUtils::LogicalToIcebergType(col.GetType())); + // Add column object to JSON + yyjson_mut_val *col_obj = yyjson_mut_obj(dd.get()); + yyjson_mut_obj_add_int(dd.get(), col_obj, "id", col.Oid()); + yyjson_mut_obj_add_bool(dd.get(), col_obj, "required", true); + yyjson_mut_obj_add_str(dd.get(), col_obj, "name", column_names.back().c_str()); + yyjson_mut_obj_add_str(dd.get(), col_obj, "type", column_types.back().c_str()); + yyjson_mut_arr_add_val(fields, col_obj); + } + + yyjson_mut_val *props = yyjson_mut_obj(dd.get()); + yyjson_mut_obj_add_val(dd.get(), rr, "properties", props); + yyjson_mut_obj_add_str(dd.get(), props, "write.parquet.compression-codec", "snappy"); + + ICAPITable table_result = createTable(catalog, schema, table_info->table); + string post_data = json_to_string(dd.get()); + struct curl_slist *extra_headers = NULL; + extra_headers = curl_slist_append(extra_headers, "X-Iceberg-Access-Delegation: vended-credentials"); + string api_result = PostRequest( + credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables", post_data, "json", credentials.token, extra_headers); + curl_slist_free_all(extra_headers); + std::unique_ptr doc(api_result_to_doc(api_result)); + auto *root = yyjson_doc_get_root(doc.get()); + populateTableMetadata(table_result, root); + return table_result; +} + } // namespace duckdb diff --git a/src/catalog_utils.cpp b/src/catalog_utils.cpp index f71b41e..075392f 100644 --- a/src/catalog_utils.cpp +++ b/src/catalog_utils.cpp @@ -7,6 +7,49 @@ namespace duckdb { +string ICUtils::LogicalToIcebergType(const LogicalType &input) { + switch (input.id()) { + case LogicalType::TINYINT: + case LogicalType::UTINYINT: + return "tinyint"; + case LogicalType::SMALLINT: + case LogicalType::USMALLINT: + return "smallint"; + case LogicalType::INTEGER: + case LogicalType::UINTEGER: + return "int"; + case LogicalType::BIGINT: + case LogicalType::UBIGINT: + return "long"; + case LogicalType::VARCHAR: + return "string"; + case LogicalType::DOUBLE: + return "double"; + case LogicalType::FLOAT: + return "float"; + case LogicalType::BOOLEAN: + return "boolean"; + case LogicalType::TIMESTAMP: + return "timestamp"; + case LogicalType::TIMESTAMP_TZ: + return "timestamptz"; + case LogicalType::BLOB: + return "binary"; + case LogicalType::DATE: + return "date"; + case LogicalTypeId::DECIMAL: + uint8_t precision = DecimalType::GetWidth(input); + uint8_t scale = DecimalType::GetScale(input); + return "decimal(" + std::to_string(precision) + ", " + std::to_string(scale) + ")"; + // case LogicalTypeId::ARRAY: + // case LogicalTypeId::STRUCT: + // case LogicalTypeId::MAP: + // default: + } + + throw std::runtime_error("Unsupported type: " + input.ToString()); +} + string ICUtils::TypeToString(const LogicalType &input) { switch (input.id()) { case LogicalType::VARCHAR: diff --git a/src/include/catalog_api.hpp b/src/include/catalog_api.hpp index b2d0683..18613dd 100644 --- a/src/include/catalog_api.hpp +++ b/src/include/catalog_api.hpp @@ -2,6 +2,7 @@ #pragma once #include "duckdb/common/types.hpp" +#include "duckdb/parser/parsed_data/create_table_info.hpp" namespace duckdb { struct ICCredentials; @@ -22,7 +23,6 @@ struct ICAPITable { string schema_name; string table_type; string data_source_format; - string storage_location; vector columns; @@ -47,11 +47,14 @@ class ICAPI { static ICAPITableCredentials GetTableCredentials(const string &internal, const string &schema, const string &table, ICCredentials credentials); static vector GetCatalogs(const string &catalog, ICCredentials credentials); static vector GetTables(const string &catalog, const string &internal, const string &schema, ICCredentials credentials); - static ICAPITable GetTable(const string &catalog, const string &internal, const string &schema, const string &table, std::optional credentials); + static ICAPITable GetTable(const string &catalog, const string &internal, const string &schema, const string &table_name, std::optional credentials); static vector GetSchemas(const string &catalog, const string &internal, ICCredentials credentials); static vector GetTablesInSchema(const string &catalog, const string &schema, ICCredentials credentials); static string GetToken(string id, string secret, string endpoint); static ICAPISchema CreateSchema(const string &catalog, const string &internal, const string &schema, ICCredentials credentials); static void DropSchema(const string &internal, const string &schema, ICCredentials credentials); + static ICAPITable CreateTable(const string &catalog, const string &internal, const string &schema, ICCredentials credentials, CreateTableInfo *table_info); + static void DropTable(const string &catalog, const string &internal, const string &schema, string &table_name, ICCredentials credentials); }; + } // namespace duckdb diff --git a/src/include/catalog_utils.hpp b/src/include/catalog_utils.hpp index 240e63b..9814272 100644 --- a/src/include/catalog_utils.hpp +++ b/src/include/catalog_utils.hpp @@ -21,6 +21,7 @@ class ICUtils { static LogicalType ToUCType(const LogicalType &input); static LogicalType TypeToLogicalType(ClientContext &context, const string &columnDefinition); static string TypeToString(const LogicalType &input); + static string LogicalToIcebergType(const LogicalType &input); }; } // namespace duckdb diff --git a/src/include/storage/ic_table_set.hpp b/src/include/storage/ic_table_set.hpp index 7e41593..f1746db 100644 --- a/src/include/storage/ic_table_set.hpp +++ b/src/include/storage/ic_table_set.hpp @@ -30,6 +30,7 @@ class ICTableSet : public ICInSchemaSet { static unique_ptr GetTableInfo(ClientContext &context, ICSchemaEntry &schema, const string &table_name); optional_ptr RefreshTable(ClientContext &context, const string &table_name); void AlterTable(ClientContext &context, AlterTableInfo &info); + void DropTable(ClientContext &context, DropInfo &info); protected: void LoadEntries(ClientContext &context) override; diff --git a/src/storage/ic_catalog_set.cpp b/src/storage/ic_catalog_set.cpp index c3b01a1..6297623 100644 --- a/src/storage/ic_catalog_set.cpp +++ b/src/storage/ic_catalog_set.cpp @@ -20,8 +20,6 @@ optional_ptr ICCatalogSet::GetEntry(ClientContext &context, const } void ICCatalogSet::DropEntry(ClientContext &context, DropInfo &info) { - std::cout << "ICCatalogSet::DropEntry" << info.type << std::endl; - EraseEntryInternal(info.name); } diff --git a/src/storage/ic_schema_entry.cpp b/src/storage/ic_schema_entry.cpp index 691c9b0..4046ac7 100644 --- a/src/storage/ic_schema_entry.cpp +++ b/src/storage/ic_schema_entry.cpp @@ -36,6 +36,14 @@ optional_ptr ICSchemaEntry::CreateTable(CatalogTransaction transac return tables.CreateTable(transaction.GetContext(), info); } +void ICSchemaEntry::DropEntry(ClientContext &context, DropInfo &info) { + if (info.type != CatalogType::TABLE_ENTRY) { + throw BinderException("Expecting table entry"); + } + tables.DropTable(context, info); + GetCatalogSet(info.type).DropEntry(context, info); +} + optional_ptr ICSchemaEntry::CreateFunction(CatalogTransaction transaction, CreateFunctionInfo &info) { throw BinderException("PC databases do not support creating functions"); } @@ -136,10 +144,6 @@ void ICSchemaEntry::Scan(CatalogType type, const std::function ICSchemaEntry::GetEntry(CatalogTransaction transaction, CatalogType type, const string &name) { if (!CatalogTypeIsSupported(type)) { diff --git a/src/storage/ic_table_entry.cpp b/src/storage/ic_table_entry.cpp index 18b9501..5bf7511 100644 --- a/src/storage/ic_table_entry.cpp +++ b/src/storage/ic_table_entry.cpp @@ -77,7 +77,6 @@ TableFunction ICTableEntry::GetScanFunction(ClientContext &context, unique_ptrschema_name, table_data->name, ic_catalog.credentials); - // Inject secret into secret manager scoped to this path CreateSecretInfo info(OnCreateConflict::REPLACE_ON_CONFLICT, SecretPersistType::TEMPORARY); info.name = "__internal_ic_" + table_data->table_id; diff --git a/src/storage/ic_table_set.cpp b/src/storage/ic_table_set.cpp index 6866ea9..6e1f284 100644 --- a/src/storage/ic_table_set.cpp +++ b/src/storage/ic_table_set.cpp @@ -9,6 +9,7 @@ #include "duckdb/parser/constraints/unique_constraint.hpp" #include "duckdb/parser/expression/constant_expression.hpp" #include "duckdb/planner/parsed_data/bound_create_table_info.hpp" +#include "duckdb/parser/parsed_data/drop_info.hpp" #include "duckdb/catalog/dependency_list.hpp" #include "duckdb/parser/parsed_data/create_table_info.hpp" #include "duckdb/parser/constraints/list.hpp" @@ -74,11 +75,20 @@ optional_ptr ICTableSet::RefreshTable(ClientContext &context, cons unique_ptr ICTableSet::GetTableInfo(ClientContext &context, ICSchemaEntry &schema, const string &table_name) { - throw NotImplementedException("ICTableSet::CreateTable"); + throw NotImplementedException("ICTableSet::GetTableInfo"); } optional_ptr ICTableSet::CreateTable(ClientContext &context, BoundCreateTableInfo &info) { - throw NotImplementedException("ICTableSet::CreateTable"); + auto &ic_catalog = catalog.Cast(); + auto *table_info = dynamic_cast(info.base.get()); + auto table = ICAPI::CreateTable(catalog.GetName(), ic_catalog.internal_name, schema.name, ic_catalog.credentials, table_info); + auto entry = _CreateCatalogEntry(context, table); + return CreateEntry(std::move(entry)); +} + +void ICTableSet::DropTable(ClientContext &context, DropInfo &info) { + auto &ic_catalog = catalog.Cast(); + ICAPI::DropTable(catalog.GetName(), ic_catalog.internal_name, schema.name, info.name, ic_catalog.credentials); } void ICTableSet::AlterTable(ClientContext &context, RenameTableInfo &info) { From 6855d238c0b7455c5f68bfcd1fd4b6893f147811 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Wed, 22 Jan 2025 16:30:35 +0100 Subject: [PATCH 14/26] remove isdirectory check because this breaks reading from HTTPFileSystem --- src/common/iceberg.cpp | 22 ++++++++++++---------- test/sql/iceberg_snapshots.test | 2 +- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/common/iceberg.cpp b/src/common/iceberg.cpp index 3297b57..7a3768b 100644 --- a/src/common/iceberg.cpp +++ b/src/common/iceberg.cpp @@ -197,28 +197,30 @@ string IcebergSnapshot::GetMetaDataPath(ClientContext &context, const string &pa if (StringUtil::EndsWith(path, ".json")) { // We've been given a real metadata path. Nothing else to do. return path; - } else if (!fs.DirectoryExists(meta_path)) { - // Make sure we have a metadata directory to look in - throw IOException("Cannot open \"%s\": Metadata directory does not exist", path); - } else if(StringUtil::EndsWith(table_version, ".text")||StringUtil::EndsWith(table_version, ".txt")) { + } + + if(StringUtil::EndsWith(table_version, ".text")||StringUtil::EndsWith(table_version, ".txt")) { // We were given a hint filename version_hint = GetTableVersionFromHint(meta_path, fs, table_version); return GenerateMetaDataUrl(fs, meta_path, version_hint, metadata_compression_codec, version_format); - } else if (table_version != UNKNOWN_TABLE_VERSION) { + } + if (table_version != UNKNOWN_TABLE_VERSION) { // We were given an explicit version number version_hint = table_version; return GenerateMetaDataUrl(fs, meta_path, version_hint, metadata_compression_codec, version_format); - } else if (fs.FileExists(fs.JoinPath(meta_path, DEFAULT_VERSION_HINT_FILE))) { + } + if (fs.FileExists(fs.JoinPath(meta_path, DEFAULT_VERSION_HINT_FILE))) { // We're guessing, but a version-hint.text exists so we'll use that version_hint = GetTableVersionFromHint(meta_path, fs, DEFAULT_VERSION_HINT_FILE); return GenerateMetaDataUrl(fs, meta_path, version_hint, metadata_compression_codec, version_format); - } else if (!UnsafeVersionGuessingEnabled(context)) { + } + if (!UnsafeVersionGuessingEnabled(context)) { // Make sure we're allowed to guess versions throw InvalidInputException("No version was provided and no version-hint could be found, globbing the filesystem to locate the latest version is disabled by default as this is considered unsafe and could result in reading uncommitted data. To enable this use 'SET %s = true;'", VERSION_GUESSING_CONFIG_VARIABLE); - } else { - // We are allowed to guess to guess from file paths - return GuessTableVersion(meta_path, fs, table_version, metadata_compression_codec, version_format); } + + // We are allowed to guess to guess from file paths + return GuessTableVersion(meta_path, fs, table_version, metadata_compression_codec, version_format); } diff --git a/test/sql/iceberg_snapshots.test b/test/sql/iceberg_snapshots.test index 40125b9..88f4b45 100644 --- a/test/sql/iceberg_snapshots.test +++ b/test/sql/iceberg_snapshots.test @@ -41,7 +41,7 @@ SELECT * FROM ICEBERG_SNAPSHOTS('data/iceberg/lineitem_iceberg', version='1'); statement error SELECT * FROM ICEBERG_SNAPSHOTS('data/iceberg/lineitem_iceberg_nonexistent'); ---- -IO Error: Cannot open "data/iceberg/lineitem_iceberg_nonexistent": Metadata directory does not exist +Invalid Input Error: No version was provided and no version-hint could be found statement error SELECT * FROM ICEBERG_SNAPSHOTS('data/iceberg/lineitem_iceberg_gz'); From 7307b1d549dbe5da48fc7ca7af9ff711162523b1 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Fri, 24 Jan 2025 11:01:14 +0100 Subject: [PATCH 15/26] fix tests, improve message --- src/common/iceberg.cpp | 3 +-- test/sql/iceberg_metadata.test | 2 +- test/sql/iceberg_snapshots.test | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/common/iceberg.cpp b/src/common/iceberg.cpp index 7a3768b..2d224d5 100644 --- a/src/common/iceberg.cpp +++ b/src/common/iceberg.cpp @@ -198,7 +198,6 @@ string IcebergSnapshot::GetMetaDataPath(ClientContext &context, const string &pa // We've been given a real metadata path. Nothing else to do. return path; } - if(StringUtil::EndsWith(table_version, ".text")||StringUtil::EndsWith(table_version, ".txt")) { // We were given a hint filename version_hint = GetTableVersionFromHint(meta_path, fs, table_version); @@ -216,7 +215,7 @@ string IcebergSnapshot::GetMetaDataPath(ClientContext &context, const string &pa } if (!UnsafeVersionGuessingEnabled(context)) { // Make sure we're allowed to guess versions - throw InvalidInputException("No version was provided and no version-hint could be found, globbing the filesystem to locate the latest version is disabled by default as this is considered unsafe and could result in reading uncommitted data. To enable this use 'SET %s = true;'", VERSION_GUESSING_CONFIG_VARIABLE); + throw IOException("Failed to read iceberg table. No version was provided and no version-hint could be found, globbing the filesystem to locate the latest version is disabled by default as this is considered unsafe and could result in reading uncommitted data. To enable this use 'SET %s = true;'", VERSION_GUESSING_CONFIG_VARIABLE); } // We are allowed to guess to guess from file paths diff --git a/test/sql/iceberg_metadata.test b/test/sql/iceberg_metadata.test index cae75df..9bb8e2f 100644 --- a/test/sql/iceberg_metadata.test +++ b/test/sql/iceberg_metadata.test @@ -51,7 +51,7 @@ lineitem_iceberg_gz/metadata/23f9dbea-1e7f-4694-a82c-dc3c9a94953e-m0.avro 0 DATA statement error SELECT * FROM ICEBERG_METADATA('data/iceberg/lineitem_iceberg_nonexistent'); ---- -IO Error: Cannot open "data/iceberg/lineitem_iceberg_nonexistent": Metadata directory does not exist +IO Error: Failed to read iceberg table. No version was provided and no version-hint could be found, statement error SELECT * FROM ICEBERG_METADATA('data/iceberg/lineitem_iceberg_no_hint', ALLOW_MOVED_PATHS=TRUE); diff --git a/test/sql/iceberg_snapshots.test b/test/sql/iceberg_snapshots.test index 88f4b45..ac05270 100644 --- a/test/sql/iceberg_snapshots.test +++ b/test/sql/iceberg_snapshots.test @@ -41,7 +41,7 @@ SELECT * FROM ICEBERG_SNAPSHOTS('data/iceberg/lineitem_iceberg', version='1'); statement error SELECT * FROM ICEBERG_SNAPSHOTS('data/iceberg/lineitem_iceberg_nonexistent'); ---- -Invalid Input Error: No version was provided and no version-hint could be found +IO Error: Failed to read iceberg table. No version was provided and no version-hint could be found, statement error SELECT * FROM ICEBERG_SNAPSHOTS('data/iceberg/lineitem_iceberg_gz'); From 9da0c08f5500a1c80c276c55cf15af483c73f7c0 Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Sun, 2 Feb 2025 02:36:54 -0500 Subject: [PATCH 16/26] Reserve space for vecs so they don't get reallocated --- .gitmodules | 3 +++ arrow | 1 + src/catalog_api.cpp | 54 ++++++++++++++++++++++++++------------------- 3 files changed, 35 insertions(+), 23 deletions(-) create mode 160000 arrow diff --git a/.gitmodules b/.gitmodules index dd490ea..b06efe8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,3 +5,6 @@ [submodule "extension-ci-tools"] path = extension-ci-tools url = https://github.com/duckdb/extension-ci-tools.git +[submodule "arrow"] + path = arrow + url = https://github.com/apache/arrow.git diff --git a/arrow b/arrow new file mode 160000 index 0000000..f872372 --- /dev/null +++ b/arrow @@ -0,0 +1 @@ +Subproject commit f8723722341df31c0091c91ec451319ded58c214 diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index 9beafb5..af70ff0 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -397,54 +397,62 @@ void ICAPI::DropTable(const string &catalog, const string &internal, const strin api_result_to_doc(api_result); // if the method returns, request was successful } -static std::string json_to_string(yyjson_mut_doc *doc, yyjson_write_flag flags = YYJSON_WRITE_PRETTY) { - char *json_chars = yyjson_mut_write(doc, flags, NULL); - std::string json_str(json_chars); +static std::string json_to_string(yyjson_mut_doc *doc) { + size_t len = 0; + yyjson_write_flag flags = YYJSON_WRITE_NOFLAG; + char *json_chars = yyjson_mut_write(doc, flags, &len); + std::string json_str(json_chars, len); free(json_chars); return json_str; } ICAPITable ICAPI::CreateTable(const string &catalog, const string &internal, const string &schema, ICCredentials credentials, CreateTableInfo *table_info) { std::unique_ptr dd(yyjson_mut_doc_new(NULL)); - yyjson_mut_val *rr = yyjson_mut_obj(dd.get()); - yyjson_mut_doc_set_root(dd.get(), rr); - yyjson_mut_obj_add_str(dd.get(), rr, "name", table_info->table.c_str()); + yyjson_mut_doc *doc = dd.get(); - yyjson_mut_val *sch = yyjson_mut_obj(dd.get()); - yyjson_mut_obj_add_val(dd.get(), rr, "schema", sch); - yyjson_mut_obj_add_str(dd.get(), sch, "type", "struct"); + yyjson_mut_val *rr = yyjson_mut_obj(doc); + yyjson_mut_doc_set_root(doc, rr); + yyjson_mut_obj_add_str(doc, rr, "name", table_info->table.c_str()); - yyjson_mut_val *fields = yyjson_mut_arr(dd.get()); - yyjson_mut_obj_add_val(dd.get(), sch, "fields", fields); + yyjson_mut_val *sch = yyjson_mut_obj(doc); + yyjson_mut_obj_add_val(doc, rr, "schema", sch); + yyjson_mut_obj_add_str(doc, sch, "type", "struct"); + + yyjson_mut_val *fields = yyjson_mut_arr(doc); + yyjson_mut_obj_add_val(doc, sch, "fields", fields); std::vector column_names; std::vector column_types; + size_t num_columns = table_info->columns.LogicalColumnCount(); + column_names.reserve(num_columns); + column_types.reserve(num_columns); for (auto &col : table_info->columns.Logical()) { // Store column name and type in vectors column_names.push_back(col.GetName()); column_types.push_back(ICUtils::LogicalToIcebergType(col.GetType())); // Add column object to JSON - yyjson_mut_val *col_obj = yyjson_mut_obj(dd.get()); - yyjson_mut_obj_add_int(dd.get(), col_obj, "id", col.Oid()); - yyjson_mut_obj_add_bool(dd.get(), col_obj, "required", true); - yyjson_mut_obj_add_str(dd.get(), col_obj, "name", column_names.back().c_str()); - yyjson_mut_obj_add_str(dd.get(), col_obj, "type", column_types.back().c_str()); + yyjson_mut_val *col_obj = yyjson_mut_obj(doc); + yyjson_mut_obj_add_int(doc, col_obj, "id", col.Oid()); + yyjson_mut_obj_add_bool(doc, col_obj, "required", true); + yyjson_mut_obj_add_str(doc, col_obj, "name", column_names.back().c_str()); + yyjson_mut_obj_add_str(doc, col_obj, "type", column_types.back().c_str()); yyjson_mut_arr_add_val(fields, col_obj); } - yyjson_mut_val *props = yyjson_mut_obj(dd.get()); - yyjson_mut_obj_add_val(dd.get(), rr, "properties", props); - yyjson_mut_obj_add_str(dd.get(), props, "write.parquet.compression-codec", "snappy"); + yyjson_mut_val *props = yyjson_mut_obj(doc); + yyjson_mut_obj_add_val(doc, rr, "properties", props); + yyjson_mut_obj_add_str(doc, props, "write.parquet.compression-codec", "snappy"); + string post_data = json_to_string(doc); - ICAPITable table_result = createTable(catalog, schema, table_info->table); - string post_data = json_to_string(dd.get()); struct curl_slist *extra_headers = NULL; extra_headers = curl_slist_append(extra_headers, "X-Iceberg-Access-Delegation: vended-credentials"); string api_result = PostRequest( credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables", post_data, "json", credentials.token, extra_headers); curl_slist_free_all(extra_headers); - std::unique_ptr doc(api_result_to_doc(api_result)); - auto *root = yyjson_doc_get_root(doc.get()); + + std::unique_ptr dd2(api_result_to_doc(api_result)); + auto *root = yyjson_doc_get_root(dd2.get()); + ICAPITable table_result = createTable(catalog, schema, table_info->table); populateTableMetadata(table_result, root); return table_result; } From 41c6dc77007f850c8d2eb20b5be43d7df60208f1 Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Tue, 4 Feb 2025 21:46:49 -0500 Subject: [PATCH 17/26] CTAS WIP --- .gitignore | 2 + CMakeLists.txt | 22 +- README-ADDENDUM.md | 56 +++ jiceberg_lib/.gitattributes | 12 + jiceberg_lib/.gitignore | 6 + jiceberg_lib/README.md | 19 + jiceberg_lib/app/build.gradle | 75 +++ .../main/java/com/fivetran/iceberg/App.java | 161 +++++++ .../native-image-config/jni-config.json | 18 + .../predefined-classes-config.json | 8 + .../native-image-config/proxy-config.json | 2 + .../native-image-config/reflect-config.json | 436 ++++++++++++++++++ .../native-image-config/resource-config.json | 63 +++ .../serialization-config.json | 8 + jiceberg_lib/gradle/libs.versions.toml | 8 + .../gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 43504 bytes .../gradle/wrapper/gradle-wrapper.properties | 7 + jiceberg_lib/gradlew | 252 ++++++++++ jiceberg_lib/gradlew.bat | 94 ++++ jiceberg_lib/scripts/copy_headers.sh | 18 + jiceberg_lib/settings.gradle | 14 + src/catalog_api.cpp | 4 +- src/ic_create_table_as_op.cpp | 219 +++++++++ src/iceberg_extension.cpp | 14 +- src/include/catalog_api.hpp | 2 +- src/include/ic_create_table_as_op.hpp | 35 ++ src/storage/ic_catalog.cpp | 24 +- 27 files changed, 1567 insertions(+), 12 deletions(-) create mode 100644 README-ADDENDUM.md create mode 100644 jiceberg_lib/.gitattributes create mode 100644 jiceberg_lib/.gitignore create mode 100644 jiceberg_lib/README.md create mode 100644 jiceberg_lib/app/build.gradle create mode 100644 jiceberg_lib/app/src/main/java/com/fivetran/iceberg/App.java create mode 100644 jiceberg_lib/app/src/main/resources/META-INF/native-image-config/jni-config.json create mode 100644 jiceberg_lib/app/src/main/resources/META-INF/native-image-config/predefined-classes-config.json create mode 100644 jiceberg_lib/app/src/main/resources/META-INF/native-image-config/proxy-config.json create mode 100644 jiceberg_lib/app/src/main/resources/META-INF/native-image-config/reflect-config.json create mode 100644 jiceberg_lib/app/src/main/resources/META-INF/native-image-config/resource-config.json create mode 100644 jiceberg_lib/app/src/main/resources/META-INF/native-image-config/serialization-config.json create mode 100644 jiceberg_lib/gradle/libs.versions.toml create mode 100644 jiceberg_lib/gradle/wrapper/gradle-wrapper.jar create mode 100644 jiceberg_lib/gradle/wrapper/gradle-wrapper.properties create mode 100755 jiceberg_lib/gradlew create mode 100644 jiceberg_lib/gradlew.bat create mode 100755 jiceberg_lib/scripts/copy_headers.sh create mode 100644 jiceberg_lib/settings.gradle create mode 100644 src/ic_create_table_as_op.cpp create mode 100644 src/include/ic_create_table_as_op.hpp diff --git a/.gitignore b/.gitignore index cb608ef..0d2864b 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ data/iceberg/generated_* scripts/metastore_db/ scripts/derby.log scripts/test-script-with-path.sql +*.parquet +src/include/jiceberg_generated/ diff --git a/CMakeLists.txt b/CMakeLists.txt index cd7bd25..6c4c1ae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,13 +8,18 @@ set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED True) set(EXTENSION_NAME ${TARGET_NAME}_extension) -include_directories(src/include) +include_directories( + src/include + arrow/cpp/src + arrow/cpp/build/src +) set(EXTENSION_SOURCES src/iceberg_extension.cpp src/iceberg_functions.cpp src/catalog_api.cpp src/catalog_utils.cpp + src/ic_create_table_as_op.cpp src/common/utils.cpp src/common/schema.cpp src/common/iceberg.cpp @@ -30,7 +35,7 @@ set(EXTENSION_SOURCES src/storage/ic_table_set.cpp src/storage/ic_transaction.cpp src/storage/ic_transaction_manager.cpp - ) +) add_library(${EXTENSION_NAME} STATIC ${EXTENSION_SOURCES}) @@ -87,11 +92,18 @@ target_link_libraries( find_package(CURL REQUIRED) +find_library(ARROW_LIB arrow PATHS arrow/cpp/build/release) +find_library(PARQUET_LIB parquet PATHS arrow/cpp/build/release) +find_library(FTICEBERGHELPER_LIB jiceberg PATHS jiceberg_lib/app/build/native/nativeCompile) + # Link dependencies into extension -target_link_libraries(${EXTENSION_NAME} PUBLIC optimized avro_static_release CURL::libcurl - debug avro_static_debug) +target_link_libraries(${EXTENSION_NAME} PUBLIC optimized avro_static_release + debug avro_static_debug + ${FTICEBERGHELPER_LIB} CURL::libcurl + ${ARROW_LIB} ${PARQUET_LIB}) target_link_libraries(${TARGET_NAME}_loadable_extension optimized CURL::libcurl - avro_static_release debug avro_static_debug) + avro_static_release ${FTICEBERGHELPER_LIB} ${ARROW_LIB} ${PARQUET_LIB} + debug avro_static_debug) install( TARGETS ${EXTENSION_NAME} ${TARGET_NAME}_loadable_extension diff --git a/README-ADDENDUM.md b/README-ADDENDUM.md new file mode 100644 index 0000000..bbc2554 --- /dev/null +++ b/README-ADDENDUM.md @@ -0,0 +1,56 @@ +# ADDENDUM + +This fork adds proof-of-concept functionality to DuckDB iceberg extension to be able to connect to an iceberg catalog as well as read and write iceberg tables. + +You can try it out using DuckDB (>= v1.1.3) by doing the following: + +1. Start duckdb in `unsigned` mode +```bash +duckdb --unsigned +``` + +2. Create a secret to provide access to your iceberg catalog +```sql +INSTALL '/path/to/this/iceberg.duckdb_extension'; +INSTALL httpfs; +LOAD '/path/to/this/iceberg.duckdb_extension'; +LOAD httpfs; +CREATE SECRET ( + TYPE ICEBERG, + CLIENT_ID '${CLIENT_ID}', + CLIENT_SECRET '${CLIENT_SECRET}', + ENDPOINT '${ENDPOINT}', + AWS_REGION '${AWS_REGION}' +) +``` + +3. Attach your iceberg catalog +```sql +ATTACH 'my_catalog' AS my_catalog (TYPE ICEBERG) +``` + +4. Read an iceberg table +```sql +SELECT * FROM my_catalog.my_schema.table_1; +``` + +5. Create a new iceberg table +```sql +CREATE TABLE my_catalog.my_schema.new_table (id BIGINT, name VARCHAR); +``` +```sql +CREATE TABLE my_catalog.my_schema.new_table_2 AS (SELECT FROM version()); +``` + +6. Delete an existing iceberg table +```sql +DROP TABLE my_catalog.my_schema.table_1; +``` + +# How to build extension from source +``` +git clone https://github.com/fivetran/duckdb-iceberg.git +git submodule update --init --recursive +brew install ninja +GEN=ninja make {debug/release} +``` diff --git a/jiceberg_lib/.gitattributes b/jiceberg_lib/.gitattributes new file mode 100644 index 0000000..f91f646 --- /dev/null +++ b/jiceberg_lib/.gitattributes @@ -0,0 +1,12 @@ +# +# https://help.github.com/articles/dealing-with-line-endings/ +# +# Linux start script should use lf +/gradlew text eol=lf + +# These are Windows script files and should use crlf +*.bat text eol=crlf + +# Binary files should be left untouched +*.jar binary + diff --git a/jiceberg_lib/.gitignore b/jiceberg_lib/.gitignore new file mode 100644 index 0000000..45c993a --- /dev/null +++ b/jiceberg_lib/.gitignore @@ -0,0 +1,6 @@ +# Ignore Gradle project-specific cache directory +.gradle/ +.idea/ + +# Ignore Gradle build output directory +build/ diff --git a/jiceberg_lib/README.md b/jiceberg_lib/README.md new file mode 100644 index 0000000..19ecfe7 --- /dev/null +++ b/jiceberg_lib/README.md @@ -0,0 +1,19 @@ +Apache Iceberg does not have a library written in C++. This project uses the Java library +to create helper methods to be used in C++. + +# Instructions +* Install [SDKMAN](https://sdkman.io/install/) +* Install GraalVM CE 21 +```build +sdk install java 21.0.2-graalce +``` +* Build library +``` +gradle nativeBuild +``` +* Copy header files +``` +./scripts/copy_headers.sh +``` +* In `src/include/jiceberg_generated`, replace includes with "quotes" in the header files starting with `libjiceberg...` + diff --git a/jiceberg_lib/app/build.gradle b/jiceberg_lib/app/build.gradle new file mode 100644 index 0000000..889d8ea --- /dev/null +++ b/jiceberg_lib/app/build.gradle @@ -0,0 +1,75 @@ +/* + * This file was generated by the Gradle 'init' task. + * + * This generated file contains a sample Java application project to get you started. + * For more details on building Java & JVM projects, please refer to https://docs.gradle.org/8.9/userguide/building_java_projects.html in the Gradle documentation. + */ + +plugins { + id 'java' + id 'application' + id 'org.graalvm.buildtools.native' version '0.9.28' +} + +repositories { + mavenCentral() +} + +dependencies { + implementation libs.guava + + implementation 'org.apache.iceberg:iceberg-core:1.7.1' + implementation 'org.apache.iceberg:iceberg-api:1.7.1' + implementation 'org.apache.iceberg:iceberg-aws:1.7.1' + implementation 'org.apache.iceberg:iceberg-parquet:1.7.1' + implementation 'org.apache.iceberg:iceberg-data:1.7.1' + implementation 'org.apache.parquet:parquet-avro:1.12.3' + implementation 'org.apache.hadoop:hadoop-common:3.3.1' + + implementation 'software.amazon.awssdk:s3:2.30.11' + implementation 'software.amazon.awssdk:sts:2.30.11' + implementation 'software.amazon.awssdk:auth:2.30.11' + implementation 'software.amazon.awssdk:regions:2.30.11' + + compileOnly 'org.graalvm.sdk:graal-sdk:22.0.0' +} + +application { + mainClass = 'com.fivetran.iceberg.App' // Update with your package and class name +} + +graalvmNative { + binaries { + main { + imageName = 'libjiceberg' // Name of the native executable + mainClass = 'com.fivetran.iceberg.App' // Update with your package and class name + fallback = false + buildArgs.add('--shared') + buildArgs.add('--initialize-at-run-time=org.apache.log4j') + buildArgs.add('--initialize-at-run-time=org.slf4j') + + // Add these for Iceberg + buildArgs.add('--initialize-at-run-time=com.github.benmanes.caffeine.cache.LocalLoadingCache') + buildArgs.add('--initialize-at-run-time=org.apache.iceberg.util.Pair') + buildArgs.add('-H:+ReportExceptionStackTraces') + + // If you need SSL support + buildArgs.add('--enable-https') + + metadataRepository { + enabled = true + } + configurationFileDirectories.from(file('src/main/resources/META-INF/native-image-config')) + } + } +} + +def void main(Closure booleanClosure) {} + +tasks.register('runWithAgent', JavaExec) { + classpath = sourceSets.main.runtimeClasspath + mainClass = 'com.fivetran.iceberg.App' + jvmArgs = [ + '-agentlib:native-image-agent=config-output-dir=src/main/resources/META-INF/native-image-config' + ] +} \ No newline at end of file diff --git a/jiceberg_lib/app/src/main/java/com/fivetran/iceberg/App.java b/jiceberg_lib/app/src/main/java/com/fivetran/iceberg/App.java new file mode 100644 index 0000000..5405e92 --- /dev/null +++ b/jiceberg_lib/app/src/main/java/com/fivetran/iceberg/App.java @@ -0,0 +1,161 @@ +package com.fivetran.iceberg; + +import org.apache.iceberg.*; +import org.apache.iceberg.io.*; +import org.apache.iceberg.rest.RESTCatalog; +import org.apache.iceberg.catalog.TableIdentifier; + +import org.graalvm.nativeimage.IsolateThread; +import org.graalvm.nativeimage.c.function.CEntryPoint; +import org.graalvm.nativeimage.c.type.CCharPointer; +import org.graalvm.nativeimage.c.type.CTypeConversion; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; + + +public class App { + @CEntryPoint(name = "append_to_table") + public static int appendToTable( + IsolateThread thread, + CCharPointer uri, + CCharPointer creds, + CCharPointer warehouse, + CCharPointer schemaName, + CCharPointer tableName, + CCharPointer filename, + int numRecords) { + try { + return append( + CTypeConversion.toJavaString(uri), + CTypeConversion.toJavaString(creds), + CTypeConversion.toJavaString(warehouse), + CTypeConversion.toJavaString(schemaName), + CTypeConversion.toJavaString(tableName), + CTypeConversion.toJavaString(filename), + numRecords); + } catch (IOException e) { + return -1; + } finally { + // Delete the local file + Path localPath = Paths.get(CTypeConversion.toJavaString(filename)); + try { + java.nio.file.Files.deleteIfExists(localPath); + } catch (IOException e) { + System.out.println("Unable to delete datafile"); + } + } + } + + public static void main(String[] args) { + try { + append("https://polaris.fivetran.com/api/catalog", + "client_id:client_secret", + "catalog", + "schema", + "new_table", + "datafile.parquet", + 1 + ); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + static int append( + String uri, + String creds, + String warehouse, + String schemaName, + String tableName, + String filename, + int numRecords) throws IOException { + Map properties = new HashMap<>(); + properties.put(CatalogProperties.URI, uri); + properties.put("credential", creds); + properties.put("oauth2-server-uri", uri + "/v1/oauth/tokens"); + properties.put("scope", "PRINCIPAL_ROLE:ALL"); + properties.put(CatalogProperties.WAREHOUSE_LOCATION, warehouse); + properties.put(CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.aws.s3.S3FileIO"); + + try(RESTCatalog catalog = new RESTCatalog()) { + catalog.initialize("catalog", properties); + TableIdentifier tableId = TableIdentifier.of(schemaName, tableName); + Table table = catalog.loadTable(tableId); + + // Copy the local file to the S3-backed OutputFile + Path localPath = Paths.get(filename); + try(FileIO fileIO = table.io()) { + OutputFile outputFile = fileIO.newOutputFile( + table.location() + "/data/" + filename); + try (PositionOutputStream out = outputFile.createOrOverwrite()) { + // Use Java's Files.copy to transfer the contents + java.nio.file.Files.copy(localPath, out); + } + + PartitionSpec spec = table.spec(); + DataFile dataFile = DataFiles.builder(spec) + .withRecordCount(numRecords) + .withFileSizeInBytes(outputFile.toInputFile().getLength()) + .withInputFile(outputFile.toInputFile()) + .withFormat(FileFormat.PARQUET) + .build(); + + // Append the data file + table.newAppend() + .appendFile(dataFile) + .commit(); + } + +// Snapshot snapshot = table.currentSnapshot(); +// FileIO io = table.io(); +// // For newer Iceberg versions, use `allManifests(io)`: +// for (ManifestFile mf : snapshot.allManifests(io)) { +// System.out.println("Manifest path: " + mf.path() +// + ", addedFilesCount=" + mf.addedFilesCount()); +// } +// +// Iterable newlyAddedFiles = snapshot.addedDataFiles(table.io()); +// for (DataFile df : newlyAddedFiles) { +// System.out.println("Added file path: " + df.path()); +// } +// +// table.newScan() +// .planFiles() +// .forEach(task -> { +// System.out.println("Active data file path: " + task.file().path()); +// System.out.println("Row count: " + task.file().recordCount()); +// }); + +// Schema schema = table.schema(); +// // Iterate over each file in the table scan +// for (FileScanTask task : table.newScan().planFiles()) { +// // Print file information +// DataFile dataFile = task.file(); +// +// // Convert the DataFile to an Iceberg InputFile +// InputFile inputFile = table.io().newInputFile(dataFile.path().toString()); +// +// // Build a Parquet reader using Iceberg's Parquet API +// try (CloseableIterable reader = Parquet.read(inputFile) +// .project(schema) // Project the full schema or a subset +// .createReaderFunc(messageType -> GenericParquetReaders.buildReader(schema, messageType)) +// .build()) { +// +// // Iterate over the records and process them +// for (Record record : reader) { +// System.out.println("Record: " + record); +// } +// } catch (IOException e) { +// System.err.println("Failed to read data from file: " + dataFile.path()); +// e.printStackTrace(); +// } +// } + + return 0; + } + } +} diff --git a/jiceberg_lib/app/src/main/resources/META-INF/native-image-config/jni-config.json b/jiceberg_lib/app/src/main/resources/META-INF/native-image-config/jni-config.json new file mode 100644 index 0000000..75f6460 --- /dev/null +++ b/jiceberg_lib/app/src/main/resources/META-INF/native-image-config/jni-config.json @@ -0,0 +1,18 @@ +[ +{ + "name":"com.fivetran.iceberg.App", + "methods":[{"name":"main","parameterTypes":["java.lang.String[]"] }] +}, +{ + "name":"java.lang.Boolean", + "methods":[{"name":"getBoolean","parameterTypes":["java.lang.String"] }] +}, +{ + "name":"java.lang.String", + "methods":[{"name":"lastIndexOf","parameterTypes":["int"] }, {"name":"substring","parameterTypes":["int"] }] +}, +{ + "name":"java.lang.System", + "methods":[{"name":"getProperty","parameterTypes":["java.lang.String"] }, {"name":"setProperty","parameterTypes":["java.lang.String","java.lang.String"] }] +} +] diff --git a/jiceberg_lib/app/src/main/resources/META-INF/native-image-config/predefined-classes-config.json b/jiceberg_lib/app/src/main/resources/META-INF/native-image-config/predefined-classes-config.json new file mode 100644 index 0000000..0e79b2c --- /dev/null +++ b/jiceberg_lib/app/src/main/resources/META-INF/native-image-config/predefined-classes-config.json @@ -0,0 +1,8 @@ +[ + { + "type":"agent-extracted", + "classes":[ + ] + } +] + diff --git a/jiceberg_lib/app/src/main/resources/META-INF/native-image-config/proxy-config.json b/jiceberg_lib/app/src/main/resources/META-INF/native-image-config/proxy-config.json new file mode 100644 index 0000000..0d4f101 --- /dev/null +++ b/jiceberg_lib/app/src/main/resources/META-INF/native-image-config/proxy-config.json @@ -0,0 +1,2 @@ +[ +] diff --git a/jiceberg_lib/app/src/main/resources/META-INF/native-image-config/reflect-config.json b/jiceberg_lib/app/src/main/resources/META-INF/native-image-config/reflect-config.json new file mode 100644 index 0000000..be86c15 --- /dev/null +++ b/jiceberg_lib/app/src/main/resources/META-INF/native-image-config/reflect-config.json @@ -0,0 +1,436 @@ +[ +{ + "name":"[B" +}, +{ + "name":"[Lcom.fasterxml.jackson.databind.deser.Deserializers;" +}, +{ + "name":"[Lcom.fasterxml.jackson.databind.ser.Serializers;" +}, +{ + "name":"[Ljava.lang.String;" +}, +{ + "name":"[Ljava.util.List;" +}, +{ + "name":"[Lorg.apache.avro.util.springframework.ConcurrentReferenceHashMap$Segment;" +}, +{ + "name":"[Lorg.apache.iceberg.avro.ValueWriter;" +}, +{ + "name":"[Lsun.security.pkcs.SignerInfo;" +}, +{ + "name":"apple.security.AppleProvider", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"com.fasterxml.jackson.databind.ext.Java7SupportImpl", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"com.github.benmanes.caffeine.cache.BLCHeader$DrainStatusRef", + "fields":[{"name":"drainStatus"}] +}, +{ + "name":"com.github.benmanes.caffeine.cache.BaseMpscLinkedArrayQueueColdProducerFields", + "fields":[{"name":"producerLimit"}] +}, +{ + "name":"com.github.benmanes.caffeine.cache.BaseMpscLinkedArrayQueueConsumerFields", + "fields":[{"name":"consumerIndex"}] +}, +{ + "name":"com.github.benmanes.caffeine.cache.BaseMpscLinkedArrayQueueProducerFields", + "fields":[{"name":"producerIndex"}] +}, +{ + "name":"com.github.benmanes.caffeine.cache.CacheLoader", + "methods":[{"name":"loadAll","parameterTypes":["java.lang.Iterable"] }] +}, +{ + "name":"com.github.benmanes.caffeine.cache.FD", + "fields":[{"name":"value"}] +}, +{ + "name":"com.github.benmanes.caffeine.cache.FDMS", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"com.github.benmanes.caffeine.cache.FS", + "fields":[{"name":"key"}, {"name":"value"}], + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"com.github.benmanes.caffeine.cache.PS", + "fields":[{"name":"key"}, {"name":"value"}] +}, +{ + "name":"com.github.benmanes.caffeine.cache.PSA", + "fields":[{"name":"accessTime"}], + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"com.github.benmanes.caffeine.cache.PW", + "fields":[{"name":"value"}], + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"com.github.benmanes.caffeine.cache.SI", + "methods":[{"name":"","parameterTypes":["com.github.benmanes.caffeine.cache.Caffeine","com.github.benmanes.caffeine.cache.CacheLoader","boolean"] }] +}, +{ + "name":"com.github.benmanes.caffeine.cache.SSA", + "methods":[{"name":"","parameterTypes":["com.github.benmanes.caffeine.cache.Caffeine","com.github.benmanes.caffeine.cache.CacheLoader","boolean"] }] +}, +{ + "name":"com.github.benmanes.caffeine.cache.SSLA", + "methods":[{"name":"","parameterTypes":["com.github.benmanes.caffeine.cache.Caffeine","com.github.benmanes.caffeine.cache.CacheLoader","boolean"] }] +}, +{ + "name":"com.github.benmanes.caffeine.cache.StripedBuffer", + "fields":[{"name":"tableBusy"}] +}, +{ + "name":"com.github.benmanes.caffeine.cache.WILSMS", + "methods":[{"name":"","parameterTypes":["com.github.benmanes.caffeine.cache.Caffeine","com.github.benmanes.caffeine.cache.CacheLoader","boolean"] }] +}, +{ + "name":"com.github.benmanes.caffeine.cache.WSL", + "methods":[{"name":"","parameterTypes":["com.github.benmanes.caffeine.cache.Caffeine","com.github.benmanes.caffeine.cache.CacheLoader","boolean"] }] +}, +{ + "name":"com.sun.crypto.provider.AESCipher$General", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"com.sun.crypto.provider.ARCFOURCipher", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"com.sun.crypto.provider.ChaCha20Cipher$ChaCha20Poly1305", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"com.sun.crypto.provider.DESCipher", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"com.sun.crypto.provider.DESedeCipher", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"com.sun.crypto.provider.DHParameters", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"com.sun.crypto.provider.GaloisCounterMode$AESGCM", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"com.sun.crypto.provider.HmacCore$HmacSHA256", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"com.sun.crypto.provider.HmacCore$HmacSHA384", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"com.sun.crypto.provider.TlsMasterSecretGenerator", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"java.lang.String" +}, +{ + "name":"java.lang.Thread", + "fields":[{"name":"threadLocalRandomProbe"}], + "methods":[{"name":"getContextClassLoader","parameterTypes":[] }] +}, +{ + "name":"java.security.AlgorithmParametersSpi" +}, +{ + "name":"java.security.KeyStoreSpi" +}, +{ + "name":"java.security.SecureRandomParameters" +}, +{ + "name":"java.security.interfaces.ECPrivateKey" +}, +{ + "name":"java.security.interfaces.ECPublicKey" +}, +{ + "name":"java.security.interfaces.RSAPrivateKey" +}, +{ + "name":"java.security.interfaces.RSAPublicKey" +}, +{ + "name":"java.util.Date" +}, +{ + "name":"java.util.concurrent.ForkJoinTask", + "fields":[{"name":"aux"}, {"name":"status"}] +}, +{ + "name":"java.util.concurrent.atomic.AtomicBoolean", + "fields":[{"name":"value"}] +}, +{ + "name":"java.util.concurrent.atomic.AtomicMarkableReference", + "fields":[{"name":"pair"}] +}, +{ + "name":"java.util.concurrent.atomic.AtomicReference", + "fields":[{"name":"value"}] +}, +{ + "name":"java.util.concurrent.atomic.Striped64", + "fields":[{"name":"base"}, {"name":"cellsBusy"}] +}, +{ + "name":"javax.security.auth.x500.X500Principal", + "fields":[{"name":"thisX500Name"}], + "methods":[{"name":"","parameterTypes":["sun.security.x509.X500Name"] }] +}, +{ + "name":"jdk.internal.misc.Unsafe" +}, +{ + "name":"kotlin.Unit" +}, +{ + "name":"org.apache.commons.logging.LogFactory" +}, +{ + "name":"org.apache.commons.logging.impl.Log4JLogger", + "methods":[{"name":"","parameterTypes":["java.lang.String"] }, {"name":"setLogFactory","parameterTypes":["org.apache.commons.logging.LogFactory"] }] +}, +{ + "name":"org.apache.commons.logging.impl.LogFactoryImpl", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"org.apache.commons.logging.impl.WeakHashtable", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"org.apache.hadoop.fs.FSDataInputStream" +}, +{ + "name":"org.apache.http.client.config.RequestConfig$Builder", + "methods":[{"name":"setNormalizeUri","parameterTypes":["boolean"] }] +}, +{ + "name":"org.apache.iceberg.GenericManifestFile", + "methods":[{"name":"","parameterTypes":["org.apache.avro.Schema"] }] +}, +{ + "name":"org.apache.iceberg.GenericPartitionFieldSummary", + "methods":[{"name":"","parameterTypes":["org.apache.avro.Schema"] }] +}, +{ + "name":"org.apache.iceberg.aws.ApacheHttpClientConfigurations", + "methods":[{"name":"create","parameterTypes":["java.util.Map"] }] +}, +{ + "name":"org.apache.iceberg.aws.AwsClientFactories$DefaultAwsClientFactory", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"org.apache.iceberg.aws.s3.S3FileIO", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"org.apache.iceberg.hadoop.HadoopMetricsContext", + "methods":[{"name":"","parameterTypes":["java.lang.String"] }] +}, +{ + "name":"org.apache.iceberg.util.Pair$1" +}, +{ + "name":"org.apache.log4j.Category" +}, +{ + "name":"org.apache.log4j.CategoryKey" +}, +{ + "name":"org.apache.log4j.Level", + "fields":[{"name":"TRACE"}] +}, +{ + "name":"org.apache.log4j.Logger" +}, +{ + "name":"org.apache.log4j.Priority" +}, +{ + "name":"org.apache.log4j.helpers.Loader" +}, +{ + "name":"org.brotli.dec.BrotliInputStream" +}, +{ + "name":"scala.util.Properties" +}, +{ + "name":"sun.misc.Unsafe", + "fields":[{"name":"theUnsafe"}] +}, +{ + "name":"sun.security.pkcs12.PKCS12KeyStore", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.security.pkcs12.PKCS12KeyStore$DualFormatPKCS12", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.security.provider.DSA$SHA224withDSA", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.security.provider.DSA$SHA256withDSA", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.security.provider.JavaKeyStore$DualFormatJKS", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.security.provider.JavaKeyStore$JKS", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.security.provider.MD5", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.security.provider.NativePRNG", + "methods":[{"name":"","parameterTypes":[] }, {"name":"","parameterTypes":["java.security.SecureRandomParameters"] }] +}, +{ + "name":"sun.security.provider.SHA", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.security.provider.SHA2$SHA224", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.security.provider.SHA2$SHA256", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.security.provider.SHA5$SHA384", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.security.provider.SHA5$SHA512", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.security.provider.X509Factory", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.security.provider.certpath.PKIXCertPathValidator", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.security.rsa.PSSParameters", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.security.rsa.RSAKeyFactory$Legacy", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.security.rsa.RSAPSSSignature", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.security.rsa.RSASignature$SHA224withRSA", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.security.rsa.RSASignature$SHA256withRSA", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.security.ssl.KeyManagerFactoryImpl$SunX509", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.security.ssl.SSLContextImpl$DefaultSSLContext", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.security.ssl.SSLContextImpl$TLSContext", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.security.ssl.TrustManagerFactoryImpl$PKIXFactory", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.security.util.ObjectIdentifier" +}, +{ + "name":"sun.security.x509.AuthorityInfoAccessExtension", + "methods":[{"name":"","parameterTypes":["java.lang.Boolean","java.lang.Object"] }] +}, +{ + "name":"sun.security.x509.AuthorityKeyIdentifierExtension", + "methods":[{"name":"","parameterTypes":["java.lang.Boolean","java.lang.Object"] }] +}, +{ + "name":"sun.security.x509.BasicConstraintsExtension", + "methods":[{"name":"","parameterTypes":["java.lang.Boolean","java.lang.Object"] }] +}, +{ + "name":"sun.security.x509.CRLDistributionPointsExtension", + "methods":[{"name":"","parameterTypes":["java.lang.Boolean","java.lang.Object"] }] +}, +{ + "name":"sun.security.x509.CertificateExtensions" +}, +{ + "name":"sun.security.x509.CertificatePoliciesExtension", + "methods":[{"name":"","parameterTypes":["java.lang.Boolean","java.lang.Object"] }] +}, +{ + "name":"sun.security.x509.ExtendedKeyUsageExtension", + "methods":[{"name":"","parameterTypes":["java.lang.Boolean","java.lang.Object"] }] +}, +{ + "name":"sun.security.x509.IssuerAlternativeNameExtension", + "methods":[{"name":"","parameterTypes":["java.lang.Boolean","java.lang.Object"] }] +}, +{ + "name":"sun.security.x509.KeyUsageExtension", + "methods":[{"name":"","parameterTypes":["java.lang.Boolean","java.lang.Object"] }] +}, +{ + "name":"sun.security.x509.NetscapeCertTypeExtension", + "methods":[{"name":"","parameterTypes":["java.lang.Boolean","java.lang.Object"] }] +}, +{ + "name":"sun.security.x509.PrivateKeyUsageExtension", + "methods":[{"name":"","parameterTypes":["java.lang.Boolean","java.lang.Object"] }] +}, +{ + "name":"sun.security.x509.SubjectAlternativeNameExtension", + "methods":[{"name":"","parameterTypes":["java.lang.Boolean","java.lang.Object"] }] +}, +{ + "name":"sun.security.x509.SubjectKeyIdentifierExtension", + "methods":[{"name":"","parameterTypes":["java.lang.Boolean","java.lang.Object"] }] +} +] diff --git a/jiceberg_lib/app/src/main/resources/META-INF/native-image-config/resource-config.json b/jiceberg_lib/app/src/main/resources/META-INF/native-image-config/resource-config.json new file mode 100644 index 0000000..e3f4ed2 --- /dev/null +++ b/jiceberg_lib/app/src/main/resources/META-INF/native-image-config/resource-config.json @@ -0,0 +1,63 @@ +{ + "resources":{ + "includes":[{ + "pattern":"\\QMETA-INF/maven/org.xerial.snappy/snappy-java/pom.properties\\E" + }, { + "pattern":"\\QMETA-INF/services/java.lang.System$LoggerFinder\\E" + }, { + "pattern":"\\QMETA-INF/services/java.net.spi.InetAddressResolverProvider\\E" + }, { + "pattern":"\\QMETA-INF/services/java.net.spi.URLStreamHandlerProvider\\E" + }, { + "pattern":"\\QMETA-INF/services/org.apache.avro.Conversion\\E" + }, { + "pattern":"\\QMETA-INF/services/org.apache.avro.LogicalTypes$LogicalTypeFactory\\E" + }, { + "pattern":"\\QMETA-INF/services/org.apache.commons.logging.LogFactory\\E" + }, { + "pattern":"\\QMETA-INF/services/org.slf4j.spi.SLF4JServiceProvider\\E" + }, { + "pattern":"\\Qcommons-logging.properties\\E" + }, { + "pattern":"\\Qcore-default.xml\\E" + }, { + "pattern":"\\Qcore-site.xml\\E" + }, { + "pattern":"\\Qhadoop-site.xml\\E" + }, { + "pattern":"\\Qiceberg-build.properties\\E" + }, { + "pattern":"\\Qlog4j.properties\\E" + }, { + "pattern":"\\Qlog4j.xml\\E" + }, { + "pattern":"\\Qmozilla/public-suffix-list.txt\\E" + }, { + "pattern":"\\Qorg-xerial-snappy.properties\\E" + }, { + "pattern":"\\Qorg/apache/hc/client5/http/psl/org/publicsuffix/list/effective_tld_names.dat\\E" + }, { + "pattern":"\\Qorg/apache/hc/client5/version.properties\\E" + }, { + "pattern":"\\Qorg/slf4j/impl/StaticLoggerBinder.class\\E" + }, { + "pattern":"\\Qorg/xerial/snappy/VERSION\\E" + }, { + "pattern":"\\Qorg/xerial/snappy/native/Mac/aarch64/libsnappyjava.dylib\\E" + }, { + "pattern":"\\Qsoftware/amazon/awssdk/global/handlers/execution.interceptors\\E" + }, { + "pattern":"\\Qsoftware/amazon/awssdk/global/partitions.json\\E" + }, { + "pattern":"\\Qsoftware/amazon/awssdk/services/s3/execution.interceptors\\E" + }, { + "pattern":"java.base:\\Qjdk/internal/icu/impl/data/icudt72b/nfc.nrm\\E" + }, { + "pattern":"java.base:\\Qjdk/internal/icu/impl/data/icudt72b/nfkc.nrm\\E" + }, { + "pattern":"java.base:\\Qjdk/internal/icu/impl/data/icudt72b/uprops.icu\\E" + }, { + "pattern":"java.base:\\Qsun/net/idn/uidna.spp\\E" + }]}, + "bundles":[] +} diff --git a/jiceberg_lib/app/src/main/resources/META-INF/native-image-config/serialization-config.json b/jiceberg_lib/app/src/main/resources/META-INF/native-image-config/serialization-config.json new file mode 100644 index 0000000..f3d7e06 --- /dev/null +++ b/jiceberg_lib/app/src/main/resources/META-INF/native-image-config/serialization-config.json @@ -0,0 +1,8 @@ +{ + "types":[ + ], + "lambdaCapturingTypes":[ + ], + "proxies":[ + ] +} diff --git a/jiceberg_lib/gradle/libs.versions.toml b/jiceberg_lib/gradle/libs.versions.toml new file mode 100644 index 0000000..334ec20 --- /dev/null +++ b/jiceberg_lib/gradle/libs.versions.toml @@ -0,0 +1,8 @@ +# This file was generated by the Gradle 'init' task. +# https://docs.gradle.org/current/userguide/platforms.html#sub::toml-dependencies-format + +[versions] +guava = "33.1.0-jre" + +[libraries] +guava = { module = "com.google.guava:guava", version.ref = "guava" } diff --git a/jiceberg_lib/gradle/wrapper/gradle-wrapper.jar b/jiceberg_lib/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..2c3521197d7c4586c843d1d3e9090525f1898cde GIT binary patch literal 43504 zcma&N1CXTcmMvW9vTb(Rwr$&4wr$(C?dmSu>@vG-+vuvg^_??!{yS%8zW-#zn-LkA z5&1^$^{lnmUON?}LBF8_K|(?T0Ra(xUH{($5eN!MR#ZihR#HxkUPe+_R8Cn`RRs(P z_^*#_XlXmGv7!4;*Y%p4nw?{bNp@UZHv1?Um8r6)Fei3p@ClJn0ECfg1hkeuUU@Or zDaPa;U3fE=3L}DooL;8f;P0ipPt0Z~9P0)lbStMS)ag54=uL9ia-Lm3nh|@(Y?B`; zx_#arJIpXH!U{fbCbI^17}6Ri*H<>OLR%c|^mh8+)*h~K8Z!9)DPf zR2h?lbDZQ`p9P;&DQ4F0sur@TMa!Y}S8irn(%d-gi0*WxxCSk*A?3lGh=gcYN?FGl z7D=Js!i~0=u3rox^eO3i@$0=n{K1lPNU zwmfjRVmLOCRfe=seV&P*1Iq=^i`502keY8Uy-WNPwVNNtJFx?IwAyRPZo2Wo1+S(xF37LJZ~%i)kpFQ3Fw=mXfd@>%+)RpYQLnr}B~~zoof(JVm^^&f zxKV^+3D3$A1G;qh4gPVjhrC8e(VYUHv#dy^)(RoUFM?o%W-EHxufuWf(l*@-l+7vt z=l`qmR56K~F|v<^Pd*p~1_y^P0P^aPC##d8+HqX4IR1gu+7w#~TBFphJxF)T$2WEa zxa?H&6=Qe7d(#tha?_1uQys2KtHQ{)Qco)qwGjrdNL7thd^G5i8Os)CHqc>iOidS} z%nFEDdm=GXBw=yXe1W-ShHHFb?Cc70+$W~z_+}nAoHFYI1MV1wZegw*0y^tC*s%3h zhD3tN8b=Gv&rj}!SUM6|ajSPp*58KR7MPpI{oAJCtY~JECm)*m_x>AZEu>DFgUcby z1Qaw8lU4jZpQ_$;*7RME+gq1KySGG#Wql>aL~k9tLrSO()LWn*q&YxHEuzmwd1?aAtI zBJ>P=&$=l1efe1CDU;`Fd+_;&wI07?V0aAIgc(!{a z0Jg6Y=inXc3^n!U0Atk`iCFIQooHqcWhO(qrieUOW8X(x?(RD}iYDLMjSwffH2~tB z)oDgNBLB^AJBM1M^c5HdRx6fBfka`(LD-qrlh5jqH~);#nw|iyp)()xVYak3;Ybik z0j`(+69aK*B>)e_p%=wu8XC&9e{AO4c~O1U`5X9}?0mrd*m$_EUek{R?DNSh(=br# z#Q61gBzEpmy`$pA*6!87 zSDD+=@fTY7<4A?GLqpA?Pb2z$pbCc4B4zL{BeZ?F-8`s$?>*lXXtn*NC61>|*w7J* z$?!iB{6R-0=KFmyp1nnEmLsA-H0a6l+1uaH^g%c(p{iT&YFrbQ$&PRb8Up#X3@Zsk zD^^&LK~111%cqlP%!_gFNa^dTYT?rhkGl}5=fL{a`UViaXWI$k-UcHJwmaH1s=S$4 z%4)PdWJX;hh5UoK?6aWoyLxX&NhNRqKam7tcOkLh{%j3K^4Mgx1@i|Pi&}<^5>hs5 zm8?uOS>%)NzT(%PjVPGa?X%`N2TQCKbeH2l;cTnHiHppPSJ<7y-yEIiC!P*ikl&!B z%+?>VttCOQM@ShFguHVjxX^?mHX^hSaO_;pnyh^v9EumqSZTi+#f&_Vaija0Q-e*| z7ulQj6Fs*bbmsWp{`auM04gGwsYYdNNZcg|ph0OgD>7O}Asn7^Z=eI>`$2*v78;sj-}oMoEj&@)9+ycEOo92xSyY344^ z11Hb8^kdOvbf^GNAK++bYioknrpdN>+u8R?JxG=!2Kd9r=YWCOJYXYuM0cOq^FhEd zBg2puKy__7VT3-r*dG4c62Wgxi52EMCQ`bKgf*#*ou(D4-ZN$+mg&7$u!! z-^+Z%;-3IDwqZ|K=ah85OLwkO zKxNBh+4QHh)u9D?MFtpbl)us}9+V!D%w9jfAMYEb>%$A;u)rrI zuBudh;5PN}_6J_}l55P3l_)&RMlH{m!)ai-i$g)&*M`eN$XQMw{v^r@-125^RRCF0 z^2>|DxhQw(mtNEI2Kj(;KblC7x=JlK$@78`O~>V!`|1Lm-^JR$-5pUANAnb(5}B}JGjBsliK4& zk6y(;$e&h)lh2)L=bvZKbvh@>vLlreBdH8No2>$#%_Wp1U0N7Ank!6$dFSi#xzh|( zRi{Uw%-4W!{IXZ)fWx@XX6;&(m_F%c6~X8hx=BN1&q}*( zoaNjWabE{oUPb!Bt$eyd#$5j9rItB-h*5JiNi(v^e|XKAj*8(k<5-2$&ZBR5fF|JA z9&m4fbzNQnAU}r8ab>fFV%J0z5awe#UZ|bz?Ur)U9bCIKWEzi2%A+5CLqh?}K4JHi z4vtM;+uPsVz{Lfr;78W78gC;z*yTch~4YkLr&m-7%-xc ztw6Mh2d>_iO*$Rd8(-Cr1_V8EO1f*^@wRoSozS) zy1UoC@pruAaC8Z_7~_w4Q6n*&B0AjOmMWa;sIav&gu z|J5&|{=a@vR!~k-OjKEgPFCzcJ>#A1uL&7xTDn;{XBdeM}V=l3B8fE1--DHjSaxoSjNKEM9|U9#m2<3>n{Iuo`r3UZp;>GkT2YBNAh|b z^jTq-hJp(ebZh#Lk8hVBP%qXwv-@vbvoREX$TqRGTgEi$%_F9tZES@z8Bx}$#5eeG zk^UsLBH{bc2VBW)*EdS({yw=?qmevwi?BL6*=12k9zM5gJv1>y#ML4!)iiPzVaH9% zgSImetD@dam~e>{LvVh!phhzpW+iFvWpGT#CVE5TQ40n%F|p(sP5mXxna+Ev7PDwA zamaV4m*^~*xV+&p;W749xhb_X=$|LD;FHuB&JL5?*Y2-oIT(wYY2;73<^#46S~Gx| z^cez%V7x$81}UWqS13Gz80379Rj;6~WdiXWOSsdmzY39L;Hg3MH43o*y8ibNBBH`(av4|u;YPq%{R;IuYow<+GEsf@R?=@tT@!}?#>zIIn0CoyV!hq3mw zHj>OOjfJM3F{RG#6ujzo?y32m^tgSXf@v=J$ELdJ+=5j|=F-~hP$G&}tDZsZE?5rX ztGj`!S>)CFmdkccxM9eGIcGnS2AfK#gXwj%esuIBNJQP1WV~b~+D7PJTmWGTSDrR` zEAu4B8l>NPuhsk5a`rReSya2nfV1EK01+G!x8aBdTs3Io$u5!6n6KX%uv@DxAp3F@{4UYg4SWJtQ-W~0MDb|j-$lwVn znAm*Pl!?Ps&3wO=R115RWKb*JKoexo*)uhhHBncEDMSVa_PyA>k{Zm2(wMQ(5NM3# z)jkza|GoWEQo4^s*wE(gHz?Xsg4`}HUAcs42cM1-qq_=+=!Gk^y710j=66(cSWqUe zklbm8+zB_syQv5A2rj!Vbw8;|$@C!vfNmNV!yJIWDQ>{+2x zKjuFX`~~HKG~^6h5FntRpnnHt=D&rq0>IJ9#F0eM)Y-)GpRjiN7gkA8wvnG#K=q{q z9dBn8_~wm4J<3J_vl|9H{7q6u2A!cW{bp#r*-f{gOV^e=8S{nc1DxMHFwuM$;aVI^ zz6A*}m8N-&x8;aunp1w7_vtB*pa+OYBw=TMc6QK=mbA-|Cf* zvyh8D4LRJImooUaSb7t*fVfih<97Gf@VE0|z>NcBwBQze);Rh!k3K_sfunToZY;f2 z^HmC4KjHRVg+eKYj;PRN^|E0>Gj_zagfRbrki68I^#~6-HaHg3BUW%+clM1xQEdPYt_g<2K+z!$>*$9nQ>; zf9Bei{?zY^-e{q_*|W#2rJG`2fy@{%6u0i_VEWTq$*(ZN37|8lFFFt)nCG({r!q#9 z5VK_kkSJ3?zOH)OezMT{!YkCuSSn!K#-Rhl$uUM(bq*jY? zi1xbMVthJ`E>d>(f3)~fozjg^@eheMF6<)I`oeJYx4*+M&%c9VArn(OM-wp%M<-`x z7sLP1&3^%Nld9Dhm@$3f2}87!quhI@nwd@3~fZl_3LYW-B?Ia>ui`ELg z&Qfe!7m6ze=mZ`Ia9$z|ARSw|IdMpooY4YiPN8K z4B(ts3p%2i(Td=tgEHX z0UQ_>URBtG+-?0E;E7Ld^dyZ;jjw0}XZ(}-QzC6+NN=40oDb2^v!L1g9xRvE#@IBR zO!b-2N7wVfLV;mhEaXQ9XAU+>=XVA6f&T4Z-@AX!leJ8obP^P^wP0aICND?~w&NykJ#54x3_@r7IDMdRNy4Hh;h*!u(Ol(#0bJdwEo$5437-UBjQ+j=Ic>Q2z` zJNDf0yO6@mr6y1#n3)s(W|$iE_i8r@Gd@!DWDqZ7J&~gAm1#~maIGJ1sls^gxL9LLG_NhU!pTGty!TbhzQnu)I*S^54U6Yu%ZeCg`R>Q zhBv$n5j0v%O_j{QYWG!R9W?5_b&67KB$t}&e2LdMvd(PxN6Ir!H4>PNlerpBL>Zvyy!yw z-SOo8caEpDt(}|gKPBd$qND5#a5nju^O>V&;f890?yEOfkSG^HQVmEbM3Ugzu+UtH zC(INPDdraBN?P%kE;*Ae%Wto&sgw(crfZ#Qy(<4nk;S|hD3j{IQRI6Yq|f^basLY; z-HB&Je%Gg}Jt@={_C{L$!RM;$$|iD6vu#3w?v?*;&()uB|I-XqEKqZPS!reW9JkLewLb!70T7n`i!gNtb1%vN- zySZj{8-1>6E%H&=V}LM#xmt`J3XQoaD|@XygXjdZ1+P77-=;=eYpoEQ01B@L*a(uW zrZeZz?HJsw_4g0vhUgkg@VF8<-X$B8pOqCuWAl28uB|@r`19DTUQQsb^pfqB6QtiT z*`_UZ`fT}vtUY#%sq2{rchyfu*pCg;uec2$-$N_xgjZcoumE5vSI{+s@iLWoz^Mf; zuI8kDP{!XY6OP~q5}%1&L}CtfH^N<3o4L@J@zg1-mt{9L`s^z$Vgb|mr{@WiwAqKg zp#t-lhrU>F8o0s1q_9y`gQNf~Vb!F%70f}$>i7o4ho$`uciNf=xgJ>&!gSt0g;M>*x4-`U)ysFW&Vs^Vk6m%?iuWU+o&m(2Jm26Y(3%TL; zA7T)BP{WS!&xmxNw%J=$MPfn(9*^*TV;$JwRy8Zl*yUZi8jWYF>==j~&S|Xinsb%c z2?B+kpet*muEW7@AzjBA^wAJBY8i|#C{WtO_or&Nj2{=6JTTX05}|H>N2B|Wf!*3_ z7hW*j6p3TvpghEc6-wufFiY!%-GvOx*bZrhZu+7?iSrZL5q9}igiF^*R3%DE4aCHZ zqu>xS8LkW+Auv%z-<1Xs92u23R$nk@Pk}MU5!gT|c7vGlEA%G^2th&Q*zfg%-D^=f z&J_}jskj|Q;73NP4<4k*Y%pXPU2Thoqr+5uH1yEYM|VtBPW6lXaetokD0u z9qVek6Q&wk)tFbQ8(^HGf3Wp16gKmr>G;#G(HRBx?F`9AIRboK+;OfHaLJ(P>IP0w zyTbTkx_THEOs%Q&aPrxbZrJlio+hCC_HK<4%f3ZoSAyG7Dn`=X=&h@m*|UYO-4Hq0 z-Bq&+Ie!S##4A6OGoC~>ZW`Y5J)*ouaFl_e9GA*VSL!O_@xGiBw!AF}1{tB)z(w%c zS1Hmrb9OC8>0a_$BzeiN?rkPLc9%&;1CZW*4}CDDNr2gcl_3z+WC15&H1Zc2{o~i) z)LLW=WQ{?ricmC`G1GfJ0Yp4Dy~Ba;j6ZV4r{8xRs`13{dD!xXmr^Aga|C=iSmor% z8hi|pTXH)5Yf&v~exp3o+sY4B^^b*eYkkCYl*T{*=-0HniSA_1F53eCb{x~1k3*`W zr~};p1A`k{1DV9=UPnLDgz{aJH=-LQo<5%+Em!DNN252xwIf*wF_zS^!(XSm(9eoj z=*dXG&n0>)_)N5oc6v!>-bd(2ragD8O=M|wGW z!xJQS<)u70m&6OmrF0WSsr@I%T*c#Qo#Ha4d3COcX+9}hM5!7JIGF>7<~C(Ear^Sn zm^ZFkV6~Ula6+8S?oOROOA6$C&q&dp`>oR-2Ym3(HT@O7Sd5c~+kjrmM)YmgPH*tL zX+znN>`tv;5eOfX?h{AuX^LK~V#gPCu=)Tigtq9&?7Xh$qN|%A$?V*v=&-2F$zTUv z`C#WyIrChS5|Kgm_GeudCFf;)!WH7FI60j^0o#65o6`w*S7R@)88n$1nrgU(oU0M9 zx+EuMkC>(4j1;m6NoGqEkpJYJ?vc|B zOlwT3t&UgL!pX_P*6g36`ZXQ; z9~Cv}ANFnJGp(;ZhS(@FT;3e)0)Kp;h^x;$*xZn*k0U6-&FwI=uOGaODdrsp-!K$Ac32^c{+FhI-HkYd5v=`PGsg%6I`4d9Jy)uW0y%) zm&j^9WBAp*P8#kGJUhB!L?a%h$hJgQrx!6KCB_TRo%9{t0J7KW8!o1B!NC)VGLM5! zpZy5Jc{`r{1e(jd%jsG7k%I+m#CGS*BPA65ZVW~fLYw0dA-H_}O zrkGFL&P1PG9p2(%QiEWm6x;U-U&I#;Em$nx-_I^wtgw3xUPVVu zqSuKnx&dIT-XT+T10p;yjo1Y)z(x1fb8Dzfn8e yu?e%!_ptzGB|8GrCfu%p?(_ zQccdaaVK$5bz;*rnyK{_SQYM>;aES6Qs^lj9lEs6_J+%nIiuQC*fN;z8md>r_~Mfl zU%p5Dt_YT>gQqfr@`cR!$NWr~+`CZb%dn;WtzrAOI>P_JtsB76PYe*<%H(y>qx-`Kq!X_; z<{RpAqYhE=L1r*M)gNF3B8r(<%8mo*SR2hu zccLRZwGARt)Hlo1euqTyM>^!HK*!Q2P;4UYrysje@;(<|$&%vQekbn|0Ruu_Io(w4#%p6ld2Yp7tlA`Y$cciThP zKzNGIMPXX%&Ud0uQh!uQZz|FB`4KGD?3!ND?wQt6!n*f4EmCoJUh&b?;B{|lxs#F- z31~HQ`SF4x$&v00@(P+j1pAaj5!s`)b2RDBp*PB=2IB>oBF!*6vwr7Dp%zpAx*dPr zb@Zjq^XjN?O4QcZ*O+8>)|HlrR>oD*?WQl5ri3R#2?*W6iJ>>kH%KnnME&TT@ZzrHS$Q%LC?n|e>V+D+8D zYc4)QddFz7I8#}y#Wj6>4P%34dZH~OUDb?uP%-E zwjXM(?Sg~1!|wI(RVuxbu)-rH+O=igSho_pDCw(c6b=P zKk4ATlB?bj9+HHlh<_!&z0rx13K3ZrAR8W)!@Y}o`?a*JJsD+twZIv`W)@Y?Amu_u zz``@-e2X}27$i(2=9rvIu5uTUOVhzwu%mNazS|lZb&PT;XE2|B&W1>=B58#*!~D&) zfVmJGg8UdP*fx(>Cj^?yS^zH#o-$Q-*$SnK(ZVFkw+er=>N^7!)FtP3y~Xxnu^nzY zikgB>Nj0%;WOltWIob|}%lo?_C7<``a5hEkx&1ku$|)i>Rh6@3h*`slY=9U}(Ql_< zaNG*J8vb&@zpdhAvv`?{=zDedJ23TD&Zg__snRAH4eh~^oawdYi6A3w8<Ozh@Kw)#bdktM^GVb zrG08?0bG?|NG+w^&JvD*7LAbjED{_Zkc`3H!My>0u5Q}m!+6VokMLXxl`Mkd=g&Xx z-a>m*#G3SLlhbKB!)tnzfWOBV;u;ftU}S!NdD5+YtOjLg?X}dl>7m^gOpihrf1;PY zvll&>dIuUGs{Qnd- zwIR3oIrct8Va^Tm0t#(bJD7c$Z7DO9*7NnRZorrSm`b`cxz>OIC;jSE3DO8`hX955ui`s%||YQtt2 z5DNA&pG-V+4oI2s*x^>-$6J?p=I>C|9wZF8z;VjR??Icg?1w2v5Me+FgAeGGa8(3S z4vg*$>zC-WIVZtJ7}o9{D-7d>zCe|z#<9>CFve-OPAYsneTb^JH!Enaza#j}^mXy1 z+ULn^10+rWLF6j2>Ya@@Kq?26>AqK{A_| zQKb*~F1>sE*=d?A?W7N2j?L09_7n+HGi{VY;MoTGr_)G9)ot$p!-UY5zZ2Xtbm=t z@dpPSGwgH=QtIcEulQNI>S-#ifbnO5EWkI;$A|pxJd885oM+ zGZ0_0gDvG8q2xebj+fbCHYfAXuZStH2j~|d^sBAzo46(K8n59+T6rzBwK)^rfPT+B zyIFw)9YC-V^rhtK`!3jrhmW-sTmM+tPH+;nwjL#-SjQPUZ53L@A>y*rt(#M(qsiB2 zx6B)dI}6Wlsw%bJ8h|(lhkJVogQZA&n{?Vgs6gNSXzuZpEyu*xySy8ro07QZ7Vk1!3tJphN_5V7qOiyK8p z#@jcDD8nmtYi1^l8ml;AF<#IPK?!pqf9D4moYk>d99Im}Jtwj6c#+A;f)CQ*f-hZ< z=p_T86jog%!p)D&5g9taSwYi&eP z#JuEK%+NULWus;0w32-SYFku#i}d~+{Pkho&^{;RxzP&0!RCm3-9K6`>KZpnzS6?L z^H^V*s!8<>x8bomvD%rh>Zp3>Db%kyin;qtl+jAv8Oo~1g~mqGAC&Qi_wy|xEt2iz zWAJEfTV%cl2Cs<1L&DLRVVH05EDq`pH7Oh7sR`NNkL%wi}8n>IXcO40hp+J+sC!W?!krJf!GJNE8uj zg-y~Ns-<~D?yqbzVRB}G>0A^f0!^N7l=$m0OdZuqAOQqLc zX?AEGr1Ht+inZ-Qiwnl@Z0qukd__a!C*CKuGdy5#nD7VUBM^6OCpxCa2A(X;e0&V4 zM&WR8+wErQ7UIc6LY~Q9x%Sn*Tn>>P`^t&idaOEnOd(Ufw#>NoR^1QdhJ8s`h^|R_ zXX`c5*O~Xdvh%q;7L!_!ohf$NfEBmCde|#uVZvEo>OfEq%+Ns7&_f$OR9xsihRpBb z+cjk8LyDm@U{YN>+r46?nn{7Gh(;WhFw6GAxtcKD+YWV?uge>;+q#Xx4!GpRkVZYu zzsF}1)7$?%s9g9CH=Zs+B%M_)+~*j3L0&Q9u7!|+T`^O{xE6qvAP?XWv9_MrZKdo& z%IyU)$Q95AB4!#hT!_dA>4e@zjOBD*Y=XjtMm)V|+IXzjuM;(l+8aA5#Kaz_$rR6! zj>#&^DidYD$nUY(D$mH`9eb|dtV0b{S>H6FBfq>t5`;OxA4Nn{J(+XihF(stSche7$es&~N$epi&PDM_N`As;*9D^L==2Q7Z2zD+CiU(|+-kL*VG+&9!Yb3LgPy?A zm7Z&^qRG_JIxK7-FBzZI3Q<;{`DIxtc48k> zc|0dmX;Z=W$+)qE)~`yn6MdoJ4co;%!`ddy+FV538Y)j(vg}5*k(WK)KWZ3WaOG!8 z!syGn=s{H$odtpqFrT#JGM*utN7B((abXnpDM6w56nhw}OY}0TiTG1#f*VFZr+^-g zbP10`$LPq_;PvrA1XXlyx2uM^mrjTzX}w{yuLo-cOClE8MMk47T25G8M!9Z5ypOSV zAJUBGEg5L2fY)ZGJb^E34R2zJ?}Vf>{~gB!8=5Z) z9y$>5c)=;o0HeHHSuE4U)#vG&KF|I%-cF6f$~pdYJWk_dD}iOA>iA$O$+4%@>JU08 zS`ep)$XLPJ+n0_i@PkF#ri6T8?ZeAot$6JIYHm&P6EB=BiaNY|aA$W0I+nz*zkz_z zkEru!tj!QUffq%)8y0y`T&`fuus-1p>=^hnBiBqD^hXrPs`PY9tU3m0np~rISY09> z`P3s=-kt_cYcxWd{de@}TwSqg*xVhp;E9zCsnXo6z z?f&Sv^U7n4`xr=mXle94HzOdN!2kB~4=%)u&N!+2;z6UYKUDqi-s6AZ!haB;@&B`? z_TRX0%@suz^TRdCb?!vNJYPY8L_}&07uySH9%W^Tc&1pia6y1q#?*Drf}GjGbPjBS zbOPcUY#*$3sL2x4v_i*Y=N7E$mR}J%|GUI(>WEr+28+V z%v5{#e!UF*6~G&%;l*q*$V?&r$Pp^sE^i-0$+RH3ERUUdQ0>rAq2(2QAbG}$y{de( z>{qD~GGuOk559Y@%$?N^1ApVL_a704>8OD%8Y%8B;FCt%AoPu8*D1 zLB5X>b}Syz81pn;xnB}%0FnwazlWfUV)Z-~rZg6~b z6!9J$EcE&sEbzcy?CI~=boWA&eeIa%z(7SE^qgVLz??1Vbc1*aRvc%Mri)AJaAG!p z$X!_9Ds;Zz)f+;%s&dRcJt2==P{^j3bf0M=nJd&xwUGlUFn?H=2W(*2I2Gdu zv!gYCwM10aeus)`RIZSrCK=&oKaO_Ry~D1B5!y0R=%!i2*KfXGYX&gNv_u+n9wiR5 z*e$Zjju&ODRW3phN925%S(jL+bCHv6rZtc?!*`1TyYXT6%Ju=|X;6D@lq$8T zW{Y|e39ioPez(pBH%k)HzFITXHvnD6hw^lIoUMA;qAJ^CU?top1fo@s7xT13Fvn1H z6JWa-6+FJF#x>~+A;D~;VDs26>^oH0EI`IYT2iagy23?nyJ==i{g4%HrAf1-*v zK1)~@&(KkwR7TL}L(A@C_S0G;-GMDy=MJn2$FP5s<%wC)4jC5PXoxrQBFZ_k0P{{s@sz+gX`-!=T8rcB(=7vW}^K6oLWMmp(rwDh}b zwaGGd>yEy6fHv%jM$yJXo5oMAQ>c9j`**}F?MCry;T@47@r?&sKHgVe$MCqk#Z_3S z1GZI~nOEN*P~+UaFGnj{{Jo@16`(qVNtbU>O0Hf57-P>x8Jikp=`s8xWs^dAJ9lCQ z)GFm+=OV%AMVqVATtN@|vp61VVAHRn87}%PC^RAzJ%JngmZTasWBAWsoAqBU+8L8u z4A&Pe?fmTm0?mK-BL9t+{y7o(7jm+RpOhL9KnY#E&qu^}B6=K_dB}*VlSEiC9fn)+V=J;OnN)Ta5v66ic1rG+dGAJ1 z1%Zb_+!$=tQ~lxQrzv3x#CPb?CekEkA}0MYSgx$Jdd}q8+R=ma$|&1a#)TQ=l$1tQ z=tL9&_^vJ)Pk}EDO-va`UCT1m#Uty1{v^A3P~83_#v^ozH}6*9mIjIr;t3Uv%@VeW zGL6(CwCUp)Jq%G0bIG%?{_*Y#5IHf*5M@wPo6A{$Um++Co$wLC=J1aoG93&T7Ho}P z=mGEPP7GbvoG!uD$k(H3A$Z))+i{Hy?QHdk>3xSBXR0j!11O^mEe9RHmw!pvzv?Ua~2_l2Yh~_!s1qS`|0~0)YsbHSz8!mG)WiJE| z2f($6TQtt6L_f~ApQYQKSb=`053LgrQq7G@98#igV>y#i==-nEjQ!XNu9 z~;mE+gtj4IDDNQJ~JVk5Ux6&LCSFL!y=>79kE9=V}J7tD==Ga+IW zX)r7>VZ9dY=V&}DR))xUoV!u(Z|%3ciQi_2jl}3=$Agc(`RPb z8kEBpvY>1FGQ9W$n>Cq=DIpski};nE)`p3IUw1Oz0|wxll^)4dq3;CCY@RyJgFgc# zKouFh!`?Xuo{IMz^xi-h=StCis_M7yq$u) z?XHvw*HP0VgR+KR6wI)jEMX|ssqYvSf*_3W8zVTQzD?3>H!#>InzpSO)@SC8q*ii- z%%h}_#0{4JG;Jm`4zg};BPTGkYamx$Xo#O~lBirRY)q=5M45n{GCfV7h9qwyu1NxOMoP4)jjZMxmT|IQQh0U7C$EbnMN<3)Kk?fFHYq$d|ICu>KbY_hO zTZM+uKHe(cIZfEqyzyYSUBZa8;Fcut-GN!HSA9ius`ltNebF46ZX_BbZNU}}ZOm{M2&nANL9@0qvih15(|`S~z}m&h!u4x~(%MAO$jHRWNfuxWF#B)E&g3ghSQ9|> z(MFaLQj)NE0lowyjvg8z0#m6FIuKE9lDO~Glg}nSb7`~^&#(Lw{}GVOS>U)m8bF}x zVjbXljBm34Cs-yM6TVusr+3kYFjr28STT3g056y3cH5Tmge~ASxBj z%|yb>$eF;WgrcOZf569sDZOVwoo%8>XO>XQOX1OyN9I-SQgrm;U;+#3OI(zrWyow3 zk==|{lt2xrQ%FIXOTejR>;wv(Pb8u8}BUpx?yd(Abh6? zsoO3VYWkeLnF43&@*#MQ9-i-d0t*xN-UEyNKeyNMHw|A(k(_6QKO=nKMCxD(W(Yop zsRQ)QeL4X3Lxp^L%wzi2-WVSsf61dqliPUM7srDB?Wm6Lzn0&{*}|IsKQW;02(Y&| zaTKv|`U(pSzuvR6Rduu$wzK_W-Y-7>7s?G$)U}&uK;<>vU}^^ns@Z!p+9?St1s)dG zK%y6xkPyyS1$~&6v{kl?Md6gwM|>mt6Upm>oa8RLD^8T{0?HC!Z>;(Bob7el(DV6x zi`I)$&E&ngwFS@bi4^xFLAn`=fzTC;aimE^!cMI2n@Vo%Ae-ne`RF((&5y6xsjjAZ zVguVoQ?Z9uk$2ON;ersE%PU*xGO@T*;j1BO5#TuZKEf(mB7|g7pcEA=nYJ{s3vlbg zd4-DUlD{*6o%Gc^N!Nptgay>j6E5;3psI+C3Q!1ZIbeCubW%w4pq9)MSDyB{HLm|k zxv-{$$A*pS@csolri$Ge<4VZ}e~78JOL-EVyrbxKra^d{?|NnPp86!q>t<&IP07?Z z^>~IK^k#OEKgRH+LjllZXk7iA>2cfH6+(e&9ku5poo~6y{GC5>(bRK7hwjiurqAiZ zg*DmtgY}v83IjE&AbiWgMyFbaRUPZ{lYiz$U^&Zt2YjG<%m((&_JUbZcfJ22(>bi5 z!J?<7AySj0JZ&<-qXX;mcV!f~>G=sB0KnjWca4}vrtunD^1TrpfeS^4dvFr!65knK zZh`d;*VOkPs4*-9kL>$GP0`(M!j~B;#x?Ba~&s6CopvO86oM?-? zOw#dIRc;6A6T?B`Qp%^<U5 z19x(ywSH$_N+Io!6;e?`tWaM$`=Db!gzx|lQ${DG!zb1Zl&|{kX0y6xvO1o z220r<-oaS^^R2pEyY;=Qllqpmue|5yI~D|iI!IGt@iod{Opz@*ml^w2bNs)p`M(Io z|E;;m*Xpjd9l)4G#KaWfV(t8YUn@A;nK^#xgv=LtnArX|vWQVuw3}B${h+frU2>9^ z!l6)!Uo4`5k`<<;E(ido7M6lKTgWezNLq>U*=uz&s=cc$1%>VrAeOoUtA|T6gO4>UNqsdK=NF*8|~*sl&wI=x9-EGiq*aqV!(VVXA57 zw9*o6Ir8Lj1npUXvlevtn(_+^X5rzdR>#(}4YcB9O50q97%rW2me5_L=%ffYPUSRc z!vv?Kv>dH994Qi>U(a<0KF6NH5b16enCp+mw^Hb3Xs1^tThFpz!3QuN#}KBbww`(h z7GO)1olDqy6?T$()R7y%NYx*B0k_2IBiZ14&8|JPFxeMF{vW>HF-ViB*%t0;Thq2} z+qP}n=Cp0wwr%5S+qN<7?r+``=l(h0z2`^8j;g2~Q4u?{cIL{JYY%l|iw&YH4FL(8 z1-*E#ANDHi+1f%lMJbRfq*`nG)*#?EJEVoDH5XdfqwR-C{zmbQoh?E zhW!|TvYv~>R*OAnyZf@gC+=%}6N90yU@E;0b_OV#xL9B?GX(D&7BkujjFC@HVKFci zb_>I5e!yuHA1LC`xm&;wnn|3ht3h7|rDaOsh0ePhcg_^Wh8Bq|AGe`4t5Gk(9^F;M z8mFr{uCm{)Uq0Xa$Fw6+da`C4%)M_#jaX$xj;}&Lzc8wTc%r!Y#1akd|6FMf(a4I6 z`cQqS_{rm0iLnhMG~CfDZc96G3O=Tihnv8g;*w?)C4N4LE0m#H1?-P=4{KeC+o}8b zZX)x#(zEysFm$v9W8-4lkW%VJIjM~iQIVW)A*RCO{Oe_L;rQ3BmF*bhWa}!=wcu@# zaRWW{&7~V-e_$s)j!lJsa-J?z;54!;KnU3vuhp~(9KRU2GKYfPj{qA?;#}H5f$Wv-_ zGrTb(EAnpR0*pKft3a}6$npzzq{}ApC&=C&9KoM3Ge@24D^8ZWJDiXq@r{hP=-02& z@Qrn-cbr2YFc$7XR0j7{jAyR;4LLBf_XNSrmd{dV3;ae;fsEjds*2DZ&@#e)Qcc}w zLgkfW=9Kz|eeM$E`-+=jQSt}*kAwbMBn7AZSAjkHUn4n||NBq*|2QPcKaceA6m)g5 z_}3?DX>90X|35eI7?n+>f9+hl5b>#q`2+`FXbOu9Q94UX-GWH;d*dpmSFd~7WM#H2 zvKNxjOtC)U_tx*0(J)eAI8xAD8SvhZ+VRUA?)| zeJjvg9)vi`Qx;;1QP!c_6hJp1=J=*%!>ug}%O!CoSh-D_6LK0JyiY}rOaqSeja&jb#P|DR7 z_JannlfrFeaE$irfrRIiN|huXmQhQUN6VG*6`bzN4Z3!*G?FjN8!`ZTn6Wn4n=Ync z_|Sq=pO7+~{W2}599SfKz@umgRYj6LR9u0*BaHqdEw^i)dKo5HomT9zzB$I6w$r?6 zs2gu*wNOAMK`+5yPBIxSOJpL$@SN&iUaM zQ3%$EQt%zQBNd`+rl9R~utRDAH%7XP@2Z1s=)ks77I(>#FuwydE5>LzFx)8ye4ClM zb*e2i*E$Te%hTKh7`&rQXz;gvm4Dam(r-!FBEcw*b$U%Wo9DIPOwlC5Ywm3WRCM4{ zF42rnEbBzUP>o>MA){;KANhAW7=FKR=DKK&S1AqSxyP;k z;fp_GVuV}y6YqAd)5p=tJ~0KtaeRQv^nvO?*hZEK-qA;vuIo!}Xgec4QGW2ipf2HK z&G&ppF*1aC`C!FR9(j4&r|SHy74IiDky~3Ab)z@9r&vF+Bapx<{u~gb2?*J zSl{6YcZ$&m*X)X?|8<2S}WDrWN3yhyY7wlf*q`n^z3LT4T$@$y``b{m953kfBBPpQ7hT;zs(Nme`Qw@{_pUO0OG zfugi3N?l|jn-Du3Qn{Aa2#6w&qT+oof=YM!Zq~Xi`vlg<;^)Jreeb^x6_4HL-j}sU z1U^^;-WetwPLKMsdx4QZ$haq3)rA#ATpEh{NXto-tOXjCwO~nJ(Z9F%plZ{z(ZW!e zF>nv&4ViOTs58M+f+sGimF^9cB*9b(gAizwyu5|--SLmBOP-uftqVnVBd$f7YrkJ8!jm*QQEQC zEQ+@T*AA1kV@SPF6H5sT%^$$6!e5;#N((^=OA5t}bqIdqf`PiMMFEDhnV#AQWSfLp zX=|ZEsbLt8Sk&wegQU0&kMC|cuY`&@<#r{t2*sq2$%epiTVpJxWm#OPC^wo_4p++U zU|%XFYs+ZCS4JHSRaVET)jV?lbYAd4ouXx0Ka6*wIFBRgvBgmg$kTNQEvs0=2s^sU z_909)3`Ut!m}}@sv<63E@aQx}-!qVdOjSOnAXTh~MKvr$0nr(1Fj-3uS{U6-T9NG1Y(Ua)Nc}Mi< zOBQz^&^v*$BqmTIO^;r@kpaq3n!BI?L{#bw)pdFV&M?D0HKqC*YBxa;QD_4(RlawI z5wBK;7T^4dT7zt%%P<*-M~m?Et;S^tdNgQSn?4$mFvIHHL!`-@K~_Ar4vBnhy{xuy zigp!>UAwPyl!@~(bkOY;un&B~Evy@5#Y&cEmzGm+)L~4o4~|g0uu&9bh8N0`&{B2b zDj2>biRE1`iw}lv!rl$Smn(4Ob>j<{4dT^TfLe-`cm#S!w_9f;U)@aXWSU4}90LuR zVcbw;`2|6ra88#Cjf#u62xq?J)}I)_y{`@hzES(@mX~}cPWI8}SRoH-H;o~`>JWU$ zhLudK3ug%iS=xjv9tnmOdTXcq_?&o30O;(+VmC&p+%+pd_`V}RY4ibQMNE&N5O+hb3bQ8bxk^33Fu4DB2*~t1909gqoutQHx^plq~;@g$d_+rzS0`2;}2UR2h#?p35B=B*f0BZS4ysiWC!kw?4B-dM%m6_BfRbey1Wh? zT1!@>-y=U}^fxH0A`u1)Mz90G6-<4aW^a@l_9L6Y;cd$3<#xIrhup)XLkFi$W&Ohu z8_j~-VeVXDf9b&6aGelt$g*BzEHgzh)KDgII_Y zb$fcY8?XI6-GEGTZVWW%O;njZld)29a_&1QvNYJ@OpFrUH{er@mnh*}326TYAK7_Z zA={KnK_o3QLk|%m@bx3U#^tCChLxjPxMesOc5D4G+&mvp@Clicz^=kQlWp1|+z|V7 zkU#7l61m@^#`1`{+m2L{sZC#j?#>0)2z4}}kqGhB{NX%~+3{5jOyij!e$5-OAs zDvq+>I2(XsY9%NNhNvKiF<%!6t^7&k{L7~FLdkP9!h%=2Kt$bUt(Zwp*&xq_+nco5 zK#5RCM_@b4WBK*~$CsWj!N!3sF>ijS=~$}_iw@vbKaSp5Jfg89?peR@51M5}xwcHW z(@1TK_kq$c4lmyb=aX3-JORe+JmuNkPP=bM*B?};c=_;h2gT-nt#qbriPkpaqoF@q z<)!80iKvTu`T-B3VT%qKO^lfPQ#m5Ei6Y%Fs@%Pt!8yX&C#tL$=|Ma8i?*^9;}Fk> zyzdQQC5YTBO&gx6kB~yhUUT&%q3a3o+zueh>5D7tdByYVcMz@>j!C@Iyg{N1)veYl`SPshuH6Rk=O6pvVrI71rI5*%uU3u81DpD%qmXsbKWMFR@2m4vO_^l6MMbO9a()DcWmYT&?0B_ zuY~tDiQ6*X7;9B*5pj?;xy_B}*{G}LjW*qU&%*QAyt30@-@O&NQTARZ+%VScr>`s^KX;M!p; z?8)|}P}L_CbOn!u(A{c5?g{s31Kn#7i)U@+_KNU-ZyVD$H7rtOjSht8%N(ST-)%r` z63;Hyp^KIm-?D;E-EnpAAWgz2#z{fawTx_;MR7)O6X~*jm*VUkam7>ueT^@+Gb3-Y zN3@wZls8ibbpaoR2xH=$b3x1Ng5Tai=LT2@_P&4JuBQ!r#Py3ew!ZVH4~T!^TcdyC ze#^@k4a(nNe~G+y zI~yXK@1HHWU4pj{gWT6v@$c(x){cLq*KlFeKy?f$_u##)hDu0X_mwL6uKei~oPd9( zRaF_k&w(J3J8b_`F~?0(Ei_pH}U^c&r$uSYawB8Ybs-JZ|&;vKLWX! z|HFZ%-uBDaP*hMcQKf*|j5!b%H40SPD*#{A`kj|~esk@1?q}-O7WyAm3mD@-vHzw( zTSOlO(K9>GW;@?@xSwpk%X3Ui4_Psm;c*HF~RW+q+C#RO_VT5(x!5B#On-W`T|u z>>=t)W{=B-8wWZejxMaBC9sHzBZGv5uz_uu281kxHg2cll_sZBC&1AKD`CYh2vKeW zm#|MMdC}6A&^DX=>_(etx8f}9o}`(G?Y``M?D+aTPJbZqONmSs>y>WSbvs>7PE~cb zjO+1Y)PMi*!=06^$%< z*{b^66BIl{7zKvz^jut7ylDQBt)ba_F*$UkDgJ2gSNfHB6+`OEiz@xs$Tcrl>X4?o zu9~~b&Xl0?w(7lJXu8-9Yh6V|A3f?)1|~+u-q&6#YV`U2i?XIqUw*lc-QTXwuf@8d zSjMe1BhBKY`Mo{$s%Ce~Hv(^B{K%w{yndEtvyYjjbvFY^rn2>C1Lbi!3RV7F>&;zlSDSk}R>{twI}V zA~NK%T!z=^!qbw(OEgsmSj?#?GR&A$0&K>^(?^4iphc3rN_(xXA%joi)k~DmRLEXl zaWmwMolK%@YiyI|HvX{X$*Ei7y+zJ%m{b}$?N7_SN&p+FpeT%4Z_2`0CP=}Y3D-*@ zL|4W4ja#8*%SfkZzn5sfVknpJv&>glRk^oUqykedE8yCgIwCV)fC1iVwMr4hc#KcV!|M-r_N|nQWw@`j+0(Ywct~kLXQ)Qyncmi{Q4`Ur7A{Ep)n`zCtm8D zVX`kxa8Syc`g$6$($Qc-(_|LtQKWZXDrTir5s*pSVmGhk#dKJzCYT?vqA9}N9DGv> zw}N$byrt?Mk*ZZbN5&zb>pv;rU}EH@Rp54)vhZ=330bLvrKPEPu!WqR%yeM3LB!(E zw|J05Y!tajnZ9Ml*-aX&5T8YtuWDq@on)_*FMhz-?m|>RT0~e3OHllrEMthVY(KwQ zu>ijTc4>Xz-q1(g!ESjaZ+C+Zk5FgmF)rFX29_RmU!`7Pw+0}>8xK^=pOxtUDV)ok zw-=p=OvEH&VO3wToRdI!hPHc`qX+_{T_mj!NxcA&xOgkEuvz`-Aa`ZlNv>qnD0`YT1T3USO0ec!%{KE~UOGPJX%I5_rZDGx@|w zVIMsRPP+}^Xxa&{x!q{hY1wat8jDO7YP0(8xHWeEdrd79lUjB8%)v{X1pQu|1dr*y9M&a(J`038}4>lK&K zIM~6wnX{XA?pFHz{hOmEq{oYBnB@56twXqEcFrFqvCy)sH9B{pQ`G50o{W^t&onwY z-l{ur4#8ylPV5YRLD%%j^d0&_WI>0nmfZ8! zaZ&vo@7D`!=?215+Vk181*U@^{U>VyoXh2F&ZNzZx5tDDtlLc)gi2=|o=GC`uaH;< zFuuF?Q9Q`>S#c(~2p|s49RA`3242`2P+)F)t2N!CIrcl^0#gN@MLRDQ2W4S#MXZJO z8<(9P>MvW;rf2qZ$6sHxCVIr0B-gP?G{5jEDn%W#{T#2_&eIjvlVqm8J$*8A#n`5r zs6PuC!JuZJ@<8cFbbP{cRnIZs>B`?`rPWWL*A?1C3QqGEG?*&!*S0|DgB~`vo_xIo z&n_Sa(>6<$P7%Py{R<>n6Jy?3W|mYYoxe5h^b6C#+UoKJ(zl?^WcBn#|7wMI5=?S# zRgk8l-J`oM%GV&jFc)9&h#9mAyowg^v%Fc-7_^ou5$*YvELa!1q>4tHfX7&PCGqW* zu8In~5`Q5qQvMdToE$w+RP^_cIS2xJjghjCTp6Z(za_D<$S;0Xjt?mAE8~Ym{)zfb zV62v9|59XOvR}wEpm~Cnhyr`=JfC$*o15k?T`3s-ZqF6Gy;Gm+_6H$%oJPywWA^Wl zzn$L=N%{VT8DkQba0|2LqGR#O2Pw!b%LV4#Ojcx5`?Cm;+aLpkyZ=!r1z@E}V= z$2v6v%Ai)MMd`@IM&UD!%%(63VH8+m0Ebk<5Du#0=WeK(E<2~3@>8TceT$wy5F52n zRFtY>G9Gp~h#&R92{G{jLruZSNJ4)gNK+zg*$P zW@~Hf>_Do)tvfEAAMKE1nQ=8coTgog&S;wj(s?Xa0!r?UU5#2>18V#|tKvay1Ka53 zl$RxpMqrkv`Sv&#!_u8$8PMken`QL0_sD2)r&dZziefzSlAdKNKroVU;gRJE#o*}w zP_bO{F4g;|t!iroy^xf~(Q5qc8a3<+vBW%VIOQ1!??d;yEn1at1wpt}*n- z0iQtfu}Isw4ZfH~8p~#RQUKwf<$XeqUr-5?8TSqokdHL7tY|47R; z#d+4NS%Cqp>LQbvvAMIhcCX@|HozKXl)%*5o>P2ZegGuOerV&_MeA}|+o-3L!ZNJd z#1xB^(r!IfE~i>*5r{u;pIfCjhY^Oev$Y1MT16w8pJ0?9@&FH*`d;hS=c#F6fq z{mqsHd*xa;>Hg?j80MwZ%}anqc@&s&2v{vHQS68fueNi5Z(VD2eH>jmv4uvE|HEQm z^=b&?1R9?<@=kjtUfm*I!wPf5Xnma(4*DfPk}Es*H$%NGCIM1qt(LSvbl7&tV>e2$ zUqvZOTiwQyxDoxL(mn?n_x%Tre?L&!FYCOy0>o}#DTC3uSPnyGBv*}!*Yv5IV)Bg_t%V+UrTXfr!Q8+eX}ANR*YLzwme7Rl z@q_*fP7wP2AZ(3WG*)4Z(q@)~c{Je&7?w^?&Wy3)v0{TvNQRGle9mIG>$M2TtQ(Vf z3*PV@1mX)}beRTPjoG#&&IO#Mn(DLGp}mn)_0e=9kXDewC8Pk@yo<8@XZjFP-_zic z{mocvT9Eo)H4Oj$>1->^#DbbiJn^M4?v7XbK>co+v=7g$hE{#HoG6ZEat!s~I<^_s zlFee93KDSbJKlv_+GPfC6P8b>(;dlJ5r9&Pc4kC2uR(0{Kjf+SMeUktef``iXD}8` zGufkM9*Sx4>+5WcK#Vqm$g#5z1DUhc_#gLGe4_icSzN5GKr|J&eB)LS;jTXWA$?(k zy?*%U9Q#Y88(blIlxrtKp6^jksNF>-K1?8=pmYAPj?qq}yO5L>_s8CAv=LQMe3J6? zOfWD>Kx_5A4jRoIU}&aICTgdYMqC|45}St;@0~7>Af+uK3vps9D!9qD)1;Y6Fz>4^ zR1X$s{QNZl7l%}Zwo2wXP+Cj-K|^wqZW?)s1WUw_APZLhH55g{wNW3liInD)WHh${ zOz&K>sB*4inVY3m)3z8w!yUz+CKF%_-s2KVr7DpwTUuZjPS9k-em^;>H4*?*B0Bg7 zLy2nfU=ac5N}x1+Tlq^lkNmB~Dj+t&l#fO&%|7~2iw*N!*xBy+ZBQ>#g_;I*+J{W* z=@*15><)Bh9f>>dgQrEhkrr2FEJ;R2rH%`kda8sD-FY6e#7S-<)V*zQA>)Ps)L- zgUuu@5;Ych#jX_KZ+;qEJJbu{_Z9WSsLSo#XqLpCK$gFidk}gddW(9$v}iyGm_OoH ztn$pv81zROq686_7@avq2heXZnkRi4n(3{5jTDO?9iP%u8S4KEqGL?^uBeg(-ws#1 z9!!Y_2Q~D?gCL3MQZO!n$+Wy(Twr5AS3{F7ak2f)Bu0iG^k^x??0}b6l!>Vjp{e*F z8r*(Y?3ZDDoS1G?lz#J4`d9jAEc9YGq1LbpYoFl!W!(j8-33Ey)@yx+BVpDIVyvpZ zq5QgKy>P}LlV?Bgy@I)JvefCG)I69H1;q@{8E8Ytw^s-rC7m5>Q>ZO(`$`9@`49s2)q#{2eN0A?~qS8%wxh%P*99h*Sv` zW_z3<=iRZBQKaDsKw^TfN;6`mRck|6Yt&e$R~tMA0ix;qgw$n~fe=62aG2v0S`7mU zI}gR#W)f+Gn=e3mm*F^r^tcv&S`Rym`X`6K`i8g-a0!p|#69@Bl!*&)QJ9(E7ycxz z)5-m9v`~$N1zszFi^=m%vw}Y{ZyYub!-6^KIY@mwF|W+|t~bZ%@rifEZ-28I@s$C` z>E+k~R1JC-M>8iC_GR>V9f9+uL2wPRATL9bC(sxd;AMJ>v6c#PcG|Xx1N5^1>ISd0 z4%vf-SNOw+1%yQq1YP`>iqq>5Q590_pr?OxS|HbLjx=9~Y)QO37RihG%JrJ^=Nj>g zPTcO$6r{jdE_096b&L;Wm8vcxUVxF0mA%W`aZz4n6XtvOi($ zaL!{WUCh&{5ar=>u)!mit|&EkGY$|YG<_)ZD)I32uEIWwu`R-_ z`FVeKyrx3>8Ep#2~%VVrQ%u#exo!anPe`bc)-M=^IP1n1?L2UQ@# zpNjoq-0+XCfqXS!LwMgFvG$PkX}5^6yxW)6%`S8{r~BA2-c%-u5SE#%mQ~5JQ=o$c z%+qa0udVq9`|=2n=0k#M=yiEh_vp?(tB|{J{EhVLPM^S@f-O*Lgb390BvwK7{wfdMKqUc0uIXKj5>g^z z#2`5^)>T73Eci+=E4n&jl42E@VYF2*UDiWLUOgF#p9`E4&-A#MJLUa&^hB@g7KL+n zr_bz+kfCcLIlAevILckIq~RCwh6dc5@%yN@#f3lhHIx4fZ_yT~o0#3@h#!HCN(rHHC6#0$+1AMq?bY~(3nn{o5g8{*e_#4RhW)xPmK zTYBEntuYd)`?`bzDksI9*MG$=^w!iiIcWg1lD&kM1NF@qKha0fDVz^W7JCam^!AQFxY@7*`a3tfBwN0uK_~YBQ18@^i%=YB}K0Iq(Q3 z=7hNZ#!N@YErE7{T|{kjVFZ+f9Hn($zih;f&q^wO)PJSF`K)|LdT>!^JLf=zXG>>G z15TmM=X`1%Ynk&dvu$Vic!XyFC(c=qM33v&SIl|p+z6Ah9(XQ0CWE^N-LgE#WF6Z+ zb_v`7^Rz8%KKg_@B>5*s-q*TVwu~MCRiXvVx&_3#r1h&L+{rM&-H6 zrcgH@I>0eY8WBX#Qj}Vml+fpv?;EQXBbD0lx%L?E4)b-nvrmMQS^}p_CI3M24IK(f| zV?tWzkaJXH87MBz^HyVKT&oHB;A4DRhZy;fIC-TlvECK)nu4-3s7qJfF-ZZGt7+6C3xZt!ZX4`M{eN|q!y*d^B+cF5W- zc9C|FzL;$bAfh56fg&y0j!PF8mjBV!qA=z$=~r-orU-{0AcQUt4 zNYC=_9(MOWe$Br9_50i#0z!*a1>U6ZvH>JYS9U$kkrCt7!mEUJR$W#Jt5vT?U&LCD zd@)kn%y|rkV|CijnZ((B2=j_rB;`b}F9+E1T46sg_aOPp+&*W~44r9t3AI}z)yUFJ z+}z5E6|oq+oPC3Jli)EPh9)o^B4KUYkk~AU9!g`OvC`a!#Q>JmDiMLTx>96_iDD9h@nW%Je4%>URwYM%5YU1&Dcdulvv3IH3GSrA4$)QjlGwUt6 zsR6+PnyJ$1x{|R=ogzErr~U|X!+b+F8=6y?Yi`E$yjWXsdmxZa^hIqa)YV9ubUqOj&IGY}bk zH4*DEn({py@MG5LQCI;J#6+98GaZYGW-K-&C`(r5#?R0Z){DlY8ZZk}lIi$xG}Q@2 z0LJhzuus-7dLAEpG1Lf+KOxn&NSwO{wn_~e0=}dovX)T(|WRMTqacoW8;A>8tTDr+0yRa+U!LW z!H#Gnf^iCy$tTk3kBBC=r@xhskjf1}NOkEEM4*r+A4`yNAIjz`_JMUI#xTf$+{UA7 zpBO_aJkKz)iaKqRA{8a6AtpdUwtc#Y-hxtZnWz~i(sfjMk`lq|kGea=`62V6y)TMPZw8q}tFDDHrW_n(Z84ZxWvRrntcw;F|Mv4ff9iaM% z4IM{=*zw}vIpbg=9%w&v`sA+a3UV@Rpn<6`c&5h+8a7izP>E@7CSsCv*AAvd-izwU z!sGJQ?fpCbt+LK`6m2Z3&cKtgcElAl){*m0b^0U#n<7?`8ktdIe#ytZTvaZy728o6 z3GDmw=vhh*U#hCo0gb9s#V5(IILXkw>(6a?BFdIb0%3~Y*5FiMh&JWHd2n(|y@?F8 zL$%!)uFu&n+1(6)oW6Hx*?{d~y zBeR)N*Z{7*gMlhMOad#k4gf`37OzEJ&pH?h!Z4#mNNCfnDI@LbiU~&2Gd^q7ix8~Y6$a=B9bK(BaTEO0$Oh=VCkBPwt0 zf#QuB25&2!m7MWY5xV_~sf(0|Y*#Wf8+FQI(sl2wgdM5H7V{aH6|ntE+OcLsTC`u; zeyrlkJgzdIb5=n#SCH)+kjN)rYW7=rppN3Eb;q_^8Zi}6jtL@eZ2XO^w{mCwX(q!t ztM^`%`ndZ5c+2@?p>R*dDNeVk#v>rsn>vEo;cP2Ecp=@E>A#n0!jZACKZ1=D0`f|{ zZnF;Ocp;$j86m}Gt~N+Ch6CJo7+Wzv|nlsXBvm z?St-5Ke&6hbGAWoO!Z2Rd8ARJhOY|a1rm*sOif%Th`*=^jlgWo%e9`3sS51n*>+Mh(9C7g@*mE|r%h*3k6I_uo;C!N z7CVMIX4kbA#gPZf_0%m18+BVeS4?D;U$QC`TT;X zP#H}tMsa=zS6N7n#BA$Fy8#R7vOesiCLM@d1UO6Tsnwv^gb}Q9I}ZQLI?--C8ok&S z9Idy06+V(_aj?M78-*vYBu|AaJ9mlEJpFEIP}{tRwm?G{ag>6u(ReBKAAx zDR6qe!3G88NQP$i99DZ~CW9lzz}iGynvGA4!yL}_9t`l*SZbEL-%N{n$%JgpDHJRn zvh<{AqR7z@ylV`kXdk+uEu-WWAt^=A4n(J=A1e8DpeLzAd;Nl#qlmp#KcHU!8`YJY zvBZy@>WiBZpx*wQ8JzKw?@k}8l99Wo&H>__vCFL}>m~MTmGvae% zPTn9?iR=@7NJ)?e+n-4kx$V#qS4tLpVUX*Je0@`f5LICdxLnph&Vjbxd*|+PbzS(l zBqqMlUeNoo8wL&_HKnM^8{iDI3IdzJAt32UupSr6XXh9KH2LjWD)Pz+`cmps%eHeD zU%i1SbPuSddp6?th;;DfUlxYnjRpd~i7vQ4V`cD%4+a9*!{+#QRBr5^Q$5Ec?gpju zv@dk9;G>d7QNEdRy}fgeA?i=~KFeibDtYffy)^OP?Ro~-X!onDpm+uGpe&6)*f@xJ zE1I3Qh}`1<7aFB@TS#}ee={<#9%1wOL%cuvOd($y4MC2?`1Nin=pVLXPkknn*0kx> z!9XHW${hYEV;r6F#iz7W=fg|a@GY0UG5>>9>$3Bj5@!N{nWDD`;JOdz_ZaZVVIUgH zo+<=+n8VGL*U%M|J$A~#ll__<`y+jL>bv;TpC!&|d=q%E2B|5p=)b-Q+ZrFO%+D_u z4%rc8BmOAO6{n(i(802yZW93?U;K^ZZlo0Gvs7B+<%}R;$%O}pe*Gi;!xP-M73W`k zXLv473Ex_VPcM-M^JO|H>KD;!sEGJ|E}Qepen;yNG2 zXqgD5sjQUDI(XLM+^8ZX1s_(X+PeyQ$Q5RukRt|Kwr-FSnW!^9?OG64UYX1^bU9d8 zJ}8K&UEYG+Je^cThf8W*^RqG07nSCmp*o5Z;#F zS?jochDWX@p+%CZ%dOKUl}q{9)^U@}qkQtA3zBF)`I&zyIKgb{mv)KtZ}?_h{r#VZ z%C+hwv&nB?we0^H+H`OKGw-&8FaF;=ei!tAclS5Q?qH9J$nt+YxdKkbRFLnWvn7GH zezC6<{mK0dd763JlLFqy&Oe|7UXII;K&2pye~yG4jldY~N;M9&rX}m76NsP=R#FEw zt(9h+=m9^zfl=6pH*D;JP~OVgbJkXh(+2MO_^;%F{V@pc2nGn~=U)Qx|JEV-e=vXk zPxA2J<9~IH{}29#X~KW$(1reJv}lc4_1JF31gdev>!CddVhf_62nsr6%w)?IWxz}{ z(}~~@w>c07!r=FZANq4R!F2Qi2?QGavZ{)PCq~X}3x;4ylsd&m;dQe;0GFSn5 zZ*J<=Xg1fEGYYDZ0{Z4}Jh*xlXa}@412nlKSM#@wjMM z*0(k>Gfd1Mj)smUuX}EM6m)811%n5zzr}T?$ZzH~*3b`3q3gHSpA<3cbzTeRDi`SA zT{O)l3%bH(CN0EEF9ph1(Osw5y$SJolG&Db~uL!I3U{X`h(h%^KsL71`2B1Yn z7(xI+Fk?|xS_Y5)x?oqk$xmjG@_+JdErI(q95~UBTvOXTQaJs?lgrC6Wa@d0%O0cC zzvslIeWMo0|C0({iEWX{=5F)t4Z*`rh@-t0ZTMse3VaJ`5`1zeUK0~F^KRY zj2z-gr%sR<(u0@SNEp%Lj38AB2v-+cd<8pKdtRU&8t3eYH#h7qH%bvKup4cnnrN>l z!5fve)~Y5_U9US`uXDFoOtx2gI&Z!t&VPIoqiv>&H(&1;J9b}kZhcOX7EiW*Bujy#MaCl52%NO-l|@2$aRKvZ!YjwpXwC#nA(tJtd1p?jx&U|?&jcb!0MT6oBlWurVRyiSCX?sN3j}d zh3==XK$^*8#zr+U^wk(UkF}bta4bKVgr`elH^az{w(m}3%23;y7dsEnH*pp{HW$Uk zV9J^I9ea7vp_A}0F8qF{>|rj`CeHZ?lf%HImvEJF<@7cgc1Tw%vAUA47{Qe(sP^5M zT=z<~l%*ZjJvObcWtlN?0$b%NdAj&l`Cr|x((dFs-njsj9%IIqoN|Q?tYtJYlRNIu zY(LtC-F14)Og*_V@gjGH^tLV4uN?f^#=dscCFV~a`r8_o?$gj3HrSk=YK2k^UW)sJ z&=a&&JkMkWshp0sto$c6j8f$J!Bsn*MTjC`3cv@l@7cINa!}fNcu(0XF7ZCAYbX|WJIL$iGx8l zGFFQsw}x|i!jOZIaP{@sw0BrV5Z5u!TGe@JGTzvH$}55Gf<;rieZlz+6E1}z_o3m2 z(t;Cp^Geen7iSt)ZVtC`+tzuv^<6--M`^5JXBeeLXV)>2;f7=l%(-4?+<5~;@=Th{1#>rK3+rLn(44TAFS@u(}dunUSYu}~))W*fr` zkBL}3k_@a4pXJ#u*_N|e#1gTqxE&WPsfDa=`@LL?PRR()9^HxG?~^SNmeO#^-5tMw zeGEW&CuX(Uz#-wZOEt8MmF}hQc%14L)0=ebo`e$$G6nVrb)afh!>+Nfa5P;N zCCOQ^NRel#saUVt$Ds0rGd%gkKP2LsQRxq6)g*`-r(FGM!Q51c|9lk!ha8Um3ys1{ zWpT7XDWYshQ{_F!8D8@3hvXhQDw;GlkUOzni&T1>^uD){WH3wRONgjh$u4u7?+$(Y zqTXEF>1aPNZCXP0nJ;zs6_%6;+D&J_|ugcih**y(4ApT`RKAi5>SZe0Bz|+l7z>P14>0ljIH*LhK z@}2O#{?1RNa&!~sEPBvIkm-uIt^Pt#%JnsbJ`-T0%pb ze}d;dzJFu7oQ=i`VHNt%Sv@?7$*oO`Rt*bRNhXh{FArB`9#f%ksG%q?Z`_<19;dBW z5pIoIo-JIK9N$IE1)g8@+4}_`sE7;Lus&WNAJ^H&=4rGjeAJP%Dw!tn*koQ&PrNZw zY88=H7qpHz11f}oTD!0lWO>pMI;i4sauS`%_!zM!n@91sLH#rz1~iEAu#1b%LA zhB}7{1(8{1{V8+SEs=*f=FcRE^;`6Pxm$Hie~|aD~W1BYy#@Y$C?pxJh*cC!T@8C9{xx*T*8P zhbkRk3*6)Zbk%}u>^?ItOhxdmX$j9KyoxxN>NrYGKMkLF4*fLsL_PRjHNNHCyaUHN z7W8yEhf&ag07fc9FD>B{t0#Civsoy0hvVepDREX(NK1LbK0n*>UJp&1FygZMg7T^G z(02BS)g#qMOI{RJIh7}pGNS8WhSH@kG+4n=(8j<+gVfTur)s*hYus70AHUBS2bN6Zp_GOHYxsbg{-Rcet{@0gzE`t$M0_!ZIqSAIW53j+Ln7N~8J zLZ0DOUjp^j`MvX#hq5dFixo^1szoQ=FTqa|@m>9F@%>7OuF9&_C_MDco&-{wfLKNrDMEN4pRUS8-SD6@GP`>_7$;r>dJo>KbeXm>GfQS? zjFS+Y6^%pDCaI0?9(z^ELsAE1`WhbhNv5DJ$Y}~r;>FynHjmjmA{bfDbseZXsKUv`%Fekv)1@f%7ti;B5hhs}5db1dP+P0${1DgKtb(DvN}6H6;0*LP6blg*rpr;Z(7? zrve>M`x6ZI(wtQc4%lO?v5vr{0iTPl&JT!@k-7qUN8b$O9YuItu7zrQ*$?xJIN#~b z#@z|*5z&D7g5>!o(^v+3N?JnJns5O2W4EkF>re*q1uVjgT#6ROP5>Ho)XTJoHDNRC zuLC(Cd_ZM?FAFPoMw;3FM4Ln0=!+vgTYBx2TdXpM@EhDCorzTS6@2`swp4J^9C0)U zq?)H8)=D;i+H`EVYge>kPy8d*AxKl};iumYu^UeM+e_3>O+LY`D4?pD%;Vextj!(; zomJ(u+dR(0m>+-61HTV7!>03vqozyo@uY@Zh^KrW`w7^ENCYh86_P2VC|4}(ilMBe zwa&B|1a7%Qkd>d14}2*_yYr@8-N}^&?LfSwr)C~UUHr)ydENu=?ZHkvoLS~xTiBH= zD%A=OdoC+10l7@rXif~Z#^AvW+4M-(KQBj=Nhgts)>xmA--IJf1jSZF6>@Ns&nmv} zXRk`|`@P5_9W4O-SI|f^DCZ-n*yX@2gf6N)epc~lRWl7QgCyXdx|zr^gy>q`Vwn^y z&r3_zS}N=HmrVtTZhAQS`3$kBmVZDqr4+o(oNok?tqel9kn3;uUerFRti=k+&W{bb zT{ZtEf51Qf+|Jc*@(nyn#U+nr1SFpu4(I7<1a=)M_yPUAcKVF+(vK!|DTL2;P)yG~ zrI*7V)wN_92cM)j`PtAOFz_dO)jIfTeawh2{d@x0nd^#?pDkBTBzr0Oxgmvjt`U^$ zcTPl=iwuen=;7ExMVh7LLFSKUrTiPJpMB&*Ml32>wl} zYn(H0N4+>MCrm2BC4p{meYPafDEXd4yf$i%ylWpC|9%R4XZBUQiha(x%wgQ5iJ?K_wQBRfw z+pYuKoIameAWV7Ex4$PCd>bYD7)A9J`ri&bwTRN*w~7DR0EeLXW|I2()Zkl6vxiw? zFBX){0zT@w_4YUT4~@TXa;nPb^Tu$DJ=vluc~9)mZ}uHd#4*V_eS7)^eZ9oI%Wws_ z`;97^W|?_Z6xHSsE!3EKHPN<3IZ^jTJW=Il{rMmlnR#OuoE6dqOO1KOMpW84ZtDHNn)(pYvs=frO`$X}sY zKY0At$G85&2>B|-{*+B*aqQn&Mqjt*DVH2kdwEm5f}~Xwn9+tPt?EPwh8=8=VWA8rjt*bHEs1FJ92QohQ)Y z4sQH~AzB5!Pisyf?pVa0?L4gthx2;SKlrr?XRU`?Y>RJgUeJn!az#sNF7oDbzksrD zw8)f=f1t*UK&$}_ktf!yf4Rjt{56ffTA{A=9n})E7~iXaQkE+%GW4zqbmlYF(|hE@ z421q9`UQf$uA5yDLx67`=EnSTxdEaG!6C%9_obpb?;u-^QFX% zU1wQ}Li{PeT^fS;&Sk2#$ZM#Zpxrn7jsd<@qhfWy*H)cw9q!I9!fDOCw~4zg zbW`EHsTp9IQUCETUse)!ZmuRICx}0Oe1KVoqdK+u>67A8v`*X*!*_i5`_qTzYRkbYXg#4vT5~A{lK#bA}Oc4ePu5hr-@;i%Z!4Y;-(yR z(1rHYTc7i1h1aipP4DaIY3g2kF#MX{XW7g&zL!39ohO98=eo5nZtq+nz}2E$OZpxx z&OFaOM1O;?mxq+`%k>YS!-=H7BB&WhqSTUC{S!x*k9E zcB;u0I!h%3nEchQwu1GnNkaQxuWnW0D@Xq5j@5WE@E(WlgDU;FLsT*eV|Bh)aH0;~@^yygFj<=+Vu3p)LlF%1AA%y5z-Oh`2 z$RDKk_6r+f#I`8fQ%y#Wx%~de1qkWL2(q^~veLKwht-dIcpt(@lc>`~@mISRIPKPm zD!Za&aX@7dy*CT!&Z7JC1jP2@8+ro8SmlH>_gzRte%ojgiwfd?TR+%Ny0`sp`QRLy zl5TiQkFhIC!2aaJ&=Ua`c9UuOk9GkSFZ}!IGeMZ5MXrL zGtMj`m{(X9+l%=d|L zW2OY?8!_pyhvJ1@O!Chsf6}@3HmKq@)x;CFItPMpkSr@npO&8zMc_O?*|sqkuL^U? zV9+x3vbr|6;Ft0J^J>IH_xpa<{S5K?u-sQWC7FB9YFMwoCKK3WZ*gvO-wAApF`K%#7@1 z^sEj4*%hH`f0@sRDGI|#Dl20o$Z*gttP$q(_?#~2!H9(!d=)I93-3)?e%@$1^*F=t9t&OQ9!p84Z`+y<$yQ9wlamK~Hz2CRpS8dWJfBl@(M2qX!9d_F= zd|4A&U~8dX^M25wyC7$Swa22$G61V;fl{%Q4Lh!t_#=SP(sr_pvQ=wqOi`R)do~QX zk*_gsy75$xoi5XE&h7;-xVECk;DLoO0lJ3|6(Ba~ezi73_SYdCZPItS5MKaGE_1My zdQpx?h&RuoQ7I=UY{2Qf ziGQ-FpR%piffR_4X{74~>Q!=i`)J@T415!{8e`AXy`J#ZK)5WWm3oH?x1PVvcAqE@ zWI|DEUgxyN({@Y99vCJVwiGyx@9)y2jNg`R{$s2o;`4!^6nDX_pb~fTuzf>ZoPV@X zXKe1ehcZ+3dxCB+vikgKz8pvH?>ZzlOEObd{(-aWY;F0XIbuIjSA+!%TNy87a>BoX zsae$}Fcw&+)z@n{Fvzo;SkAw0U*}?unSO)^-+sbpNRjD8&qyfp%GNH;YKdHlz^)4( z;n%`#2Pw&DPA8tc)R9FW7EBR3?GDWhf@0(u3G4ijQV;{qp3B)`Fd}kMV}gB2U%4Sy z3x>YU&`V^PU$xWc4J!OG{Jglti@E3rdYo62K31iu!BU&pdo}S66Ctq{NB<88P92Y9 zTOqX$h6HH_8fKH(I>MEJZl1_2GB~xI+!|BLvN;CnQrjHuh?grzUO7h;1AbzLi|_O= z2S=(0tX#nBjN92gRsv;7`rDCATA!o(ZA}6)+;g;T#+1~HXGFD1@3D#|Ky9!E@)u=h z3@zg3Us0BCYmq(pB`^QTp|RB9!lX*{;7r|Z(^>J+av(0-oUmIdR78c4(q%hP#=R@W ze{;yy$T^8kXr(oC*#NQMZSQlgU)aa=BrZDwpLUk5tm&(AkNt&Gel`=ydcL*<@Ypx{ z2uOxl>2vSY2g3%Si&JU<9D5#{_z{9PzJh=miNH;STk^;5#%8iMRfPe#G~T>^U_zt? zgSE)`UQhb!G$at%yCf5MU)<&(L73(hY3*%qqPbX;`%QDHed3ZaWw^k)8Vjd#ePg@;I&pMe+A18k+S+bou|QX?8eQ`{P-0vrm=uR;Y(bHV>d>Gen4LHILqcm_ z3peDMRE3JMA8wWgPkSthI^K<|8aal38qvIcEgLjHAFB0P#IfqP2y}L>=8eBR}Fm^V*mw2Q4+o=exP@*#=Zs zIqHh@neG)Vy%v4cB1!L}w9J>IqAo}CsqbFPrUVc@;~Ld7t_2IIG=15mT7Itrjq#2~ zqX*&nwZP>vso$6W!#` z-YZ}jhBwQku-Qc>TIMpn%_z~`^u4v3Skyf)KA}V{`dr!Q;3xK1TuGYdl}$sKF^9X!*a-R*Oq1#tLq!W)gO}{q`1HM;oh1-k4FU@8W(qe>P05$+ z`ud2&;4IW4vq8#2yA{G>OH=G+pS_jctJ*BqD$j-MI#avR+<>m-`H1@{3VgKYn2_Ih z0`2_1qUMRuzgj_V^*;5Ax_0s{_3tYR>|$i#c!F7)#`oVGmsD*M2?%930cBSI4Mj>P zTm&JmUrvDXlB%zeA_7$&ogjGK3>SOlV$ct{4)P0k)Kua%*fx9?)_fkvz<(G=F`KCp zE`0j*=FzH$^Y@iUI}MM2Hf#Yr@oQdlJMB5xe0$aGNk%tgex;0)NEuVYtLEvOt{}ti zL`o$K9HnnUnl*;DTGTNiwr&ydfDp@3Y)g5$pcY9l1-9g;yn6SBr_S9MV8Xl+RWgwb zXL%kZLE4#4rUO(Pj484!=`jy74tQxD0Zg>99vvQ}R$7~GW)-0DVJR@$5}drsp3IQG zlrJL}M{+SdWbrO@+g2BY^a}0VdQtuoml`jJ2s6GsG5D@(^$5pMi3$27psEIOe^n=*Nj|Ug7VXN0OrwMrRq&@sR&vdnsRlI%*$vfmJ~)s z^?lstAT$Ked`b&UZ@A6I<(uCHGZ9pLqNhD_g-kj*Sa#0%(=8j}4zd;@!o;#vJ+Bsd z4&K4RIP>6It9Ir)ey?M6Gi6@JzKNg;=jM=$)gs2#u_WhvuTRwm1x2^*!e%l&j02xz zYInQgI$_V7Epzf3*BU~gos}|EurFj8l}hsI(!5yX!~ECL%cnYMS-e<`AKDL%(G)62 zPU;uF1(~(YbH2444JGh58coXT>(*CdEwaFuyvB|%CULgVQesH$ znB`vk3BMP<-QauWOZ0W6xB5y7?tE5cisG|V;bhY^8+*BH1T0ZLbn&gi12|a9Oa%;I zxvaxX_xe3@ng%;4C?zPHQ1v%dbhjA6Sl7w<*)Nr#F{Ahzj}%n9c&!g5HVrlvUO&R2C)_$x6M9 zahficAbeHL2%jILO>Pq&RPPxl;i{K5#O*Yt15AORTCvkjNfJ)LrN4K{sY7>tGuTQ@ z^?N*+xssG&sfp0c$^vV*H)U1O!fTHk8;Q7@42MT@z6UTd^&DKSxVcC-1OLjl7m63& zBb&goU!hes(GF^yc!107bkV6Pr%;A-WWd@DK2;&=zyiK*0i^0@f?fh2c)4&DRSjrI zk!W^=l^JKlPW9US{*yo?_XT@T2Bx+Cm^+r{*5LVcKVw*ll3+)lkebA-4)o z8f5xHWOx0!FDSs4nv@o@>mxTQrOeKzj@5uL`d>mXSp|#{FE54EE_!KtQNq>-G(&5) ztz?xkqPU16A-8@-quJ|SU^ClZ?bJ2kCJPB|6L>NTDYBprw$WcwCH{B z5qlJ6wK_9sT@Kl6G|Q&$gsl@WT>hE;nDAbH#%f1ZwuOkvWLj{qV$m3LF423&l!^iV zhym*>R>Yyens++~6F5+uZQTCz9t~PEW+e?w)XF2g!^^%6k?@Jcu;MG0FG9!T+Gx{Z zK;31y@(J{!-$k4E{5#Sv(2DGy3EZQY}G_*z*G&CZ_J?m&Fg4IBrvPx1w z1zAb3k}6nT?E)HNCi%}aR^?)%w-DcpBR*tD(r_c{QU6V&2vU-j0;{TVDN6los%YJZ z5C(*ZE#kv-BvlGLDf9>EO#RH_jtolA)iRJ>tSfJpF!#DO+tk% zBAKCwVZwO^p)(Rhk2en$XLfWjQQ`ix>K}Ru6-sn8Ih6k&$$y`zQ}}4dj~o@9gX9_= z#~EkchJqd5$**l}~~6mOl(q#GMIcFg&XCKO;$w>!K14 zko1egAORiG{r|8qj*FsN>?7d`han?*MD#xe^)sOqj;o;hgdaVnBH$BM{_73?znS+R z*G2VHM!Jw6#<FfJ-J%-9AuDW$@mc-Eyk~F{Jbvt` zn;(%DbBDnKIYr~|I>ZTvbH@cxUyw%bp*)OSs}lwO^HTJ2M#u5QsPF0?Jv*OVPfdKv z+t$Z5P!~jzZ~Y!d#iP?S{?M_g%Ua0Q)WawbIx+2uYpcf(7Im%W=rAu4dSceo7RZh# zN38=RmwOJQE$qbPXIuO^E`wSeJKCx3Q76irp~QS#19dusEVCWPrKhK9{7cbIMg9U} TZiJi*F`$tkWLn) literal 0 HcmV?d00001 diff --git a/jiceberg_lib/gradle/wrapper/gradle-wrapper.properties b/jiceberg_lib/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..09523c0 --- /dev/null +++ b/jiceberg_lib/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/jiceberg_lib/gradlew b/jiceberg_lib/gradlew new file mode 100755 index 0000000..f5feea6 --- /dev/null +++ b/jiceberg_lib/gradlew @@ -0,0 +1,252 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s +' "$PWD" ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/jiceberg_lib/gradlew.bat b/jiceberg_lib/gradlew.bat new file mode 100644 index 0000000..9d21a21 --- /dev/null +++ b/jiceberg_lib/gradlew.bat @@ -0,0 +1,94 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/jiceberg_lib/scripts/copy_headers.sh b/jiceberg_lib/scripts/copy_headers.sh new file mode 100755 index 0000000..6870989 --- /dev/null +++ b/jiceberg_lib/scripts/copy_headers.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +# Make sure we are in the correct folder +working_dir=$(pwd) +if [ "${working_dir%jiceberg_lib}" != "${working_dir}" ]; then + cd scripts +else + if [ "${working_dir%jiceberg_lib/scripts}" == "${working_dir}" ]; then + echo "Please run from jiceberg_lib/ folder" + exit + fi +fi + +echo "Copying header files..." + +rm -rf ../../src/include/jiceberg_generated +mkdir ../../src/include/jiceberg_generated +cp ../app/build/native/nativeCompile/*.h ../../src/include/jiceberg_generated/ diff --git a/jiceberg_lib/settings.gradle b/jiceberg_lib/settings.gradle new file mode 100644 index 0000000..e863d10 --- /dev/null +++ b/jiceberg_lib/settings.gradle @@ -0,0 +1,14 @@ +/* + * This file was generated by the Gradle 'init' task. + * + * The settings file is used to specify which projects to include in your build. + * For more detailed information on multi-project builds, please refer to https://docs.gradle.org/8.9/userguide/multi_project_builds.html in the Gradle documentation. + */ + +plugins { + // Apply the foojay-resolver plugin to allow automatic download of JDKs + id 'org.gradle.toolchains.foojay-resolver-convention' version '0.8.0' +} + +rootProject.name = 'jiceberg_lib' +include('app') diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index af70ff0..f6bb8a6 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -406,7 +406,7 @@ static std::string json_to_string(yyjson_mut_doc *doc) { return json_str; } -ICAPITable ICAPI::CreateTable(const string &catalog, const string &internal, const string &schema, ICCredentials credentials, CreateTableInfo *table_info) { +ICAPITable ICAPI::CreateTable(const string &catalog, const string &internal, const string &schema, ICCredentials &credentials, CreateTableInfo *table_info) { std::unique_ptr dd(yyjson_mut_doc_new(NULL)); yyjson_mut_doc *doc = dd.get(); @@ -433,7 +433,7 @@ ICAPITable ICAPI::CreateTable(const string &catalog, const string &internal, con // Add column object to JSON yyjson_mut_val *col_obj = yyjson_mut_obj(doc); yyjson_mut_obj_add_int(doc, col_obj, "id", col.Oid()); - yyjson_mut_obj_add_bool(doc, col_obj, "required", true); + yyjson_mut_obj_add_bool(doc, col_obj, "required", false); yyjson_mut_obj_add_str(doc, col_obj, "name", column_names.back().c_str()); yyjson_mut_obj_add_str(doc, col_obj, "type", column_types.back().c_str()); yyjson_mut_arr_add_val(fields, col_obj); diff --git a/src/ic_create_table_as_op.cpp b/src/ic_create_table_as_op.cpp new file mode 100644 index 0000000..e10a486 --- /dev/null +++ b/src/ic_create_table_as_op.cpp @@ -0,0 +1,219 @@ + +#include "duckdb.hpp" +#include "duckdb/execution/physical_operator.hpp" +#include "duckdb/planner/logical_operator.hpp" +#include "duckdb/storage/data_table.hpp" +#include "parquet/arrow/writer.h" +#include "parquet/properties.h" + +#include "arrow/api.h" +#include "arrow/io/api.h" +#include "catalog_api.hpp" +#include "storage/ic_catalog.hpp" +#include "ic_create_table_as_op.hpp" +#include "jiceberg_generated/libjiceberg.h" +#include "jiceberg_generated/graal_isolate.h" + +namespace duckdb { + +template +void ProcessColumnData(const T *data_ptr, + arrow::NumericBuilder &builder, + const UnifiedVectorFormat &vdata, + std::shared_ptr &arr, + idx_t num_rows) { + + auto &validity = vdata.validity; + auto sel = vdata.sel; + for (idx_t row_idx = 0; row_idx < num_rows; row_idx++) { + idx_t source_idx = sel->get_index(row_idx); + if (validity.RowIsValid(source_idx)) { + if (!(builder.Append(data_ptr[source_idx])).ok()) { + throw std::runtime_error("Failed to append data"); + }; + } else { + if (!(builder.AppendNull()).ok()) { + throw std::runtime_error("Failed to append NULL"); + }; + } + } + + if (!(builder.Finish(&arr)).ok()) { + throw std::runtime_error("Failed to build Int32 Arrow array"); + } +} + +std::shared_ptr DataChunkToArrowTable(duckdb::DataChunk &chunk, ColumnList &columnList) { + namespace ar = arrow; + using namespace duckdb; + + int num_columns = chunk.ColumnCount(); + int64_t num_rows = chunk.size(); + + // Prepare to build schema (one Arrow field per DuckDB column) + std::vector> fields; + fields.reserve(num_columns); + + // Prepare a container for the final Arrow arrays (one per column) + std::vector> arrays; + arrays.reserve(num_columns); + + for (int col_idx = 0; col_idx < num_columns; col_idx++) { + auto &col_vector = chunk.data[col_idx]; + auto &duck_type = col_vector.GetType(); + string column_name = columnList.GetColumnNames()[col_idx]; + // We'll unify the vector so we can safely read data regardless of whether it's flat, constant, dictionary, etc. + UnifiedVectorFormat vdata; + col_vector.ToUnifiedFormat(num_rows, vdata); + // A selection vector mapping row_idx -> actual index in `vdata.data` + auto sel = vdata.sel; + // Validity (null) mask for each row + auto &validity = vdata.validity; + + auto metadata = std::make_shared(); + metadata->Append("PARQUET:field_id", std::to_string(col_idx+1)); + + // TODO: Handle other types as needed: LogicalTypeId::FLOAT, ::BOOLEAN, ::DATE, etc. + switch (duck_type.id()) { + case LogicalTypeId::INTEGER: { + fields.push_back( + ar::field(column_name, ar::int32())->WithMetadata(metadata)); + auto data_ptr = reinterpret_cast(vdata.data); + ar::Int32Builder builder; + std::shared_ptr arr; + ProcessColumnData(data_ptr, builder, vdata, arr, num_rows); + arrays.push_back(arr); + break; + } + + case LogicalTypeId::BIGINT: { + fields.push_back(ar::field(column_name, ar::int64())->WithMetadata(metadata)); + auto data_ptr = reinterpret_cast(vdata.data); + ar::Int64Builder builder; + std::shared_ptr arr; + ProcessColumnData(data_ptr, builder, vdata, arr, num_rows); + arrays.push_back(arr); + break; + } + + case LogicalTypeId::DOUBLE: { + fields.push_back(ar::field(column_name, ar::float64())->WithMetadata(metadata)); + auto data_ptr = reinterpret_cast(vdata.data); + ar::DoubleBuilder builder; + std::shared_ptr arr; + ProcessColumnData(data_ptr, builder, vdata, arr, num_rows); + arrays.push_back(arr); + break; + } + + case LogicalTypeId::VARCHAR: { + fields.push_back(ar::field(column_name, ar::utf8())->WithMetadata(metadata)); + ar::StringBuilder builder; + for (idx_t row_idx = 0; row_idx < num_rows; row_idx++) { + idx_t source_idx = sel->get_index(row_idx); + if (validity.RowIsValid(source_idx)) { + auto val = col_vector.GetValue(source_idx); + builder.Append(val.ToString()); + } else { + builder.AppendNull(); + } + } + std::shared_ptr arr; + if (!(builder.Finish(&arr)).ok()) { + throw std::runtime_error("Failed to build Int32 Arrow array"); + } + arrays.push_back(arr); + break; + } + + default: + throw std::runtime_error("Unsupported type: " + duck_type.ToString() + " for column " + column_name); + } + } + + // Build a final Arrow schema and table + auto schema = std::make_shared(fields); + return ar::Table::Make(schema, arrays, 1); +} + +OperatorResultType ICCreateTableAsOp::Execute(ExecutionContext &context, + DataChunk &input, + DataChunk &chunk, + GlobalOperatorState &gstate, + OperatorState &state) const { + auto *table_info = dynamic_cast(info->base.get()); + + // Create arrow table to write to parquet file + auto arrow_table = DataChunkToArrowTable(input, table_info->columns); + + // Create a file output stream + const std::string datafile_filename = + catalog_internal_name + "." + table_info->schema + "." + table_info->table + ".parquet"; + auto open_result = arrow::io::FileOutputStream::Open(datafile_filename); + + if (!open_result.ok()) { + throw std::runtime_error("Failed to open file: " + open_result.status().ToString()); + } + + std::shared_ptr datafile = *open_result; + parquet::WriterProperties::Builder builder; + builder.compression(parquet::Compression::SNAPPY); + builder.version(parquet::ParquetVersion::PARQUET_1_0); + std::shared_ptr writer_props = builder.build(); + arrow::Status write_status = parquet::arrow::WriteTable( + *arrow_table, + arrow::default_memory_pool(), + datafile, + 1024, // TODO: Is this a good chunk_size? + writer_props + ); + + if (!write_status.ok()) { + throw std::runtime_error("Error writing to Parquet: " + write_status.ToString()); + } + + // Create the table at the destination + ICAPI::CreateTable(table_info->catalog, catalog_internal_name, table_info->schema, table_credentials, table_info); + + // Initialize the GraalVM isolate + graal_isolate_t* isolate = nullptr; + graal_isolatethread_t* thread = nullptr; + if (graal_create_isolate(nullptr, &isolate, &thread) != 0) { + throw std::runtime_error("Failed to create GraalVM isolate"); + } + + string creds = table_credentials.client_id + ":" + table_credentials.client_secret; + int result = -1; + try { + result = append_to_table( + thread, + const_cast(table_credentials.endpoint.c_str()), + const_cast(creds.c_str()), + const_cast(catalog_internal_name.c_str()), + const_cast(table_info->schema.c_str()), + const_cast(table_info->table.c_str()), + const_cast(datafile_filename.c_str()), + chunk.size()); + + } catch (...) { + ICAPI::DropTable(table_info->catalog, catalog_internal_name, table_info->schema, table_info->table, table_credentials); + } + + // Clean up the GraalVM isolate + if (graal_detach_thread(thread) != 0) { + throw std::runtime_error("Failed to detach GraalVM thread"); + } + + if (result < 0) { + throw std::runtime_error("Failed to append to table"); + } + + // TODO: Add the new table to the catalog + + // Set output chunk to empty + chunk.SetCardinality(0); + + return duckdb::OperatorResultType::FINISHED; +} + +} // namespace duckdb \ No newline at end of file diff --git a/src/iceberg_extension.cpp b/src/iceberg_extension.cpp index 114b3ff..2838ff4 100644 --- a/src/iceberg_extension.cpp +++ b/src/iceberg_extension.cpp @@ -111,9 +111,21 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in const auto &kv_secret = dynamic_cast(*secret_entry->secret); string new_connection_info; + Value client_id = kv_secret.TryGetValue("client_id"); + if (client_id.IsNull()) { + throw std::runtime_error("CLIENT_ID is blank"); + } + credentials.client_id = client_id.ToString(); + + Value client_secret = kv_secret.TryGetValue("client_secret"); + if (client_secret.IsNull()) { + throw std::runtime_error("CLIENT_SECRET is blank"); + } + credentials.client_secret = client_secret.ToString(); + Value token_val = kv_secret.TryGetValue("token"); if (token_val.IsNull()) { - throw std::runtime_error("Token is blank"); + throw std::runtime_error("TOKEN is blank"); } credentials.token = token_val.ToString(); diff --git a/src/include/catalog_api.hpp b/src/include/catalog_api.hpp index 18613dd..10b708a 100644 --- a/src/include/catalog_api.hpp +++ b/src/include/catalog_api.hpp @@ -53,7 +53,7 @@ class ICAPI { static string GetToken(string id, string secret, string endpoint); static ICAPISchema CreateSchema(const string &catalog, const string &internal, const string &schema, ICCredentials credentials); static void DropSchema(const string &internal, const string &schema, ICCredentials credentials); - static ICAPITable CreateTable(const string &catalog, const string &internal, const string &schema, ICCredentials credentials, CreateTableInfo *table_info); + static ICAPITable CreateTable(const string &catalog, const string &internal, const string &schema, ICCredentials &credentials, CreateTableInfo *table_info); static void DropTable(const string &catalog, const string &internal, const string &schema, string &table_name, ICCredentials credentials); }; diff --git a/src/include/ic_create_table_as_op.hpp b/src/include/ic_create_table_as_op.hpp new file mode 100644 index 0000000..a141a9c --- /dev/null +++ b/src/include/ic_create_table_as_op.hpp @@ -0,0 +1,35 @@ + +#pragma once + +#include "duckdb/execution/physical_operator.hpp" +#include "duckdb/execution/operator/schema/physical_create_table.hpp" +#include "duckdb/planner/parsed_data/bound_create_table_info.hpp" + +namespace duckdb { + +class ICCreateTableAsOp : public PhysicalOperator { +public: + ICCreateTableAsOp(vector types, + unique_ptr info, + idx_t estimated_cardinality, + string &internal_name, + ICCredentials &credentials) + : PhysicalOperator(PhysicalOperatorType::CREATE_TABLE_AS, types, estimated_cardinality), + info(std::move(info)), + catalog_internal_name(internal_name), + table_credentials(credentials) {} + + // Override the Execute method + OperatorResultType Execute(ExecutionContext &context, + DataChunk &input, + DataChunk &chunk, + GlobalOperatorState &gstate, + OperatorState &state) const override; + +private: + unique_ptr info; + string &catalog_internal_name; + ICCredentials &table_credentials; +}; + +} \ No newline at end of file diff --git a/src/storage/ic_catalog.cpp b/src/storage/ic_catalog.cpp index a920c29..76c1f10 100644 --- a/src/storage/ic_catalog.cpp +++ b/src/storage/ic_catalog.cpp @@ -4,7 +4,10 @@ #include "duckdb/storage/database_size.hpp" #include "duckdb/parser/parsed_data/drop_info.hpp" #include "duckdb/parser/parsed_data/create_schema_info.hpp" +#include "duckdb/planner/operator/logical_create_table.hpp" #include "duckdb/main/attached_database.hpp" +#include "duckdb/execution/operator/schema/physical_create_table.hpp" +#include "ic_create_table_as_op.hpp" namespace duckdb { @@ -80,10 +83,25 @@ unique_ptr ICCatalog::PlanInsert(ClientContext &context, Logic unique_ptr plan) { throw NotImplementedException("ICCatalog PlanInsert"); } -unique_ptr ICCatalog::PlanCreateTableAs(ClientContext &context, LogicalCreateTable &op, - unique_ptr plan) { - throw NotImplementedException("ICCatalog PlanCreateTableAs"); + +unique_ptr ICCatalog::PlanCreateTableAs( + ClientContext &context, LogicalCreateTable &op, unique_ptr plan) { + + auto create_table_as = make_uniq( + op.types, + std::move(op.info), + op.estimated_cardinality, + internal_name, + credentials); + + // If your operator has children (input plan), attach them + if (plan) { + create_table_as->children.push_back(std::move(plan)); + } + + return create_table_as; } + unique_ptr ICCatalog::PlanDelete(ClientContext &context, LogicalDelete &op, unique_ptr plan) { throw NotImplementedException("ICCatalog PlanDelete"); From 6f70f3af3ec8bbb45040228fffc5a143c7d25e8f Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Wed, 5 Feb 2025 01:10:54 -0500 Subject: [PATCH 18/26] Rename for clarity --- src/include/ic_create_table_as_op.hpp | 3 +++ src/include/storage/ic_table_set.hpp | 4 ++-- src/storage/ic_catalog.cpp | 1 + src/storage/ic_catalog_set.cpp | 15 ++++----------- src/storage/ic_schema_set.cpp | 4 ++-- src/storage/ic_table_set.cpp | 26 +++++++++++++++++++------- 6 files changed, 31 insertions(+), 22 deletions(-) diff --git a/src/include/ic_create_table_as_op.hpp b/src/include/ic_create_table_as_op.hpp index a141a9c..90e5d0f 100644 --- a/src/include/ic_create_table_as_op.hpp +++ b/src/include/ic_create_table_as_op.hpp @@ -13,10 +13,12 @@ class ICCreateTableAsOp : public PhysicalOperator { unique_ptr info, idx_t estimated_cardinality, string &internal_name, + ICCatalogSet &catalog_set, ICCredentials &credentials) : PhysicalOperator(PhysicalOperatorType::CREATE_TABLE_AS, types, estimated_cardinality), info(std::move(info)), catalog_internal_name(internal_name), + catalog_set(catalog_set), table_credentials(credentials) {} // Override the Execute method @@ -29,6 +31,7 @@ class ICCreateTableAsOp : public PhysicalOperator { private: unique_ptr info; string &catalog_internal_name; + ICCatalogSet &catalog_set; ICCredentials &table_credentials; }; diff --git a/src/include/storage/ic_table_set.hpp b/src/include/storage/ic_table_set.hpp index f1746db..116ce49 100644 --- a/src/include/storage/ic_table_set.hpp +++ b/src/include/storage/ic_table_set.hpp @@ -14,7 +14,7 @@ class ICInSchemaSet : public ICCatalogSet { public: ICInSchemaSet(ICSchemaEntry &schema); - optional_ptr CreateEntry(unique_ptr entry) override; + optional_ptr AddEntry(unique_ptr entry) override; protected: ICSchemaEntry &schema; @@ -44,7 +44,7 @@ class ICTableSet : public ICInSchemaSet { static void AddColumn(ClientContext &context, ICResult &result, ICTableInfo &table_info, idx_t column_offset = 0); private: - unique_ptr _CreateCatalogEntry(ClientContext &context, ICAPITable table); + unique_ptr CreateTableEntry(ClientContext &context, ICAPITable table); }; diff --git a/src/storage/ic_catalog.cpp b/src/storage/ic_catalog.cpp index 76c1f10..e74e25e 100644 --- a/src/storage/ic_catalog.cpp +++ b/src/storage/ic_catalog.cpp @@ -92,6 +92,7 @@ unique_ptr ICCatalog::PlanCreateTableAs( std::move(op.info), op.estimated_cardinality, internal_name, + schemas, credentials); // If your operator has children (input plan), attach them diff --git a/src/storage/ic_catalog_set.cpp b/src/storage/ic_catalog_set.cpp index 6297623..fb54255 100644 --- a/src/storage/ic_catalog_set.cpp +++ b/src/storage/ic_catalog_set.cpp @@ -37,7 +37,10 @@ void ICCatalogSet::Scan(ClientContext &context, const std::function ICCatalogSet::CreateEntry(unique_ptr entry) { +optional_ptr ICCatalogSet::AddEntry(unique_ptr entry) { + // Also ICSchemaSet::AddEntry + std::cout << "ICCatalogSet::AddEntry" << std::endl; + lock_guard l(entry_lock); auto result = entry.get(); if (result->name.empty()) { @@ -51,14 +54,4 @@ void ICCatalogSet::ClearEntries() { entries.clear(); } -ICInSchemaSet::ICInSchemaSet(ICSchemaEntry &schema) : ICCatalogSet(schema.ParentCatalog()), schema(schema) { -} - -optional_ptr ICInSchemaSet::CreateEntry(unique_ptr entry) { - if (!entry->internal) { - entry->internal = schema.internal; - } - return ICCatalogSet::CreateEntry(std::move(entry)); -} - } // namespace duckdb diff --git a/src/storage/ic_schema_set.cpp b/src/storage/ic_schema_set.cpp index b35b624..28db677 100644 --- a/src/storage/ic_schema_set.cpp +++ b/src/storage/ic_schema_set.cpp @@ -31,7 +31,7 @@ void ICSchemaSet::LoadEntries(ClientContext &context) { info.internal = IsInternalTable(schema.catalog_name, schema.schema_name); auto schema_entry = make_uniq(catalog, info); schema_entry->schema_data = make_uniq(schema); - CreateEntry(std::move(schema_entry)); + AddEntry(std::move(schema_entry)); } } @@ -44,7 +44,7 @@ optional_ptr ICSchemaSet::CreateSchema(ClientContext &context, Cre auto schema = ICAPI::CreateSchema(catalog.GetName(), ic_catalog.internal_name, info.schema, ic_catalog.credentials); auto schema_entry = make_uniq(catalog, info); schema_entry->schema_data = make_uniq(schema); - return CreateEntry(std::move(schema_entry)); + return AddEntry(std::move(schema_entry)); } void ICSchemaSet::DropSchema(ClientContext &context, DropInfo &info) { diff --git a/src/storage/ic_table_set.cpp b/src/storage/ic_table_set.cpp index 6e1f284..7203e4b 100644 --- a/src/storage/ic_table_set.cpp +++ b/src/storage/ic_table_set.cpp @@ -25,7 +25,7 @@ static ColumnDefinition CreateColumnDefinition(ClientContext &context, ICAPIColu return {coldef.name, ICUtils::TypeToLogicalType(context, coldef.type_text)}; } -unique_ptr ICTableSet::_CreateCatalogEntry(ClientContext &context, ICAPITable table) { +unique_ptr ICTableSet::CreateTableEntry(ClientContext &context, ICAPITable table) { D_ASSERT(schema.name == table.schema_name); CreateTableInfo info; info.table = table.name; @@ -47,7 +47,7 @@ void ICTableSet::FillEntry(ClientContext &context, unique_ptr &ent auto &ic_catalog = catalog.Cast(); auto table = ICAPI::GetTable(catalog.GetName(), catalog.GetDBPath(), schema.name, entry->name, ic_catalog.credentials); - entry = _CreateCatalogEntry(context, table); + entry = CreateTableEntry(context, table); } void ICTableSet::LoadEntries(ClientContext &context) { @@ -60,8 +60,8 @@ void ICTableSet::LoadEntries(ClientContext &context) { auto tables = ICAPI::GetTables(catalog.GetName(), catalog.GetDBPath(), schema.name, ic_catalog.credentials); for (auto &table : tables) { - auto entry = _CreateCatalogEntry(context, table); - CreateEntry(std::move(entry)); + auto entry = CreateTableEntry(context, table); + AddEntry(std::move(entry)); } } @@ -69,7 +69,7 @@ optional_ptr ICTableSet::RefreshTable(ClientContext &context, cons auto table_info = GetTableInfo(context, schema, table_name); auto table_entry = make_uniq(catalog, schema, *table_info); auto table_ptr = table_entry.get(); - CreateEntry(std::move(table_entry)); + AddEntry(std::move(table_entry)); return table_ptr; } @@ -82,8 +82,8 @@ optional_ptr ICTableSet::CreateTable(ClientContext &context, Bound auto &ic_catalog = catalog.Cast(); auto *table_info = dynamic_cast(info.base.get()); auto table = ICAPI::CreateTable(catalog.GetName(), ic_catalog.internal_name, schema.name, ic_catalog.credentials, table_info); - auto entry = _CreateCatalogEntry(context, table); - return CreateEntry(std::move(entry)); + auto entry = CreateTableEntry(context, table); + return AddEntry(std::move(entry)); } void ICTableSet::DropTable(ClientContext &context, DropInfo &info) { @@ -111,4 +111,16 @@ void ICTableSet::AlterTable(ClientContext &context, AlterTableInfo &alter) { throw NotImplementedException("ICTableSet::AlterTable"); } +ICInSchemaSet::ICInSchemaSet(ICSchemaEntry &schema) : ICCatalogSet(schema.ParentCatalog()), schema(schema) { +} + +optional_ptr ICInSchemaSet::AddEntry(unique_ptr entry) { + std::cout << "ICInSchemaSet::CreateEntry" << std::endl; + + if (!entry->internal) { + entry->internal = schema.internal; + } + return ICCatalogSet::AddEntry(std::move(entry)); +} + } // namespace duckdb From da54cc2904f35ab4531a81fae3371d3dd99cae39 Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Wed, 5 Feb 2025 22:41:27 -0500 Subject: [PATCH 19/26] Complete CTAS functionality --- jiceberg_lib/app/build.gradle | 12 +-- .../main/java/com/fivetran/iceberg/App.java | 95 +++++++------------ .../native-image-config/reflect-config.json | 31 +++--- .../native-image-config/resource-config.json | 6 -- src/catalog_api.cpp | 8 +- src/catalog_utils.cpp | 8 +- src/ic_create_table_as_op.cpp | 21 ++-- src/include/catalog_api.hpp | 2 +- src/include/catalog_utils.hpp | 2 +- src/include/ic_create_table_as_op.hpp | 9 +- src/include/storage/ic_catalog_set.hpp | 2 +- src/include/storage/ic_table_set.hpp | 2 +- src/storage/ic_catalog.cpp | 8 +- src/storage/ic_catalog_set.cpp | 3 - src/storage/ic_table_entry.cpp | 8 +- src/storage/ic_table_set.cpp | 19 ++-- 16 files changed, 114 insertions(+), 122 deletions(-) diff --git a/jiceberg_lib/app/build.gradle b/jiceberg_lib/app/build.gradle index 889d8ea..6484007 100644 --- a/jiceberg_lib/app/build.gradle +++ b/jiceberg_lib/app/build.gradle @@ -16,8 +16,6 @@ repositories { } dependencies { - implementation libs.guava - implementation 'org.apache.iceberg:iceberg-core:1.7.1' implementation 'org.apache.iceberg:iceberg-api:1.7.1' implementation 'org.apache.iceberg:iceberg-aws:1.7.1' @@ -32,6 +30,8 @@ dependencies { implementation 'software.amazon.awssdk:regions:2.30.11' compileOnly 'org.graalvm.sdk:graal-sdk:22.0.0' + implementation 'org.slf4j:slf4j-nop:2.0.9' + implementation 'org.slf4j:log4j-over-slf4j:2.0.9' } application { @@ -47,13 +47,13 @@ graalvmNative { buildArgs.add('--shared') buildArgs.add('--initialize-at-run-time=org.apache.log4j') buildArgs.add('--initialize-at-run-time=org.slf4j') - - // Add these for Iceberg buildArgs.add('--initialize-at-run-time=com.github.benmanes.caffeine.cache.LocalLoadingCache') buildArgs.add('--initialize-at-run-time=org.apache.iceberg.util.Pair') + buildArgs.add('--initialize-at-run-time=org.apache.logging.log4j.util.ProviderUtil') + buildArgs.add('--initialize-at-run-time=org.apache.logging.log4j.core.impl.Log4jContextFactory') + buildArgs.add('--initialize-at-run-time=org.apache.commons.logging.LogFactory') + buildArgs.add('--initialize-at-build-time=org.slf4j,org.apache.logging.slf4j') buildArgs.add('-H:+ReportExceptionStackTraces') - - // If you need SSL support buildArgs.add('--enable-https') metadataRepository { diff --git a/jiceberg_lib/app/src/main/java/com/fivetran/iceberg/App.java b/jiceberg_lib/app/src/main/java/com/fivetran/iceberg/App.java index 5405e92..f69e56d 100644 --- a/jiceberg_lib/app/src/main/java/com/fivetran/iceberg/App.java +++ b/jiceberg_lib/app/src/main/java/com/fivetran/iceberg/App.java @@ -9,17 +9,19 @@ import org.graalvm.nativeimage.c.function.CEntryPoint; import org.graalvm.nativeimage.c.type.CCharPointer; import org.graalvm.nativeimage.c.type.CTypeConversion; +import org.graalvm.word.WordFactory; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; +import java.util.UUID; public class App { @CEntryPoint(name = "append_to_table") - public static int appendToTable( + public static CCharPointer appendToTable( IsolateThread thread, CCharPointer uri, CCharPointer creds, @@ -29,7 +31,7 @@ public static int appendToTable( CCharPointer filename, int numRecords) { try { - return append( + String newMetadataLoc = append( CTypeConversion.toJavaString(uri), CTypeConversion.toJavaString(creds), CTypeConversion.toJavaString(warehouse), @@ -37,10 +39,13 @@ public static int appendToTable( CTypeConversion.toJavaString(tableName), CTypeConversion.toJavaString(filename), numRecords); + try (CTypeConversion.CCharPointerHolder holder = CTypeConversion.toCString(newMetadataLoc)) { + return holder.get(); + } } catch (IOException e) { - return -1; + return WordFactory.nullPointer(); } finally { - // Delete the local file + // Delete the local file no matter what Path localPath = Paths.get(CTypeConversion.toJavaString(filename)); try { java.nio.file.Files.deleteIfExists(localPath); @@ -52,20 +57,21 @@ public static int appendToTable( public static void main(String[] args) { try { - append("https://polaris.fivetran.com/api/catalog", - "client_id:client_secret", - "catalog", - "schema", - "new_table", - "datafile.parquet", - 1 - ); + System.out.println( + append("https://polaris.fivetran.com/api/catalog", + "client_id:client_secret", + "catalog", + "schema", + "new_table", + "datafile.parquet", + 1 + )); } catch (IOException e) { throw new RuntimeException(e); } } - static int append( + static String append( String uri, String creds, String warehouse, @@ -104,58 +110,27 @@ static int append( .withFormat(FileFormat.PARQUET) .build(); + // Force a new metadata commit + table.updateProperties() + .set("last-modified-time", String.valueOf(System.currentTimeMillis())) + .commit(); + + // Optionally, you can try to expire old metadata + table.expireSnapshots() + .expireOlderThan(System.currentTimeMillis()) + .commit(); + // Append the data file table.newAppend() .appendFile(dataFile) + .set("write.format.default", "parquet") + .set("format-version", "2") + .set("write.metadata.compression-codec", "none") + .set("snapshot-id", UUID.randomUUID().toString()) // Force new snapshot .commit(); - } - -// Snapshot snapshot = table.currentSnapshot(); -// FileIO io = table.io(); -// // For newer Iceberg versions, use `allManifests(io)`: -// for (ManifestFile mf : snapshot.allManifests(io)) { -// System.out.println("Manifest path: " + mf.path() -// + ", addedFilesCount=" + mf.addedFilesCount()); -// } -// -// Iterable newlyAddedFiles = snapshot.addedDataFiles(table.io()); -// for (DataFile df : newlyAddedFiles) { -// System.out.println("Added file path: " + df.path()); -// } -// -// table.newScan() -// .planFiles() -// .forEach(task -> { -// System.out.println("Active data file path: " + task.file().path()); -// System.out.println("Row count: " + task.file().recordCount()); -// }); -// Schema schema = table.schema(); -// // Iterate over each file in the table scan -// for (FileScanTask task : table.newScan().planFiles()) { -// // Print file information -// DataFile dataFile = task.file(); -// -// // Convert the DataFile to an Iceberg InputFile -// InputFile inputFile = table.io().newInputFile(dataFile.path().toString()); -// -// // Build a Parquet reader using Iceberg's Parquet API -// try (CloseableIterable reader = Parquet.read(inputFile) -// .project(schema) // Project the full schema or a subset -// .createReaderFunc(messageType -> GenericParquetReaders.buildReader(schema, messageType)) -// .build()) { -// -// // Iterate over the records and process them -// for (Record record : reader) { -// System.out.println("Record: " + record); -// } -// } catch (IOException e) { -// System.err.println("Failed to read data from file: " + dataFile.path()); -// e.printStackTrace(); -// } -// } - - return 0; + return ((BaseTable) table).operations().current().metadataFileLocation(); + } } } } diff --git a/jiceberg_lib/app/src/main/resources/META-INF/native-image-config/reflect-config.json b/jiceberg_lib/app/src/main/resources/META-INF/native-image-config/reflect-config.json index be86c15..7fd21c4 100644 --- a/jiceberg_lib/app/src/main/resources/META-INF/native-image-config/reflect-config.json +++ b/jiceberg_lib/app/src/main/resources/META-INF/native-image-config/reflect-config.json @@ -147,8 +147,7 @@ }, { "name":"java.lang.Thread", - "fields":[{"name":"threadLocalRandomProbe"}], - "methods":[{"name":"getContextClassLoader","parameterTypes":[] }] + "fields":[{"name":"threadLocalRandomProbe"}] }, { "name":"java.security.AlgorithmParametersSpi" @@ -227,6 +226,14 @@ "name":"org.apache.http.client.config.RequestConfig$Builder", "methods":[{"name":"setNormalizeUri","parameterTypes":["boolean"] }] }, +{ + "name":"org.apache.iceberg.GenericDataFile", + "methods":[{"name":"","parameterTypes":["org.apache.iceberg.types.Types$StructType"] }] +}, +{ + "name":"org.apache.iceberg.GenericManifestEntry", + "methods":[{"name":"","parameterTypes":["org.apache.iceberg.types.Types$StructType"] }] +}, { "name":"org.apache.iceberg.GenericManifestFile", "methods":[{"name":"","parameterTypes":["org.apache.avro.Schema"] }] @@ -235,6 +242,10 @@ "name":"org.apache.iceberg.GenericPartitionFieldSummary", "methods":[{"name":"","parameterTypes":["org.apache.avro.Schema"] }] }, +{ + "name":"org.apache.iceberg.PartitionData", + "methods":[{"name":"","parameterTypes":["org.apache.iceberg.types.Types$StructType"] }] +}, { "name":"org.apache.iceberg.aws.ApacheHttpClientConfigurations", "methods":[{"name":"create","parameterTypes":["java.util.Map"] }] @@ -252,27 +263,23 @@ "methods":[{"name":"","parameterTypes":["java.lang.String"] }] }, { - "name":"org.apache.iceberg.util.Pair$1" + "name":"org.apache.iceberg.relocated.com.google.common.util.concurrent.AbstractFuture", + "fields":[{"name":"listeners"}, {"name":"value"}, {"name":"waiters"}] }, { - "name":"org.apache.log4j.Category" + "name":"org.apache.iceberg.relocated.com.google.common.util.concurrent.AbstractFuture$Waiter", + "fields":[{"name":"next"}, {"name":"thread"}] }, { - "name":"org.apache.log4j.CategoryKey" + "name":"org.apache.iceberg.util.Pair$1" }, { "name":"org.apache.log4j.Level", "fields":[{"name":"TRACE"}] }, -{ - "name":"org.apache.log4j.Logger" -}, { "name":"org.apache.log4j.Priority" }, -{ - "name":"org.apache.log4j.helpers.Loader" -}, { "name":"org.brotli.dec.BrotliInputStream" }, @@ -281,7 +288,7 @@ }, { "name":"sun.misc.Unsafe", - "fields":[{"name":"theUnsafe"}] + "allDeclaredFields":true }, { "name":"sun.security.pkcs12.PKCS12KeyStore", diff --git a/jiceberg_lib/app/src/main/resources/META-INF/native-image-config/resource-config.json b/jiceberg_lib/app/src/main/resources/META-INF/native-image-config/resource-config.json index e3f4ed2..040692b 100644 --- a/jiceberg_lib/app/src/main/resources/META-INF/native-image-config/resource-config.json +++ b/jiceberg_lib/app/src/main/resources/META-INF/native-image-config/resource-config.json @@ -26,10 +26,6 @@ "pattern":"\\Qhadoop-site.xml\\E" }, { "pattern":"\\Qiceberg-build.properties\\E" - }, { - "pattern":"\\Qlog4j.properties\\E" - }, { - "pattern":"\\Qlog4j.xml\\E" }, { "pattern":"\\Qmozilla/public-suffix-list.txt\\E" }, { @@ -38,8 +34,6 @@ "pattern":"\\Qorg/apache/hc/client5/http/psl/org/publicsuffix/list/effective_tld_names.dat\\E" }, { "pattern":"\\Qorg/apache/hc/client5/version.properties\\E" - }, { - "pattern":"\\Qorg/slf4j/impl/StaticLoggerBinder.class\\E" }, { "pattern":"\\Qorg/xerial/snappy/VERSION\\E" }, { diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index f6bb8a6..c0a4029 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -273,7 +273,7 @@ string ICAPI::GetToken(string id, string secret, string endpoint) { } static void populateTableMetadata(ICAPITable &table, yyjson_val *metadata_root) { - table.storage_location = TryGetStrFromObject(metadata_root, "metadata-location"); + table.metadata_location = TryGetStrFromObject(metadata_root, "metadata-location"); auto *metadata = yyjson_obj_get(metadata_root, "metadata"); //table_result.table_id = TryGetStrFromObject(metadata, "table-uuid"); @@ -442,6 +442,12 @@ ICAPITable ICAPI::CreateTable(const string &catalog, const string &internal, con yyjson_mut_val *props = yyjson_mut_obj(doc); yyjson_mut_obj_add_val(doc, rr, "properties", props); yyjson_mut_obj_add_str(doc, props, "write.parquet.compression-codec", "snappy"); + yyjson_mut_obj_add_str(doc, props, "format-version", "2"); // Important for newer Iceberg features + yyjson_mut_obj_add_str(doc, props, "write.metadata.compression-codec", "none"); // Ensure metadata writes work + yyjson_mut_obj_add_str(doc, props, "write.metadata.metrics.default", "full"); // Complete metadata + yyjson_mut_obj_add_str(doc, props, "write.metadata.delete-after-commit.enabled", "true"); + yyjson_mut_obj_add_str(doc, props, "write.metadata.previous-versions-max", "1"); + string post_data = json_to_string(doc); struct curl_slist *extra_headers = NULL; diff --git a/src/catalog_utils.cpp b/src/catalog_utils.cpp index 075392f..906bf92 100644 --- a/src/catalog_utils.cpp +++ b/src/catalog_utils.cpp @@ -71,7 +71,7 @@ string ICUtils::TypeToString(const LogicalType &input) { } } -LogicalType ICUtils::TypeToLogicalType(ClientContext &context, const string &type_text) { +LogicalType ICUtils::TypeToLogicalType(const string &type_text) { if (type_text == "tinyint") { return LogicalType::TINYINT; } else if (type_text == "smallint") { @@ -112,7 +112,7 @@ LogicalType ICUtils::TypeToLogicalType(ClientContext &context, const string &typ size_t type_end = type_text.rfind('>'); // find last, to deal with nested if (type_end != string::npos) { auto child_type_str = type_text.substr(6, type_end - 6); - auto child_type = ICUtils::TypeToLogicalType(context, child_type_str); + auto child_type = ICUtils::TypeToLogicalType(child_type_str); return LogicalType::LIST(child_type); } } else if (type_text.find("map<") == 0) { @@ -137,7 +137,7 @@ LogicalType ICUtils::TypeToLogicalType(ClientContext &context, const string &typ } } auto child_str = type_text.substr(cur, next_sep - cur); - auto child_type = ICUtils::TypeToLogicalType(context, child_str); + auto child_type = ICUtils::TypeToLogicalType(child_str); key_val.push_back(child_type); if (next_sep == type_end) { break; @@ -175,7 +175,7 @@ LogicalType ICUtils::TypeToLogicalType(ClientContext &context, const string &typ throw NotImplementedException("Invalid struct child type specifier: %s", child_str); } auto child_name = child_str.substr(0, type_sep); - auto child_type = ICUtils::TypeToLogicalType(context, child_str.substr(type_sep + 1, string::npos)); + auto child_type = ICUtils::TypeToLogicalType(child_str.substr(type_sep + 1, string::npos)); children.push_back({child_name, child_type}); if (next_sep == type_end) { break; diff --git a/src/ic_create_table_as_op.cpp b/src/ic_create_table_as_op.cpp index e10a486..7ad4aed 100644 --- a/src/ic_create_table_as_op.cpp +++ b/src/ic_create_table_as_op.cpp @@ -1,6 +1,7 @@ #include "duckdb.hpp" #include "duckdb/execution/physical_operator.hpp" +#include "duckdb/parser/parsed_data/drop_info.hpp" #include "duckdb/planner/logical_operator.hpp" #include "duckdb/storage/data_table.hpp" #include "parquet/arrow/writer.h" @@ -8,8 +9,6 @@ #include "arrow/api.h" #include "arrow/io/api.h" -#include "catalog_api.hpp" -#include "storage/ic_catalog.hpp" #include "ic_create_table_as_op.hpp" #include "jiceberg_generated/libjiceberg.h" #include "jiceberg_generated/graal_isolate.h" @@ -173,7 +172,8 @@ OperatorResultType ICCreateTableAsOp::Execute(ExecutionContext &context, } // Create the table at the destination - ICAPI::CreateTable(table_info->catalog, catalog_internal_name, table_info->schema, table_credentials, table_info); + auto transaction = schemaEntry->catalog.GetCatalogTransaction(context.client); + auto entry = schemaEntry->CreateTable(transaction, *info); // Initialize the GraalVM isolate graal_isolate_t* isolate = nullptr; @@ -183,7 +183,7 @@ OperatorResultType ICCreateTableAsOp::Execute(ExecutionContext &context, } string creds = table_credentials.client_id + ":" + table_credentials.client_secret; - int result = -1; + char *result = nullptr; try { result = append_to_table( thread, @@ -196,7 +196,12 @@ OperatorResultType ICCreateTableAsOp::Execute(ExecutionContext &context, chunk.size()); } catch (...) { - ICAPI::DropTable(table_info->catalog, catalog_internal_name, table_info->schema, table_info->table, table_credentials); + DropInfo drop_info; + drop_info.type = CatalogType::TABLE_ENTRY; + drop_info.catalog = info->base->catalog; + drop_info.schema = table_info->schema; + drop_info.name = table_info->table; + schemaEntry->DropEntry(context.client, drop_info); } // Clean up the GraalVM isolate @@ -204,11 +209,13 @@ OperatorResultType ICCreateTableAsOp::Execute(ExecutionContext &context, throw std::runtime_error("Failed to detach GraalVM thread"); } - if (result < 0) { + if (result == nullptr) { throw std::runtime_error("Failed to append to table"); } - // TODO: Add the new table to the catalog + // Update the table entry with the new metadata location + auto table_entry = dynamic_cast(entry.get()); + table_entry->table_data->metadata_location = result; // Set output chunk to empty chunk.SetCardinality(0); diff --git a/src/include/catalog_api.hpp b/src/include/catalog_api.hpp index 10b708a..ad4bfdf 100644 --- a/src/include/catalog_api.hpp +++ b/src/include/catalog_api.hpp @@ -23,7 +23,7 @@ struct ICAPITable { string schema_name; string table_type; string data_source_format; - string storage_location; + string metadata_location; vector columns; }; diff --git a/src/include/catalog_utils.hpp b/src/include/catalog_utils.hpp index 9814272..75186d3 100644 --- a/src/include/catalog_utils.hpp +++ b/src/include/catalog_utils.hpp @@ -19,7 +19,7 @@ struct ICType { class ICUtils { public: static LogicalType ToUCType(const LogicalType &input); - static LogicalType TypeToLogicalType(ClientContext &context, const string &columnDefinition); + static LogicalType TypeToLogicalType(const string &columnDefinition); static string TypeToString(const LogicalType &input); static string LogicalToIcebergType(const LogicalType &input); }; diff --git a/src/include/ic_create_table_as_op.hpp b/src/include/ic_create_table_as_op.hpp index 90e5d0f..299e678 100644 --- a/src/include/ic_create_table_as_op.hpp +++ b/src/include/ic_create_table_as_op.hpp @@ -5,20 +5,23 @@ #include "duckdb/execution/operator/schema/physical_create_table.hpp" #include "duckdb/planner/parsed_data/bound_create_table_info.hpp" +#include "storage/ic_catalog.hpp" +#include "storage/ic_schema_entry.hpp" + namespace duckdb { class ICCreateTableAsOp : public PhysicalOperator { public: ICCreateTableAsOp(vector types, unique_ptr info, + ICSchemaEntry *schemaEntry, idx_t estimated_cardinality, string &internal_name, - ICCatalogSet &catalog_set, ICCredentials &credentials) : PhysicalOperator(PhysicalOperatorType::CREATE_TABLE_AS, types, estimated_cardinality), info(std::move(info)), + schemaEntry(schemaEntry), catalog_internal_name(internal_name), - catalog_set(catalog_set), table_credentials(credentials) {} // Override the Execute method @@ -31,7 +34,7 @@ class ICCreateTableAsOp : public PhysicalOperator { private: unique_ptr info; string &catalog_internal_name; - ICCatalogSet &catalog_set; + ICSchemaEntry *schemaEntry; ICCredentials &table_credentials; }; diff --git a/src/include/storage/ic_catalog_set.hpp b/src/include/storage/ic_catalog_set.hpp index 927c6a2..fd988d4 100644 --- a/src/include/storage/ic_catalog_set.hpp +++ b/src/include/storage/ic_catalog_set.hpp @@ -17,7 +17,7 @@ class ICCatalogSet { optional_ptr GetEntry(ClientContext &context, const string &name); virtual void DropEntry(ClientContext &context, DropInfo &info); void Scan(ClientContext &context, const std::function &callback); - virtual optional_ptr CreateEntry(unique_ptr entry); + virtual optional_ptr AddEntry(unique_ptr entry); void ClearEntries(); protected: diff --git a/src/include/storage/ic_table_set.hpp b/src/include/storage/ic_table_set.hpp index 116ce49..bbfbf80 100644 --- a/src/include/storage/ic_table_set.hpp +++ b/src/include/storage/ic_table_set.hpp @@ -44,7 +44,7 @@ class ICTableSet : public ICInSchemaSet { static void AddColumn(ClientContext &context, ICResult &result, ICTableInfo &table_info, idx_t column_offset = 0); private: - unique_ptr CreateTableEntry(ClientContext &context, ICAPITable table); + unique_ptr CreateTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, ICAPITable table); }; diff --git a/src/storage/ic_catalog.cpp b/src/storage/ic_catalog.cpp index e74e25e..243893c 100644 --- a/src/storage/ic_catalog.cpp +++ b/src/storage/ic_catalog.cpp @@ -87,12 +87,18 @@ unique_ptr ICCatalog::PlanInsert(ClientContext &context, Logic unique_ptr ICCatalog::PlanCreateTableAs( ClientContext &context, LogicalCreateTable &op, unique_ptr plan) { + CatalogEntry *catalog_entry = schemas.GetEntry(context, op.info->base->schema).get(); + if (!catalog_entry) { + throw BinderException("Schema not found"); + } + + auto schemaEntry = static_cast(catalog_entry); auto create_table_as = make_uniq( op.types, std::move(op.info), + schemaEntry, op.estimated_cardinality, internal_name, - schemas, credentials); // If your operator has children (input plan), attach them diff --git a/src/storage/ic_catalog_set.cpp b/src/storage/ic_catalog_set.cpp index fb54255..13fb2e6 100644 --- a/src/storage/ic_catalog_set.cpp +++ b/src/storage/ic_catalog_set.cpp @@ -38,9 +38,6 @@ void ICCatalogSet::Scan(ClientContext &context, const std::function ICCatalogSet::AddEntry(unique_ptr entry) { - // Also ICSchemaSet::AddEntry - std::cout << "ICCatalogSet::AddEntry" << std::endl; - lock_guard l(entry_lock); auto result = entry.get(); if (result->name.empty()) { diff --git a/src/storage/ic_table_entry.cpp b/src/storage/ic_table_entry.cpp index 5bf7511..7ce387c 100644 --- a/src/storage/ic_table_entry.cpp +++ b/src/storage/ic_table_entry.cpp @@ -72,7 +72,7 @@ TableFunction ICTableEntry::GetScanFunction(ClientContext &context, unique_ptrdata_source_format); } - if (table_data->storage_location.find("file://") != 0) { + if (table_data->metadata_location.find("file://") != 0) { auto &secret_manager = SecretManager::Get(context); // Get Credentials from ICAPI auto table_credentials = ICAPI::GetTableCredentials( @@ -91,8 +91,8 @@ TableFunction ICTableEntry::GetScanFunction(ClientContext &context, unique_ptrstorage_location.size()); - std::transform(table_data->storage_location.begin(), table_data->storage_location.end(), lc_storage_location.begin(), ::tolower); + lc_storage_location.resize(table_data->metadata_location.size()); + std::transform(table_data->metadata_location.begin(), table_data->metadata_location.end(), lc_storage_location.begin(), ::tolower); size_t metadata_pos = lc_storage_location.find("metadata"); if (metadata_pos != std::string::npos) { info.scope = {lc_storage_location.substr(0, metadata_pos)}; @@ -109,7 +109,7 @@ TableFunction ICTableEntry::GetScanFunction(ClientContext &context, unique_ptr inputs = {table_data->storage_location}; + vector inputs = {table_data->metadata_location}; TableFunctionBindInput bind_input(inputs, param_map, return_types, names, nullptr, nullptr, iceberg_scan_function, empty_ref); diff --git a/src/storage/ic_table_set.cpp b/src/storage/ic_table_set.cpp index 7203e4b..2dca47e 100644 --- a/src/storage/ic_table_set.cpp +++ b/src/storage/ic_table_set.cpp @@ -21,17 +21,16 @@ namespace duckdb { ICTableSet::ICTableSet(ICSchemaEntry &schema) : ICInSchemaSet(schema) { } -static ColumnDefinition CreateColumnDefinition(ClientContext &context, ICAPIColumnDefinition &coldef) { - return {coldef.name, ICUtils::TypeToLogicalType(context, coldef.type_text)}; +static ColumnDefinition CreateColumnDefinition(ICAPIColumnDefinition &coldef) { + return {coldef.name, ICUtils::TypeToLogicalType(coldef.type_text)}; } -unique_ptr ICTableSet::CreateTableEntry(ClientContext &context, ICAPITable table) { +unique_ptr ICTableSet::CreateTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, ICAPITable table) { D_ASSERT(schema.name == table.schema_name); CreateTableInfo info; info.table = table.name; - for (auto &col : table.columns) { - info.columns.AddColumn(CreateColumnDefinition(context, col)); + info.columns.AddColumn(CreateColumnDefinition(col)); } auto table_entry = make_uniq(catalog, schema, info); @@ -41,13 +40,13 @@ unique_ptr ICTableSet::CreateTableEntry(ClientContext &context, IC void ICTableSet::FillEntry(ClientContext &context, unique_ptr &entry) { auto* derived = static_cast(entry.get()); - if (!derived->table_data->storage_location.empty()) { + if (!derived->table_data->metadata_location.empty()) { return; } auto &ic_catalog = catalog.Cast(); auto table = ICAPI::GetTable(catalog.GetName(), catalog.GetDBPath(), schema.name, entry->name, ic_catalog.credentials); - entry = CreateTableEntry(context, table); + entry = CreateTableEntry(catalog, schema, table); } void ICTableSet::LoadEntries(ClientContext &context) { @@ -60,7 +59,7 @@ void ICTableSet::LoadEntries(ClientContext &context) { auto tables = ICAPI::GetTables(catalog.GetName(), catalog.GetDBPath(), schema.name, ic_catalog.credentials); for (auto &table : tables) { - auto entry = CreateTableEntry(context, table); + auto entry = CreateTableEntry(catalog, schema, table); AddEntry(std::move(entry)); } } @@ -82,7 +81,7 @@ optional_ptr ICTableSet::CreateTable(ClientContext &context, Bound auto &ic_catalog = catalog.Cast(); auto *table_info = dynamic_cast(info.base.get()); auto table = ICAPI::CreateTable(catalog.GetName(), ic_catalog.internal_name, schema.name, ic_catalog.credentials, table_info); - auto entry = CreateTableEntry(context, table); + auto entry = CreateTableEntry(catalog, schema, table); return AddEntry(std::move(entry)); } @@ -115,8 +114,6 @@ ICInSchemaSet::ICInSchemaSet(ICSchemaEntry &schema) : ICCatalogSet(schema.Parent } optional_ptr ICInSchemaSet::AddEntry(unique_ptr entry) { - std::cout << "ICInSchemaSet::CreateEntry" << std::endl; - if (!entry->internal) { entry->internal = schema.internal; } From 848021f3d7c1403491edfaefb6039fe9dd62118b Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Wed, 5 Feb 2025 22:58:07 -0500 Subject: [PATCH 20/26] Move iceberg catalog name to ICAPISchema --- src/catalog_api.cpp | 45 ++++++++++++++------------- src/ic_create_table_as_op.cpp | 4 +-- src/include/catalog_api.hpp | 17 +++++----- src/include/ic_create_table_as_op.hpp | 3 -- src/storage/ic_catalog.cpp | 1 - 5 files changed, 34 insertions(+), 36 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index c0a4029..f80d033 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -219,11 +219,11 @@ static yyjson_doc *api_result_to_doc(const string &api_result) { return doc; } -static string GetTableMetadata(const string &internal, const string &schema, const string &table, ICCredentials credentials) { +static string GetTableMetadata(const string &iceberg_catalog, const string &schema, const string &table, ICCredentials credentials) { struct curl_slist *extra_headers = NULL; extra_headers = curl_slist_append(extra_headers, "X-Iceberg-Access-Delegation: vended-credentials"); string api_result = GetRequest( - credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables/" + table, + credentials.endpoint + "/v1/" + iceberg_catalog + "/namespaces/" + schema + "/tables/" + table, credentials.token, extra_headers); curl_slist_free_all(extra_headers); @@ -248,9 +248,9 @@ static ICAPIColumnDefinition ParseColumnDefinition(yyjson_val *column_def) { return result; } -ICAPITableCredentials ICAPI::GetTableCredentials(const string &internal, const string &schema, const string &table, ICCredentials credentials) { +ICAPITableCredentials ICAPI::GetTableCredentials(const string &iceberg_catalog, const string &schema, const string &table, ICCredentials credentials) { ICAPITableCredentials result; - string api_result = GetTableMetadata(internal, schema, table, credentials); + string api_result = GetTableMetadata(iceberg_catalog, schema, table, credentials); std::unique_ptr doc(api_result_to_doc(api_result)); auto *root = yyjson_doc_get_root(doc.get()); auto *aws_temp_credentials = yyjson_obj_get(root, "config"); @@ -313,11 +313,11 @@ static ICAPITable createTable(const string &catalog, const string &schema, const } ICAPITable ICAPI::GetTable( - const string &catalog, const string &internal, const string &schema, const string &table_name, std::optional credentials) { + const string &catalog, const string &iceberg_catalog, const string &schema, const string &table_name, std::optional credentials) { ICAPITable table_result = createTable(catalog, schema, table_name); if (credentials) { - string result = GetTableMetadata(internal, schema, table_result.name, *credentials); + string result = GetTableMetadata(iceberg_catalog, schema, table_result.name, *credentials); std::unique_ptr doc(api_result_to_doc(result)); auto *metadata_root = yyjson_doc_get_root(doc.get()); populateTableMetadata(table_result, metadata_root); @@ -336,26 +336,26 @@ ICAPITable ICAPI::GetTable( } // TODO: handle out-of-order columns using position property -vector ICAPI::GetTables(const string &catalog, const string &internal, const string &schema, ICCredentials credentials) { +vector ICAPI::GetTables(const string &catalog, const string &iceberg_catalog, const string &schema, ICCredentials credentials) { vector result; - string api_result = GetRequest(credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables", credentials.token); + string api_result = GetRequest(credentials.endpoint + "/v1/" + iceberg_catalog + "/namespaces/" + schema + "/tables", credentials.token); std::unique_ptr doc(api_result_to_doc(api_result)); auto *root = yyjson_doc_get_root(doc.get()); auto *tables = yyjson_obj_get(root, "identifiers"); size_t idx, max; yyjson_val *table; yyjson_arr_foreach(tables, idx, max, table) { - auto table_result = GetTable(catalog, internal, schema, TryGetStrFromObject(table, "name"), std::nullopt); + auto table_result = GetTable(catalog, iceberg_catalog, schema, TryGetStrFromObject(table, "name"), std::nullopt); result.push_back(table_result); } return result; } -vector ICAPI::GetSchemas(const string &catalog, const string &internal, ICCredentials credentials) { +vector ICAPI::GetSchemas(const string &catalog, const string &iceberg_catalog, ICCredentials credentials) { vector result; string api_result = - GetRequest(credentials.endpoint + "/v1/" + internal + "/namespaces", credentials.token); + GetRequest(credentials.endpoint + "/v1/" + iceberg_catalog + "/namespaces", credentials.token); std::unique_ptr doc(api_result_to_doc(api_result)); auto *root = yyjson_doc_get_root(doc.get()); auto *schemas = yyjson_obj_get(root, "namespaces"); @@ -364,35 +364,36 @@ vector ICAPI::GetSchemas(const string &catalog, const string &inter yyjson_arr_foreach(schemas, idx, max, schema) { ICAPISchema schema_result; schema_result.catalog_name = catalog; - yyjson_val *value = yyjson_arr_get(schema, 0); - schema_result.schema_name = yyjson_get_str(value); + schema_result.schema_name = yyjson_get_str(yyjson_arr_get(schema, 0)); + schema_result.iceberg_catalog = iceberg_catalog; result.push_back(schema_result); } return result; } -ICAPISchema ICAPI::CreateSchema(const string &catalog, const string &internal, const string &schema, ICCredentials credentials) { +ICAPISchema ICAPI::CreateSchema(const string &catalog, const string &iceberg_catalog, const string &schema, ICCredentials credentials) { string post_data = "{\"namespace\":[\"" + schema + "\"]}"; string api_result = PostRequest( - credentials.endpoint + "/v1/" + internal + "/namespaces", post_data, "json", credentials.token); + credentials.endpoint + "/v1/" + iceberg_catalog + "/namespaces", post_data, "json", credentials.token); api_result_to_doc(api_result); // if the method returns, request was successful ICAPISchema schema_result; schema_result.catalog_name = catalog; - schema_result.schema_name = schema; //yyjson_get_str(value); + schema_result.schema_name = schema; + schema_result.iceberg_catalog = iceberg_catalog; return schema_result; } -void ICAPI::DropSchema(const string &internal, const string &schema, ICCredentials credentials) { +void ICAPI::DropSchema(const string &iceberg_catalog, const string &schema, ICCredentials credentials) { string api_result = DeleteRequest( - credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema, credentials.token); + credentials.endpoint + "/v1/" + iceberg_catalog + "/namespaces/" + schema, credentials.token); api_result_to_doc(api_result); // if the method returns, request was successful } -void ICAPI::DropTable(const string &catalog, const string &internal, const string &schema, string &table_name, ICCredentials credentials) { +void ICAPI::DropTable(const string &catalog, const string &iceberg_catalog, const string &schema, string &table_name, ICCredentials credentials) { string api_result = DeleteRequest( - credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables/" + table_name + "?purgeRequested=true", + credentials.endpoint + "/v1/" + iceberg_catalog + "/namespaces/" + schema + "/tables/" + table_name + "?purgeRequested=true", credentials.token); api_result_to_doc(api_result); // if the method returns, request was successful } @@ -406,7 +407,7 @@ static std::string json_to_string(yyjson_mut_doc *doc) { return json_str; } -ICAPITable ICAPI::CreateTable(const string &catalog, const string &internal, const string &schema, ICCredentials &credentials, CreateTableInfo *table_info) { +ICAPITable ICAPI::CreateTable(const string &catalog, const string &iceberg_catalog, const string &schema, ICCredentials &credentials, CreateTableInfo *table_info) { std::unique_ptr dd(yyjson_mut_doc_new(NULL)); yyjson_mut_doc *doc = dd.get(); @@ -453,7 +454,7 @@ ICAPITable ICAPI::CreateTable(const string &catalog, const string &internal, con struct curl_slist *extra_headers = NULL; extra_headers = curl_slist_append(extra_headers, "X-Iceberg-Access-Delegation: vended-credentials"); string api_result = PostRequest( - credentials.endpoint + "/v1/" + internal + "/namespaces/" + schema + "/tables", post_data, "json", credentials.token, extra_headers); + credentials.endpoint + "/v1/" + iceberg_catalog + "/namespaces/" + schema + "/tables", post_data, "json", credentials.token, extra_headers); curl_slist_free_all(extra_headers); std::unique_ptr dd2(api_result_to_doc(api_result)); diff --git a/src/ic_create_table_as_op.cpp b/src/ic_create_table_as_op.cpp index 7ad4aed..1510ac8 100644 --- a/src/ic_create_table_as_op.cpp +++ b/src/ic_create_table_as_op.cpp @@ -147,7 +147,7 @@ OperatorResultType ICCreateTableAsOp::Execute(ExecutionContext &context, // Create a file output stream const std::string datafile_filename = - catalog_internal_name + "." + table_info->schema + "." + table_info->table + ".parquet"; + schemaEntry->schema_data->iceberg_catalog + "." + table_info->schema + "." + table_info->table + ".parquet"; auto open_result = arrow::io::FileOutputStream::Open(datafile_filename); if (!open_result.ok()) { @@ -189,7 +189,7 @@ OperatorResultType ICCreateTableAsOp::Execute(ExecutionContext &context, thread, const_cast(table_credentials.endpoint.c_str()), const_cast(creds.c_str()), - const_cast(catalog_internal_name.c_str()), + const_cast(schemaEntry->schema_data->iceberg_catalog.c_str()), const_cast(table_info->schema.c_str()), const_cast(table_info->table.c_str()), const_cast(datafile_filename.c_str()), diff --git a/src/include/catalog_api.hpp b/src/include/catalog_api.hpp index ad4bfdf..f6c412e 100644 --- a/src/include/catalog_api.hpp +++ b/src/include/catalog_api.hpp @@ -31,6 +31,7 @@ struct ICAPITable { struct ICAPISchema { string schema_name; string catalog_name; + string iceberg_catalog; }; struct ICAPITableCredentials { @@ -44,17 +45,17 @@ class ICAPI { //! WARNING: not thread-safe. To be called once on extension initialization static void InitializeCurl(); - static ICAPITableCredentials GetTableCredentials(const string &internal, const string &schema, const string &table, ICCredentials credentials); + static ICAPITableCredentials GetTableCredentials(const string &iceberg_catalog, const string &schema, const string &table, ICCredentials credentials); static vector GetCatalogs(const string &catalog, ICCredentials credentials); - static vector GetTables(const string &catalog, const string &internal, const string &schema, ICCredentials credentials); - static ICAPITable GetTable(const string &catalog, const string &internal, const string &schema, const string &table_name, std::optional credentials); - static vector GetSchemas(const string &catalog, const string &internal, ICCredentials credentials); + static vector GetTables(const string &catalog, const string &iceberg_catalog, const string &schema, ICCredentials credentials); + static ICAPITable GetTable(const string &catalog, const string &iceberg_catalog, const string &schema, const string &table_name, std::optional credentials); + static vector GetSchemas(const string &catalog, const string &iceberg_catalog, ICCredentials credentials); static vector GetTablesInSchema(const string &catalog, const string &schema, ICCredentials credentials); static string GetToken(string id, string secret, string endpoint); - static ICAPISchema CreateSchema(const string &catalog, const string &internal, const string &schema, ICCredentials credentials); - static void DropSchema(const string &internal, const string &schema, ICCredentials credentials); - static ICAPITable CreateTable(const string &catalog, const string &internal, const string &schema, ICCredentials &credentials, CreateTableInfo *table_info); - static void DropTable(const string &catalog, const string &internal, const string &schema, string &table_name, ICCredentials credentials); + static ICAPISchema CreateSchema(const string &catalog, const string &iceberg_catalog, const string &schema, ICCredentials credentials); + static void DropSchema(const string &iceberg_catalog, const string &schema, ICCredentials credentials); + static ICAPITable CreateTable(const string &catalog, const string &iceberg_catalog, const string &schema, ICCredentials &credentials, CreateTableInfo *table_info); + static void DropTable(const string &catalog, const string &iceberg_catalog, const string &schema, string &table_name, ICCredentials credentials); }; } // namespace duckdb diff --git a/src/include/ic_create_table_as_op.hpp b/src/include/ic_create_table_as_op.hpp index 299e678..a2ae47c 100644 --- a/src/include/ic_create_table_as_op.hpp +++ b/src/include/ic_create_table_as_op.hpp @@ -16,12 +16,10 @@ class ICCreateTableAsOp : public PhysicalOperator { unique_ptr info, ICSchemaEntry *schemaEntry, idx_t estimated_cardinality, - string &internal_name, ICCredentials &credentials) : PhysicalOperator(PhysicalOperatorType::CREATE_TABLE_AS, types, estimated_cardinality), info(std::move(info)), schemaEntry(schemaEntry), - catalog_internal_name(internal_name), table_credentials(credentials) {} // Override the Execute method @@ -33,7 +31,6 @@ class ICCreateTableAsOp : public PhysicalOperator { private: unique_ptr info; - string &catalog_internal_name; ICSchemaEntry *schemaEntry; ICCredentials &table_credentials; }; diff --git a/src/storage/ic_catalog.cpp b/src/storage/ic_catalog.cpp index 243893c..acae844 100644 --- a/src/storage/ic_catalog.cpp +++ b/src/storage/ic_catalog.cpp @@ -98,7 +98,6 @@ unique_ptr ICCatalog::PlanCreateTableAs( std::move(op.info), schemaEntry, op.estimated_cardinality, - internal_name, credentials); // If your operator has children (input plan), attach them From 71424492f3c5b3511dc1b27ea08a66d37906da78 Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Wed, 5 Feb 2025 23:18:06 -0500 Subject: [PATCH 21/26] Add roadmap --- README-ADDENDUM.md | 60 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 51 insertions(+), 9 deletions(-) diff --git a/README-ADDENDUM.md b/README-ADDENDUM.md index bbc2554..a355db1 100644 --- a/README-ADDENDUM.md +++ b/README-ADDENDUM.md @@ -2,14 +2,13 @@ This fork adds proof-of-concept functionality to DuckDB iceberg extension to be able to connect to an iceberg catalog as well as read and write iceberg tables. -You can try it out using DuckDB (>= v1.1.3) by doing the following: - -1. Start duckdb in `unsigned` mode +You can try it out using DuckDB (>= v1.1.3) by running duckdb in unsigned mode: ```bash duckdb --unsigned ``` -2. Create a secret to provide access to your iceberg catalog +# SQL commands +## Create a secret to provide access to an iceberg catalog ```sql INSTALL '/path/to/this/iceberg.duckdb_extension'; INSTALL httpfs; @@ -24,25 +23,25 @@ CREATE SECRET ( ) ``` -3. Attach your iceberg catalog +## Attach an iceberg catalog ```sql ATTACH 'my_catalog' AS my_catalog (TYPE ICEBERG) ``` -4. Read an iceberg table +## Read an iceberg table ```sql SELECT * FROM my_catalog.my_schema.table_1; ``` -5. Create a new iceberg table +## Create a new iceberg table ```sql CREATE TABLE my_catalog.my_schema.new_table (id BIGINT, name VARCHAR); ``` ```sql -CREATE TABLE my_catalog.my_schema.new_table_2 AS (SELECT FROM version()); +CREATE TABLE my_catalog.my_schema.new_table_2 AS (SELECT FROM version() as "version"); ``` -6. Delete an existing iceberg table +## Delete an existing iceberg table ```sql DROP TABLE my_catalog.my_schema.table_1; ``` @@ -54,3 +53,46 @@ git submodule update --init --recursive brew install ninja GEN=ninja make {debug/release} ``` + +# Roadmap +## # SQL commands +### ✅ CREATE SECRET +### ✅ ATTACH +### 🔳 USE +### ✅ SELECT +### ✅ CREATE SCHEMA +### ✅ DROP SCHEMA +### 🔳 CREATE VIEW +### 🔳 DROP VIEW +### ✅ CREATE TABLE +### ✅ CREATE TABLE AS SELECT +### 🔳 ALTER TABLE +### ✅ DROP TABLE +### 🔳 INSERT +### 🔳 UPDATE +### 🔳 DELETE + +## # Data Types ([ref](https://docs.snowflake.com/en/user-guide/tables-iceberg-data-types)) +### 🔳 boolean +### ✅ string +### 🔳 tinyint +### 🔳 smallint +### ✅ int +### ✅ long +### ✅ double +### 🔳 float +### 🔳 timestamp +### 🔳 timestamptz +### 🔳 binary +### 🔳 date +### 🔳 decimal(prec,scale) +### 🔳 array +### 🔳 map +### 🔳 struct + + + + + + + From 17c30a46d4a773e23121f42c33cdc6dfd575ba42 Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Thu, 6 Feb 2025 19:07:22 -0500 Subject: [PATCH 22/26] Add misc to roadmap --- README-ADDENDUM.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README-ADDENDUM.md b/README-ADDENDUM.md index a355db1..35fe393 100644 --- a/README-ADDENDUM.md +++ b/README-ADDENDUM.md @@ -90,6 +90,9 @@ GEN=ninja make {debug/release} ### 🔳 map ### 🔳 struct +## # Miscellaneous +### 🔳 Bundle `jiceberg` statically into the extension + From 6d2fb229aa384e451494d3433a0062929b604ca7 Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Thu, 6 Feb 2025 19:30:53 -0500 Subject: [PATCH 23/26] Build against duckdb v1.2.0 --- duckdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/duckdb b/duckdb index f99785b..5f5512b 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit f99785b78ae4724b31d9b41435ad8c17e57ee8f4 +Subproject commit 5f5512b827df6397afd31daedb4bbdee76520019 From aa52e10d75d799ff5b268a75b87ff7ea89703b3f Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Thu, 6 Feb 2025 19:31:01 -0500 Subject: [PATCH 24/26] Update instructions --- README-ADDENDUM.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/README-ADDENDUM.md b/README-ADDENDUM.md index 35fe393..74c8c30 100644 --- a/README-ADDENDUM.md +++ b/README-ADDENDUM.md @@ -2,18 +2,26 @@ This fork adds proof-of-concept functionality to DuckDB iceberg extension to be able to connect to an iceberg catalog as well as read and write iceberg tables. -You can try it out using DuckDB (>= v1.1.3) by running duckdb in unsigned mode: +You can try it out using DuckDB (>= v1.2.0) by running duckdb in unsigned mode: ```bash duckdb --unsigned ``` # SQL commands -## Create a secret to provide access to an iceberg catalog +## Install this extension and load it. If you already have the official `iceberg` extension installed, you will need to force install this one. ```sql INSTALL '/path/to/this/iceberg.duckdb_extension'; -INSTALL httpfs; LOAD '/path/to/this/iceberg.duckdb_extension'; +``` + +## Install `httpfs` extension (if you don't have it already) and load it +```sql +INSTALL httpfs; LOAD httpfs; +``` + +## Create a secret to provide access to an iceberg catalog +```sql CREATE SECRET ( TYPE ICEBERG, CLIENT_ID '${CLIENT_ID}', From 2f6d81ea897e470f8b63e71dded5d8d7a2d69e0d Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Thu, 6 Feb 2025 19:49:17 -0500 Subject: [PATCH 25/26] Improve instructions --- README-ADDENDUM.md | 42 ++++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/README-ADDENDUM.md b/README-ADDENDUM.md index 74c8c30..f9018bf 100644 --- a/README-ADDENDUM.md +++ b/README-ADDENDUM.md @@ -1,25 +1,40 @@ # ADDENDUM -This fork adds proof-of-concept functionality to DuckDB iceberg extension to be able to connect to an iceberg catalog as well as read and write iceberg tables. +This fork adds proof-of-concept functionality to DuckDB iceberg extension to be able to connect to an iceberg catalog and write to iceberg tables as well as read from them. -You can try it out using DuckDB (>= v1.2.0) by running duckdb in unsigned mode: +# Requirements +You will need the following to be able to use this new version of the extension: +1. DuckDB version 1.2.0 or later +2. `httpfs` extension + +Since this extension is not official yet, you will need to run duckdb in `unsigned` mode to be able to use it: ```bash duckdb --unsigned ``` -# SQL commands -## Install this extension and load it. If you already have the official `iceberg` extension installed, you will need to force install this one. +# Installation +The following steps need to be done once: +1. Download the zip from github and unzip it +2. Change directory to the directory where you unzipped the files +3. Install the extension ```sql -INSTALL '/path/to/this/iceberg.duckdb_extension'; -LOAD '/path/to/this/iceberg.duckdb_extension'; +INSTALL './iceberg.duckdb_extension'; ``` - -## Install `httpfs` extension (if you don't have it already) and load it +4. If you already have the official `iceberg` extension installed, you will need to force the install +```sql +FORCE INSTALL './iceberg.duckdb_extension'; +``` +5. Install `httpfs` extension if you don't have it installed already ```sql INSTALL httpfs; -LOAD httpfs; ``` +# Usage +## Load `httpfs` and `iceberg` extensions +```sql +LOAD httpfs; +LOAD iceberg; +``` ## Create a secret to provide access to an iceberg catalog ```sql CREATE SECRET ( @@ -55,6 +70,9 @@ DROP TABLE my_catalog.my_schema.table_1; ``` # How to build extension from source +Requirements: +* A compiler that supports C++17 +* CMake version 3.28 or later ``` git clone https://github.com/fivetran/duckdb-iceberg.git git submodule update --init --recursive @@ -63,7 +81,7 @@ GEN=ninja make {debug/release} ``` # Roadmap -## # SQL commands +## 1. SQL commands ### ✅ CREATE SECRET ### ✅ ATTACH ### 🔳 USE @@ -80,7 +98,7 @@ GEN=ninja make {debug/release} ### 🔳 UPDATE ### 🔳 DELETE -## # Data Types ([ref](https://docs.snowflake.com/en/user-guide/tables-iceberg-data-types)) +## 2. Iceberg Data Types ([ref](https://docs.snowflake.com/en/user-guide/tables-iceberg-data-types)) ### 🔳 boolean ### ✅ string ### 🔳 tinyint @@ -98,7 +116,7 @@ GEN=ninja make {debug/release} ### 🔳 map ### 🔳 struct -## # Miscellaneous +## 3. Miscellaneous ### 🔳 Bundle `jiceberg` statically into the extension From 8cb8cd4c41941db2157525b601f4fd3f38c7e76c Mon Sep 17 00:00:00 2001 From: "emrah.diril" Date: Thu, 6 Feb 2025 21:33:27 -0500 Subject: [PATCH 26/26] Clarify roadmap items --- README-ADDENDUM.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README-ADDENDUM.md b/README-ADDENDUM.md index f9018bf..f992200 100644 --- a/README-ADDENDUM.md +++ b/README-ADDENDUM.md @@ -81,7 +81,7 @@ GEN=ninja make {debug/release} ``` # Roadmap -## 1. SQL commands +## 1. Supported SQL commands ### ✅ CREATE SECRET ### ✅ ATTACH ### 🔳 USE @@ -98,7 +98,7 @@ GEN=ninja make {debug/release} ### 🔳 UPDATE ### 🔳 DELETE -## 2. Iceberg Data Types ([ref](https://docs.snowflake.com/en/user-guide/tables-iceberg-data-types)) +## 2. Supported [Iceberg data types](https://docs.snowflake.com/en/user-guide/tables-iceberg-data-types) (writing) ### 🔳 boolean ### ✅ string ### 🔳 tinyint