From bba97e54ce1e8594730bcbdaf55d011c091e4e44 Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Sun, 22 Dec 2024 20:00:14 +0100 Subject: [PATCH 1/4] Add chsql_native chsql_native is an extension for chsql adding ClickHouse native file reading capabilities --- extensions/chsql_native/description.yml | 62 +++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 extensions/chsql_native/description.yml diff --git a/extensions/chsql_native/description.yml b/extensions/chsql_native/description.yml new file mode 100644 index 0000000..c6d720a --- /dev/null +++ b/extensions/chsql_native/description.yml @@ -0,0 +1,62 @@ +extension: + name: chsql_native + description: ClickHouse Native File reader for chsql + version: 0.0.1 + language: Rust + build: cmake + license: MIT + excluded_platforms: "windows_amd64_rtools;windows_amd64;wasm_threads;wasm_eh;wasm_mvp" + requires_toolchains: "rust;python3" + maintainers: + - lmangani + - adubovikov + +repo: + github: quackscience/duckdb-extension-clickhouse-native + ref: 0116eb462ec85fa000f1cb15a3b0ee6165711b78 + +docs: + hello_world: | + --- This experimental rust extension allows reading ClickHouse Native files with DuckDB + --- Test files can be generated with clickhouse-local. See README for full examples. + + --- Simple Example + D SELECT * FROM clickhouse_native('/tmp/numbers.clickhouse'); + ┌──────────────┬─────────┐ + │ version() │ number │ + │ varchar │ int32 │ + ├──────────────┼─────────┤ + │ 24.12.1.1273 │ 0 │ + └──────────────┴─────────┘ + + --- Long Example + D SELECT count(*), max(number) FROM clickhouse_native('/tmp/100000.clickhouse'); + ┌──────────────┬─────────────┐ + │ count_star() │ max(number) │ + │ int64 │ int32 │ + ├──────────────┼─────────────┤ + │ 100000 │ 99999 │ + └──────────────┴─────────────┘ + + --- Wide Example + D SELECT * FROM clickhouse_native('/tmp/functions.clickhouse') WHERE alias_to != '' LIMIT 10; + ┌────────────────────┬──────────────┬──────────────────┬──────────────────────┬──────────────┬─────────┬───┬─────────┬───────────┬────────────────┬──────────┬────────────┐ + │ name │ is_aggregate │ case_insensitive │ alias_to │ create_query │ origin │ … │ syntax │ arguments │ returned_value │ examples │ categories │ + │ varchar │ int32 │ int32 │ varchar │ varchar │ varchar │ │ varchar │ varchar │ varchar │ varchar │ varchar │ + ├────────────────────┼──────────────┼──────────────────┼──────────────────────┼──────────────┼─────────┼───┼─────────┼───────────┼────────────────┼──────────┼────────────┤ + │ connection_id │ 0 │ 1 │ connectionID │ │ System │ … │ │ │ │ │ │ + │ rand32 │ 0 │ 0 │ rand │ │ System │ … │ │ │ │ │ │ + │ INET6_ATON │ 0 │ 1 │ IPv6StringToNum │ │ System │ … │ │ │ │ │ │ + │ INET_ATON │ 0 │ 1 │ IPv4StringToNum │ │ System │ … │ │ │ │ │ │ + │ truncate │ 0 │ 1 │ trunc │ │ System │ … │ │ │ │ │ │ + │ ceiling │ 0 │ 1 │ ceil │ │ System │ … │ │ │ │ │ │ + │ replace │ 0 │ 1 │ replaceAll │ │ System │ … │ │ │ │ │ │ + │ from_utc_timestamp │ 0 │ 1 │ fromUTCTimestamp │ │ System │ … │ │ │ │ │ │ + │ mapFromString │ 0 │ 0 │ extractKeyValuePairs │ │ System │ … │ │ │ │ │ │ + │ str_to_map │ 0 │ 1 │ extractKeyValuePairs │ │ System │ … │ │ │ │ │ │ + ├────────────────────┴──────────────┴──────────────────┴──────────────────────┴──────────────┴─────────┴───┴─────────┴───────────┴────────────────┴──────────┴────────────┤ + │ 10 rows 12 columns (11 shown) │ + └─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ + + extended_description: | + This extension is highly experimental and potentially unstable. All reads are full-scans. Do not use in production. From 4d60188bc5dbbe5e347f95db4ed16d72dc7b1d21 Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Wed, 1 Jan 2025 18:26:06 +0100 Subject: [PATCH 2/4] chsql_native 0.0.2 --- extensions/chsql_native/description.yml | 47 +++++++++++++++++++++---- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/extensions/chsql_native/description.yml b/extensions/chsql_native/description.yml index c6d720a..2298a84 100644 --- a/extensions/chsql_native/description.yml +++ b/extensions/chsql_native/description.yml @@ -1,7 +1,7 @@ extension: name: chsql_native - description: ClickHouse Native File reader for chsql - version: 0.0.1 + description: ClickHouse Native Client & File Reader for chsql + version: 0.0.2 language: Rust build: cmake license: MIT @@ -13,12 +13,47 @@ extension: repo: github: quackscience/duckdb-extension-clickhouse-native - ref: 0116eb462ec85fa000f1cb15a3b0ee6165711b78 + ref: 1c9d08775def3b5a05a7d58590e5924845390657 docs: hello_world: | - --- This experimental rust extension allows reading ClickHouse Native files with DuckDB - --- Test files can be generated with clickhouse-local. See README for full examples. + --- This experimental rust extension implements Native Clickhouse formats for DuckDB. + --- Native Binary Client for chsql + --- CLICKHOUSE_URL="tcp://localhost:9000" + --- CLICKHOUSE_USER="default" + + --- Simple Example + D SELECT * FROM clickhouse_scan("SELECT version(), 'hello', 123"); + ┌────────────┬─────────┬────────┐ + │ version() │ 'hello' │ 123 │ + │ varchar │ varchar │ uint32 │ + ├────────────┼─────────┼────────┤ + │ 24.10.2.80 │ hello │ 123 │ + └────────────┴─────────┴────────┘ + + --- Wide Example + D SELECT * FROM clickhouse_scan("SELECT * FROM system.functions WHERE alias_to != '' LIMIT 10"); + ┌────────────────────┬──────────────┬──────────────────┬──────────────────────┬───┬───────────┬────────────────┬──────────┬────────────┐ + │ name │ is_aggregate │ case_insensitive │ alias_to │ … │ arguments │ returned_value │ examples │ categories │ + │ varchar │ uint32 │ uint32 │ varchar │ │ varchar │ varchar │ varchar │ varchar │ + ├────────────────────┼──────────────┼──────────────────┼──────────────────────┼───┼───────────┼────────────────┼──────────┼────────────┤ + │ connection_id │ 0 │ 1 │ connectionID │ … │ │ │ │ │ + │ rand32 │ 0 │ 0 │ rand │ … │ │ │ │ │ + │ INET6_ATON │ 0 │ 1 │ IPv6StringToNum │ … │ │ │ │ │ + │ INET_ATON │ 0 │ 1 │ IPv4StringToNum │ … │ │ │ │ │ + │ truncate │ 0 │ 1 │ trunc │ … │ │ │ │ │ + │ ceiling │ 0 │ 1 │ ceil │ … │ │ │ │ │ + │ replace │ 0 │ 1 │ replaceAll │ … │ │ │ │ │ + │ from_utc_timestamp │ 0 │ 1 │ fromUTCTimestamp │ … │ │ │ │ │ + │ mapFromString │ 0 │ 0 │ extractKeyValuePairs │ … │ │ │ │ │ + │ str_to_map │ 0 │ 1 │ extractKeyValuePairs │ … │ │ │ │ │ + ├────────────────────┴──────────────┴──────────────────┴──────────────────────┴───┴───────────┴────────────────┴──────────┴────────────┤ + │ 10 rows 12 columns (8 shown) │ + └──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ + + + --- Native File Reader for chsql + --- Test files can be generated with clickhouse-local. File reads are full-scans. --- Simple Example D SELECT * FROM clickhouse_native('/tmp/numbers.clickhouse'); @@ -59,4 +94,4 @@ docs: └─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ extended_description: | - This extension is highly experimental and potentially unstable. All reads are full-scans. Do not use in production. + This extension is highly experimental and potentially unstable. Do not use in production. See README for full examples. From d0634a84cbfe493de00c58c33a6a5d89a6a489fe Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Thu, 2 Jan 2025 11:09:43 +0100 Subject: [PATCH 3/4] Update description.yml --- extensions/chsql_native/description.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/chsql_native/description.yml b/extensions/chsql_native/description.yml index 2298a84..221798f 100644 --- a/extensions/chsql_native/description.yml +++ b/extensions/chsql_native/description.yml @@ -13,7 +13,7 @@ extension: repo: github: quackscience/duckdb-extension-clickhouse-native - ref: 1c9d08775def3b5a05a7d58590e5924845390657 + ref: 7d00778fb5f3ba575e33d975faef7f68527d188a docs: hello_world: | From 92115e854b2d07d32c3d274118cb49e2b0ca79c5 Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Thu, 2 Jan 2025 11:37:31 +0100 Subject: [PATCH 4/4] Update description.yml --- extensions/chsql_native/description.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/extensions/chsql_native/description.yml b/extensions/chsql_native/description.yml index 221798f..1ff5513 100644 --- a/extensions/chsql_native/description.yml +++ b/extensions/chsql_native/description.yml @@ -13,16 +13,15 @@ extension: repo: github: quackscience/duckdb-extension-clickhouse-native - ref: 7d00778fb5f3ba575e33d975faef7f68527d188a + ref: f2e3b6d0c327d71e0989f078b7fc6d13dbac52b9 docs: hello_world: | --- This experimental rust extension implements Native Clickhouse formats for DuckDB. - --- Native Binary Client for chsql - --- CLICKHOUSE_URL="tcp://localhost:9000" - --- CLICKHOUSE_USER="default" - - --- Simple Example + --- ClickHouse Native Binary Client for chsql + --- export CLICKHOUSE_URL="tcp://localhost:9000" + --- export CLICKHOUSE_URL="tcp://user:pass@remote:9440/?secure=true&skip_verify=true" + --- Simple Query Example D SELECT * FROM clickhouse_scan("SELECT version(), 'hello', 123"); ┌────────────┬─────────┬────────┐ │ version() │ 'hello' │ 123 │ @@ -31,7 +30,7 @@ docs: │ 24.10.2.80 │ hello │ 123 │ └────────────┴─────────┴────────┘ - --- Wide Example + --- Wide Query Example D SELECT * FROM clickhouse_scan("SELECT * FROM system.functions WHERE alias_to != '' LIMIT 10"); ┌────────────────────┬──────────────┬──────────────────┬──────────────────────┬───┬───────────┬────────────────┬──────────┬────────────┐ │ name │ is_aggregate │ case_insensitive │ alias_to │ … │ arguments │ returned_value │ examples │ categories │