From 8d07a56c67fe8185f7619d9ab53eacfa271079bf Mon Sep 17 00:00:00 2001 From: Alex Huang Date: Sun, 29 Sep 2024 21:31:31 +0800 Subject: [PATCH 1/9] feat: Add GitHub Actions Workflow to check for typos (#141) * feat: Add GitHub Actions Workflow to check for typos * chore: Fixed typos * chore: Fixed typos --- .codespellignore | 2 ++ .github/workflows/check-typo.yml | 32 ++++++++++++++++++++++++++++++++ CONTRIBUTING.md | 2 +- tests/fixtures/arrow.rs | 2 +- 4 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 .codespellignore create mode 100644 .github/workflows/check-typo.yml diff --git a/.codespellignore b/.codespellignore new file mode 100644 index 00000000..2a20e345 --- /dev/null +++ b/.codespellignore @@ -0,0 +1,2 @@ +crate +socio-economic diff --git a/.github/workflows/check-typo.yml b/.github/workflows/check-typo.yml new file mode 100644 index 00000000..0e176d8d --- /dev/null +++ b/.github/workflows/check-typo.yml @@ -0,0 +1,32 @@ +# workflows/check-typo.yml +# +# Check Typo +# Check Typo using codespell. + +name: Check Typo + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + workflow_dispatch: + +concurrency: + group: check-typo-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + check-typo: + name: Check Typo using codespell + runs-on: depot-ubuntu-latest-2 + if: github.event.pull_request.draft == false + + steps: + - name: Checkout Git Repository + uses: actions/checkout@v4 + + - name: Check Typo using codespell + uses: codespell-project/actions-codespell@v2 + with: + check_filenames: true + ignore_words_file: .codespellignore + skip: "Cargo.lock" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 151043ed..e516cc60 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,7 +37,7 @@ All development of ParadeDB is done via Docker and Compose. Our Docker setup is - The `docker-compose.yml` file pulls the latest published ParadeDB image from DockerHub. It is used for hobby production deployments. We recommend using it to deploy ParadeDB in your own infrastructure. -### Pull Request Worfklow +### Pull Request Workflow All changes to ParadeDB happen through Github Pull Requests. Here is the recommended flow for making a change: diff --git a/tests/fixtures/arrow.rs b/tests/fixtures/arrow.rs index b3b74f25..4e98dd99 100644 --- a/tests/fixtures/arrow.rs +++ b/tests/fixtures/arrow.rs @@ -617,7 +617,7 @@ pub fn schema_to_batch(schema: &SchemaRef, rows: &[PgRow]) -> Result match unit { TimeUnit::Second => bail!("arrow time64i does not support seconds"), - TimeUnit::Millisecond => bail!("arrow time64 does not support millseconds"), + TimeUnit::Millisecond => bail!("arrow time64 does not support milliseconds"), TimeUnit::Microsecond => Arc::new(Time64MicrosecondArray::from( rows.iter() .map(|row| decode::>(field, row)) From b1166df5e3ab2d633a0ea8479c48ac01fa0b8005 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20No=C3=ABl?= <21990816+philippemnoel@users.noreply.github.com> Date: Tue, 1 Oct 2024 17:10:29 -0400 Subject: [PATCH 2/9] Update check-pg_analytics-schema-upgrade.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Philippe Noël <21990816+philippemnoel@users.noreply.github.com> --- .github/workflows/check-pg_analytics-schema-upgrade.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/check-pg_analytics-schema-upgrade.yml b/.github/workflows/check-pg_analytics-schema-upgrade.yml index 90b148ce..de65ea4d 100644 --- a/.github/workflows/check-pg_analytics-schema-upgrade.yml +++ b/.github/workflows/check-pg_analytics-schema-upgrade.yml @@ -84,6 +84,7 @@ # - name: Switch to Base git rev and Generate Schema Again # run: | # # Switch to the base git rev +# git checkout . # git checkout ${{ github.event.pull_request.base.ref }} # # See if we need a different cargo-pgrx and install it if so From e373c707099afe3ccea4ec15921cb000511ed4e2 Mon Sep 17 00:00:00 2001 From: Alex Huang Date: Sat, 5 Oct 2024 14:18:44 +0800 Subject: [PATCH 3/9] Refactor enum serialization in CSV, Delta, Iceberg, Json, Parquet, Secret, and Spatial modules (#145) --- src/duckdb/csv.rs | 35 +---------------------------------- src/duckdb/delta.rs | 4 +--- src/duckdb/iceberg.rs | 5 +---- src/duckdb/json.rs | 18 +----------------- src/duckdb/parquet.rs | 11 +---------- src/duckdb/secret.rs | 23 +---------------------- src/duckdb/spatial.rs | 10 +--------- 7 files changed, 7 insertions(+), 99 deletions(-) diff --git a/src/duckdb/csv.rs b/src/duckdb/csv.rs index fe82de31..c87f9652 100644 --- a/src/duckdb/csv.rs +++ b/src/duckdb/csv.rs @@ -24,74 +24,41 @@ use crate::fdw::base::OptionValidator; use super::utils; #[derive(EnumIter, AsRefStr, PartialEq, Debug)] +#[strum(serialize_all = "snake_case")] pub enum CsvOption { - #[strum(serialize = "all_varchar")] AllVarchar, - #[strum(serialize = "allow_quoted_nulls")] AllowQuotedNulls, - #[strum(serialize = "auto_detect")] AutoDetect, - #[strum(serialize = "auto_type_candidates")] AutoTypeCandidates, - #[strum(serialize = "columns")] Columns, - #[strum(serialize = "compression")] Compression, - #[strum(serialize = "dateformat")] Dateformat, - #[strum(serialize = "decimal_separator")] DecimalSeparator, - #[strum(serialize = "delim")] Delim, - #[strum(serialize = "escape")] Escape, - #[strum(serialize = "filename")] Filename, - #[strum(serialize = "files")] Files, - #[strum(serialize = "force_not_null")] ForceNotNull, - #[strum(serialize = "header")] Header, - #[strum(serialize = "hive_partitioning")] HivePartitioning, - #[strum(serialize = "hive_types")] HiveTypes, - #[strum(serialize = "hive_types_autocast")] HiveTypesAutocast, - #[strum(serialize = "ignore_errors")] IgnoreErrors, - #[strum(serialize = "max_line_size")] MaxLineSize, - #[strum(serialize = "names")] Names, - #[strum(serialize = "new_line")] NewLine, - #[strum(serialize = "normalize_names")] NormalizeNames, - #[strum(serialize = "null_padding")] NullPadding, - #[strum(serialize = "nullstr")] Nullstr, - #[strum(serialize = "parallel")] Parallel, - #[strum(serialize = "preserve_casing")] PreserveCasing, - #[strum(serialize = "quote")] Quote, - #[strum(serialize = "sample_size")] SampleSize, - #[strum(serialize = "select")] Select, - #[strum(serialize = "sep")] Sep, - #[strum(serialize = "skip")] Skip, - #[strum(serialize = "timestampformat")] Timestampformat, - #[strum(serialize = "types")] Types, - #[strum(serialize = "union_by_name")] UnionByName, } diff --git a/src/duckdb/delta.rs b/src/duckdb/delta.rs index 0d95e65a..70c8a09c 100644 --- a/src/duckdb/delta.rs +++ b/src/duckdb/delta.rs @@ -21,12 +21,10 @@ use std::collections::HashMap; use strum::{AsRefStr, EnumIter}; #[derive(EnumIter, AsRefStr, PartialEq, Debug)] +#[strum(serialize_all = "snake_case")] pub enum DeltaOption { - #[strum(serialize = "files")] Files, - #[strum(serialize = "preserve_casing")] PreserveCasing, - #[strum(serialize = "select")] Select, } diff --git a/src/duckdb/iceberg.rs b/src/duckdb/iceberg.rs index 689afc52..75d1d620 100644 --- a/src/duckdb/iceberg.rs +++ b/src/duckdb/iceberg.rs @@ -22,14 +22,11 @@ use strum::{AsRefStr, EnumIter}; use crate::fdw::base::OptionValidator; #[derive(EnumIter, AsRefStr, PartialEq, Debug)] +#[strum(serialize_all = "snake_case")] pub enum IcebergOption { - #[strum(serialize = "allow_moved_paths")] AllowMovedPaths, - #[strum(serialize = "files")] Files, - #[strum(serialize = "preserve_casing")] PreserveCasing, - #[strum(serialize = "select")] Select, } diff --git a/src/duckdb/json.rs b/src/duckdb/json.rs index 2b5bcf45..0772cdd6 100644 --- a/src/duckdb/json.rs +++ b/src/duckdb/json.rs @@ -24,40 +24,24 @@ use crate::fdw::base::OptionValidator; use super::utils; #[derive(EnumIter, AsRefStr, PartialEq, Debug, Display)] +#[strum(serialize_all = "snake_case")] pub enum JsonOption { - #[strum(serialize = "auto_detect")] AutoDetect, - #[strum(serialize = "columns")] Columns, - #[strum(serialize = "compression")] Compression, - #[strum(serialize = "convert_strings_to_integers")] ConvertStringsToIntegers, - #[strum(serialize = "dateformat")] Dateformat, - #[strum(serialize = "filename")] Filename, - #[strum(serialize = "files")] Files, - #[strum(serialize = "format")] Format, - #[strum(serialize = "hive_partitioning")] HivePartitioning, - #[strum(serialize = "ignore_errors")] IgnoreErrors, - #[strum(serialize = "maximum_depth")] MaximumDepth, - #[strum(serialize = "maximum_object_size")] MaximumObjectSize, - #[strum(serialize = "records")] Records, - #[strum(serialize = "sample_size")] SampleSize, - #[strum(serialize = "select")] Select, - #[strum(serialize = "timestampformat")] Timestampformat, - #[strum(serialize = "union_by_name")] UnionByName, } diff --git a/src/duckdb/parquet.rs b/src/duckdb/parquet.rs index 96e45ea2..dc515415 100644 --- a/src/duckdb/parquet.rs +++ b/src/duckdb/parquet.rs @@ -24,26 +24,17 @@ use crate::fdw::base::OptionValidator; use super::utils; #[derive(EnumIter, AsRefStr, PartialEq, Debug)] +#[strum(serialize_all = "snake_case")] pub enum ParquetOption { - #[strum(serialize = "binary_as_string")] BinaryAsString, - #[strum(serialize = "filename")] FileName, - #[strum(serialize = "file_row_number")] FileRowNumber, - #[strum(serialize = "files")] Files, - #[strum(serialize = "hive_partitioning")] HivePartitioning, - #[strum(serialize = "hive_types")] HiveTypes, - #[strum(serialize = "hive_types_autocast")] HiveTypesAutocast, - #[strum(serialize = "preserve_casing")] PreserveCasing, - #[strum(serialize = "union_by_name")] UnionByName, - #[strum(serialize = "select")] Select, // TODO: EncryptionConfig } diff --git a/src/duckdb/secret.rs b/src/duckdb/secret.rs index 81104152..c9a5ae83 100644 --- a/src/duckdb/secret.rs +++ b/src/duckdb/secret.rs @@ -22,53 +22,32 @@ use strum::{AsRefStr, EnumIter}; use crate::fdw::base::OptionValidator; #[derive(EnumIter, AsRefStr, PartialEq, Debug)] +#[strum(serialize_all = "snake_case")] pub enum UserMappingOptions { // Universal - #[strum(serialize = "type")] Type, - #[strum(serialize = "provider")] Provider, - #[strum(serialize = "scope")] Scope, - #[strum(serialize = "chain")] Chain, // S3/GCS/R2 - #[strum(serialize = "key_id")] KeyId, - #[strum(serialize = "secret")] Secret, - #[strum(serialize = "region")] Region, - #[strum(serialize = "session_token")] SessionToken, - #[strum(serialize = "endpoint")] Endpoint, - #[strum(serialize = "url_style")] UrlStyle, - #[strum(serialize = "use_ssl")] UseSsl, - #[strum(serialize = "url_compatibility_mode")] UrlCompatibilityMode, - #[strum(serialize = "account_id")] AccountId, // Azure - #[strum(serialize = "connection_string")] ConnectionString, - #[strum(serialize = "account_name")] AccountName, - #[strum(serialize = "tenant_id")] TenantId, - #[strum(serialize = "client_id")] ClientId, - #[strum(serialize = "client_secret")] ClientSecret, - #[strum(serialize = "client_certificate_path")] ClientCertificatePath, - #[strum(serialize = "http_proxy")] HttpProxy, - #[strum(serialize = "proxy_user_name")] ProxyUserName, - #[strum(serialize = "proxy_password")] ProxyPassword, } diff --git a/src/duckdb/spatial.rs b/src/duckdb/spatial.rs index b0d54e24..cd76239b 100644 --- a/src/duckdb/spatial.rs +++ b/src/duckdb/spatial.rs @@ -25,24 +25,16 @@ use crate::fdw::base::OptionValidator; /// SpatialOption is an enum that represents the options that can be passed to the st_read function. /// Reference https://github.com/duckdb/duckdb_spatial/blob/main/docs/functions.md#st_read #[derive(EnumIter, AsRefStr, PartialEq, Debug)] +#[strum(serialize_all = "snake_case")] pub enum SpatialOption { - #[strum(serialize = "files")] Files, - #[strum(serialize = "sequential_layer_scan")] SequentialLayerScan, - #[strum(serialize = "spatial_filter")] SpatialFilter, - #[strum(serialize = "open_options")] OpenOptions, - #[strum(serialize = "layer")] Layer, - #[strum(serialize = "allowed_drivers")] AllowedDrivers, - #[strum(serialize = "sibling_files")] SiblingFiles, - #[strum(serialize = "spatial_filter_box")] SpatialFilterBox, - #[strum(serialize = "keep_wkb")] KeepWkb, } From 0a47f6472d1724df88f13a1663e44a0b34983e61 Mon Sep 17 00:00:00 2001 From: rakesh-tmdc <105636602+rakesh-tmdc@users.noreply.github.com> Date: Tue, 8 Oct 2024 06:42:34 +0530 Subject: [PATCH 4/9] fix: iceberg metadata comp codec (#143) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * FIX: iceberg meta data compression coded supppor * FIX: iceberg meta data compression coded supppor * FIX: added skip_schema_inference for iceberg * Lint Signed-off-by: Philippe Noël <21990816+philippemnoel@users.noreply.github.com> * Update iceberg.rs Signed-off-by: Philippe Noël <21990816+philippemnoel@users.noreply.github.com> --------- Signed-off-by: Philippe Noël <21990816+philippemnoel@users.noreply.github.com> Co-authored-by: Philippe Noël <21990816+philippemnoel@users.noreply.github.com> --- src/duckdb/iceberg.rs | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/src/duckdb/iceberg.rs b/src/duckdb/iceberg.rs index 75d1d620..a803ae56 100644 --- a/src/duckdb/iceberg.rs +++ b/src/duckdb/iceberg.rs @@ -25,6 +25,8 @@ use crate::fdw::base::OptionValidator; #[strum(serialize_all = "snake_case")] pub enum IcebergOption { AllowMovedPaths, + MetadataCompressionCodec, + SkipSchemaInference, Files, PreserveCasing, Select, @@ -34,6 +36,8 @@ impl OptionValidator for IcebergOption { fn is_required(&self) -> bool { match self { Self::AllowMovedPaths => false, + Self::MetadataCompressionCodec => false, + Self::SkipSchemaInference => false, Self::Files => true, Self::PreserveCasing => false, Self::Select => false, @@ -57,11 +61,24 @@ pub fn create_view( .get(IcebergOption::AllowMovedPaths.as_ref()) .map(|option| format!("allow_moved_paths = {option}")); - let create_iceberg_str = [files, allow_moved_paths] - .into_iter() - .flatten() - .collect::>() - .join(", "); + let metadata_compression_codec = table_options + .get(IcebergOption::MetadataCompressionCodec.as_ref()) + .map(|option| format!("metadata_compression_codec = '{option}'")); + + let skip_schema_inference = table_options + .get(IcebergOption::SkipSchemaInference.as_ref()) + .map(|option| format!("skip_schema_inference = {option}")); + + let create_iceberg_str = [ + files, + allow_moved_paths, + metadata_compression_codec, + skip_schema_inference, + ] + .into_iter() + .flatten() + .collect::>() + .join(", "); let default_select = "*".to_string(); let select = table_options From 5b14368c87a92a03b0903dc3b17d65f87b615e9e Mon Sep 17 00:00:00 2001 From: Petr Kada <63555269+pert5432@users.noreply.github.com> Date: Tue, 8 Oct 2024 19:41:13 +0200 Subject: [PATCH 5/9] feat: support pg13 (#134) * run tests on pg13 * print pg logs to debug * always * fetch schema_name if its null * stop logging postgres logs * newline * re-enable schema check workflow * add pg13 to publish workflow * update readme * echo schema diff * only use current schema query on pg13 * fix lint issue --- .../check-pg_analytics-schema-upgrade.yml | 309 +++++++++--------- .github/workflows/publish-pg_analytics.yml | 40 +++ .github/workflows/test-pg_analytics.yml | 3 + README.md | 6 +- src/fdw/trigger.rs | 18 + 5 files changed, 218 insertions(+), 158 deletions(-) diff --git a/.github/workflows/check-pg_analytics-schema-upgrade.yml b/.github/workflows/check-pg_analytics-schema-upgrade.yml index de65ea4d..939485e0 100644 --- a/.github/workflows/check-pg_analytics-schema-upgrade.yml +++ b/.github/workflows/check-pg_analytics-schema-upgrade.yml @@ -1,155 +1,154 @@ -# TODO: Uncomment once we have PG13 support, which is required for pg-schema-diff -# # workflows/check-pg_analytics-schema-upgrade.yml -# # -# # Check pg_analytics Schema Upgrade pg_analytics -# # Determine if a commit introduces an extension schema change for pg_analytics. - -# name: Check pg_analytics Schema Upgrade - -# on: -# pull_request: -# types: [opened, synchronize, reopened, ready_for_review] -# branches: -# - dev -# - main -# paths: -# - ".github/workflows/check-pg_analytics-schema-upgrade.yml" -# - "pg_analytics/**" -# - "!pg_analytics/README.md" -# workflow_dispatch: - -# # Required to post a comment to the PR -# permissions: -# pull-requests: write - -# concurrency: -# group: check-pg_analytics-schema-upgrade-${{ github.head_ref || github.ref }} -# cancel-in-progress: true - -# jobs: -# check-pg_analytics-schema-upgrade: -# name: Post Schema Change to PR -# runs-on: depot-ubuntu-latest-8 -# if: github.event.pull_request.draft == false -# env: -# pg_version: 13 # Required by pg-schema-diff - -# steps: -# - name: Checkout Git Repository -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 # Fetch the entire history - -# - name: Install Rust -# uses: dtolnay/rust-toolchain@stable - -# # Caches from base branches are available to PRs, but not across unrelated branches, so we only -# # save the cache on the 'dev' branch, but load it on all branches. -# - name: Install Rust Cache -# uses: Swatinem/rust-cache@v2 -# with: -# prefix-key: "v1" -# shared-key: ${{ runner.os }}-rust-cache-pg_analytics-${{ HashFiles('Cargo.lock') }} -# cache-targets: true -# cache-on-failure: true -# cache-all-crates: true -# save-if: ${{ github.ref == 'refs/heads/dev' }} - -# - name: Install & Configure Supported PostgreSQL Version -# run: | -# wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - -# sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' -# sudo apt-get update && sudo apt-get install -y postgresql-${{ env.pg_version }} postgresql-server-dev-${{ env.pg_version }} -# sudo chown -R $(whoami) /usr/share/postgresql/${{ env.pg_version }}/ /usr/lib/postgresql/${{ env.pg_version }}/ /var/lib/postgresql/${{ env.pg_version }}/ -# rustup component add llvm-tools-preview -# echo "/usr/lib/postgresql/${{ env.pg_version }}/bin" >> $GITHUB_PATH - -# - name: Install pg-schema-diff and its Required Dependencies -# run: | -# sudo apt install clang llvm diffutils -# cargo install --git https://github.com/zombodb/pg-schema-diff.git - -# - name: Extract pgrx Version & Install cargo-pgrx -# run: | -# PGRX_VERSION=$(cargo tree --depth 1 -i pgrx -p pg_analytics | head -n 1 | cut -f2 -dv) -# cargo install -j $(nproc) --locked cargo-pgrx --version ${PGRX_VERSION} -# cargo pgrx init "--pg${{ env.pg_version }}=/usr/lib/postgresql/${{ env.pg_version }}/bin/pg_config" - -# # Save the pgrx version for comparison later -# echo "FIRST_PGRX_VERSION=${PGRX_VERSION}" >> $GITHUB_ENV - -# - name: Generate Schema from this git rev -# run: cargo pgrx schema -p pg_analytics pg${{ env.pg_version }} > ~/this.sql - -# - name: Switch to Base git rev and Generate Schema Again -# run: | -# # Switch to the base git rev -# git checkout . -# git checkout ${{ github.event.pull_request.base.ref }} - -# # See if we need a different cargo-pgrx and install it if so -# THIS_PGRX_VERSION=$(cargo tree --depth 1 -i pgrx -p pg_analytics | head -n 1 | cut -f2 -dv) -# if [[ "${THIS_PGRX_VERSION}" != "${FIRST_PGRX_VERSION}" ]]; then -# # Install cargo-pgrx -# cargo install -j $(nproc) --locked cargo-pgrx --version ${THIS_PGRX_VERSION} --force - -# # Initialize it (again) -- probably unnecessary, but might as well in case ~/.pgrx/config.toml ever changes -# cargo pgrx init "--pg${{ env.pg_version }}=/usr/lib/postgresql/${{ env.pg_version }}/bin/pg_config" -# fi - -# # Generate schema -# cargo pgrx schema -p pg_analytics pg${{ env.pg_version }} > ~/old.sql - -# - name: Generate Schema Diffs -# run: | -# (pg-schema-diff diff ~/old.sql ~/this.sql | grep -v "^$" > ~/diff.sql) || true -# (diff ~/old.sql ~/this.sql > ~/diff.patch) || true - -# - name: Generate Commit Message -# id: generate_commit_message -# run: | -# if test -s ~/diff.sql; then -# echo "Generating GitHub comment message" -# { -# echo 'DIFF<> "$GITHUB_ENV" - -# # Set a flag to indicate a schema difference was detected -# echo "schema_diff_detected=true" >> $GITHUB_OUTPUT -# else -# echo "No schema difference detected" -# echo "schema_diff_detected=false" >> $GITHUB_OUTPUT -# fi - -# - name: Attach Schema Diff to PR -# uses: actions/github-script@v6 -# if: steps.generate_commit_message.outputs.schema_diff_detected == 'true' -# with: -# script: | -# github.rest.issues.createComment({ -# issue_number: context.issue.number, -# owner: context.repo.owner, -# repo: context.repo.repo, -# body: process.env.DIFF -# }) +# workflows/check-pg_analytics-schema-upgrade.yml +# +# Check pg_analytics Schema Upgrade pg_analytics +# Determine if a commit introduces an extension schema change for pg_analytics. + +name: Check pg_analytics Schema Upgrade + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + branches: + - dev + - main + paths: + - ".github/workflows/check-pg_analytics-schema-upgrade.yml" + - "pg_analytics/**" + - "!pg_analytics/README.md" + workflow_dispatch: + +# Required to post a comment to the PR +permissions: + pull-requests: write + +concurrency: + group: check-pg_analytics-schema-upgrade-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + check-pg_analytics-schema-upgrade: + name: Post Schema Change to PR + runs-on: depot-ubuntu-latest-8 + if: github.event.pull_request.draft == false + env: + pg_version: 13 # Required by pg-schema-diff + + steps: + - name: Checkout Git Repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch the entire history + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + # Caches from base branches are available to PRs, but not across unrelated branches, so we only + # save the cache on the 'dev' branch, but load it on all branches. + - name: Install Rust Cache + uses: Swatinem/rust-cache@v2 + with: + prefix-key: "v1" + shared-key: ${{ runner.os }}-rust-cache-pg_analytics-${{ HashFiles('Cargo.lock') }} + cache-targets: true + cache-on-failure: true + cache-all-crates: true + save-if: ${{ github.ref == 'refs/heads/dev' }} + + - name: Install & Configure Supported PostgreSQL Version + run: | + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + sudo apt-get update && sudo apt-get install -y postgresql-${{ env.pg_version }} postgresql-server-dev-${{ env.pg_version }} + sudo chown -R $(whoami) /usr/share/postgresql/${{ env.pg_version }}/ /usr/lib/postgresql/${{ env.pg_version }}/ /var/lib/postgresql/${{ env.pg_version }}/ + rustup component add llvm-tools-preview + echo "/usr/lib/postgresql/${{ env.pg_version }}/bin" >> $GITHUB_PATH + + - name: Install pg-schema-diff and its Required Dependencies + run: | + sudo apt install clang llvm diffutils + cargo install --git https://github.com/zombodb/pg-schema-diff.git + + - name: Extract pgrx Version & Install cargo-pgrx + run: | + PGRX_VERSION=$(cargo tree --depth 1 -i pgrx -p pg_analytics | head -n 1 | cut -f2 -dv) + cargo install -j $(nproc) --locked cargo-pgrx --version ${PGRX_VERSION} + cargo pgrx init "--pg${{ env.pg_version }}=/usr/lib/postgresql/${{ env.pg_version }}/bin/pg_config" + + # Save the pgrx version for comparison later + echo "FIRST_PGRX_VERSION=${PGRX_VERSION}" >> $GITHUB_ENV + + - name: Generate Schema from this git rev + run: cargo pgrx schema -p pg_analytics pg${{ env.pg_version }} > ~/this.sql + + - name: Switch to Base git rev and Generate Schema Again + run: | + # Switch to the base git rev + git checkout . + git checkout ${{ github.event.pull_request.base.ref }} + + # See if we need a different cargo-pgrx and install it if so + THIS_PGRX_VERSION=$(cargo tree --depth 1 -i pgrx -p pg_analytics | head -n 1 | cut -f2 -dv) + if [[ "${THIS_PGRX_VERSION}" != "${FIRST_PGRX_VERSION}" ]]; then + # Install cargo-pgrx + cargo install -j $(nproc) --locked cargo-pgrx --version ${THIS_PGRX_VERSION} --force + + # Initialize it (again) -- probably unnecessary, but might as well in case ~/.pgrx/config.toml ever changes + cargo pgrx init "--pg${{ env.pg_version }}=/usr/lib/postgresql/${{ env.pg_version }}/bin/pg_config" + fi + + # Generate schema + cargo pgrx schema -p pg_analytics pg${{ env.pg_version }} > ~/old.sql + + - name: Generate Schema Diffs + run: | + (pg-schema-diff diff ~/old.sql ~/this.sql | grep -v "^$" > ~/diff.sql) || true + (diff ~/old.sql ~/this.sql > ~/diff.patch) || true + + - name: Generate Commit Message + id: generate_commit_message + run: | + if test -s ~/diff.sql; then + echo "Generating GitHub comment message" + { + echo 'DIFF<' + echo 'The full diff between both schemas is:' + echo + echo '```diff' + cat ~/diff.patch + echo '```' + echo '' + echo EOF + } >> "$GITHUB_ENV" + + # Set a flag to indicate a schema difference was detected + echo "schema_diff_detected=true" >> $GITHUB_OUTPUT + else + echo "No schema difference detected" + echo "schema_diff_detected=false" >> $GITHUB_OUTPUT + fi + + - name: Attach Schema Diff to PR + uses: actions/github-script@v6 + if: steps.generate_commit_message.outputs.schema_diff_detected == 'true' + with: + script: | + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: process.env.DIFF + }) diff --git a/.github/workflows/publish-pg_analytics.yml b/.github/workflows/publish-pg_analytics.yml index 45f60e6a..9eb94798 100644 --- a/.github/workflows/publish-pg_analytics.yml +++ b/.github/workflows/publish-pg_analytics.yml @@ -35,6 +35,14 @@ jobs: matrix: include: # Ubuntu 22.04 + - runner: depot-ubuntu-latest-8 + image: ubuntu:22.04 + pg_version: 13 + arch: amd64 + - runner: depot-ubuntu-latest-arm-4 + image: ubuntu:22.04 + pg_version: 13 + arch: arm64 - runner: depot-ubuntu-latest-8 image: ubuntu:22.04 pg_version: 14 @@ -68,6 +76,14 @@ jobs: pg_version: 17 arch: arm64 # Ubuntu 24.04 + - runner: depot-ubuntu-latest-8 + image: ubuntu:24.04 + pg_version: 13 + arch: amd64 + - runner: depot-ubuntu-latest-arm-4 + image: ubuntu:24.04 + pg_version: 13 + arch: arm64 - runner: depot-ubuntu-latest-8 image: ubuntu:24.04 pg_version: 14 @@ -101,6 +117,14 @@ jobs: pg_version: 17 arch: arm64 # Debian 12 + - runner: depot-ubuntu-latest-8 + image: debian:12-slim + pg_version: 13 + arch: amd64 + - runner: depot-ubuntu-latest-arm-4 + image: debian:12-slim + pg_version: 13 + arch: arm64 - runner: depot-ubuntu-latest-8 image: debian:12-slim pg_version: 14 @@ -134,6 +158,14 @@ jobs: pg_version: 17 arch: arm64 # Red Hat Enterprise Linux 8 + - runner: depot-ubuntu-latest-8 + image: redhat/ubi8:latest + pg_version: 13 + arch: amd64 + - runner: depot-ubuntu-latest-arm-4 + image: redhat/ubi8:latest + pg_version: 13 + arch: arm64 - runner: depot-ubuntu-latest-8 image: redhat/ubi8:latest pg_version: 14 @@ -167,6 +199,14 @@ jobs: pg_version: 17 arch: arm64 # Red Hat Enterprise Linux 9 + - runner: depot-ubuntu-latest-8 + image: redhat/ubi9:latest + pg_version: 13 + arch: amd64 + - runner: depot-ubuntu-latest-arm-4 + image: redhat/ubi9:latest + pg_version: 13 + arch: arm64 - runner: depot-ubuntu-latest-8 image: redhat/ubi9:latest pg_version: 14 diff --git a/.github/workflows/test-pg_analytics.yml b/.github/workflows/test-pg_analytics.yml index 1b3772b5..572d6fe8 100644 --- a/.github/workflows/test-pg_analytics.yml +++ b/.github/workflows/test-pg_analytics.yml @@ -39,6 +39,9 @@ jobs: strategy: matrix: include: + - runner: depot-ubuntu-latest-16 + pg_version: 13 + arch: amd64 - runner: depot-ubuntu-latest-16 pg_version: 14 arch: amd64 diff --git a/README.md b/README.md index 482fb7e1..a76bbec6 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ Today, a vast amount of non-operational data — events, metrics, historical sna - [x] Delta Lake - [x] JSON -`pg_analytics` uses DuckDB v1.0.0 and is supported on Postgres 14, 15, 16 and 17. Support for Postgres 13 is coming soon. +`pg_analytics` uses DuckDB v1.0.0 and is supported on Postgres 17, 16, 15, 14 and 13. ## Installation @@ -85,7 +85,7 @@ This ensures the best query performance from the extension . #### Debian/Ubuntu -We provide prebuilt binaries for Debian-based Linux for Postgres 17, 16, 15 and 14. You can download the latest version for your architecture from the [releases page](https://github.com/paradedb/paradedb/releases). +We provide prebuilt binaries for Debian-based Linux for Postgres 17, 16, 15, 14 and 13. You can download the latest version for your architecture from the [releases page](https://github.com/paradedb/paradedb/releases). ParadeDB collects anonymous telemetry to help us understand how many people are using the project. You can opt out of telemetry by setting `export PARADEDB_TELEMETRY=false` (or unsetting the variable) in your shell or in your `~/.bashrc` file before running the extension. @@ -198,7 +198,7 @@ export PATH="$PATH:/Applications/Postgres.app/Contents/Versions/latest/bin" Then, install and initialize `pgrx`: ```bash -# Note: Replace --pg17 with your version of Postgres, if different (i.e. --pg16, --pg15, --pg14, etc.) +# Note: Replace --pg17 with your version of Postgres, if different (i.e. --pg17, --pg16, --pg15, --pg14, --pg13 etc.) cargo install --locked cargo-pgrx --version 0.12.5 # macOS arm64 diff --git a/src/fdw/trigger.rs b/src/fdw/trigger.rs index 9b900963..7ac2f334 100644 --- a/src/fdw/trigger.rs +++ b/src/fdw/trigger.rs @@ -93,7 +93,25 @@ unsafe fn auto_create_schema_impl(fcinfo: pg_sys::FunctionCallInfo) -> Result<() // Get relation name, oid, etc. that triggered the event let relation = create_stmt.relation; + + #[cfg(feature = "pg13")] + // Fetch the current schema name if the pointer to it is null + // This pointer is null in pg13 when the user doesn't specify the schema in the CREATE FOREIGN TABLE statement + let schema_name = match ((*relation).schemaname as *const std::ffi::c_char).is_null() { + true => Spi::connect(|client| { + client + // Casting as TEXT because the default return type is name (Oid(19)) and the conversion to Rust string fails + .select("SELECT CURRENT_SCHEMA::TEXT", None, None)? + .first() + .get_one() + })? + .unwrap(), + false => CStr::from_ptr((*relation).schemaname).to_str()?, + }; + + #[cfg(not(feature = "pg13"))] let schema_name = CStr::from_ptr((*relation).schemaname).to_str()?; + let table_name = CStr::from_ptr((*relation).relname).to_str()?; let oid = pg_sys::RangeVarGetRelidExtended( relation, From c2cb9309bfda6652b354f2a8d1d2fa082c66f4ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20No=C3=ABl?= <21990816+philippemnoel@users.noreply.github.com> Date: Thu, 10 Oct 2024 23:50:53 -0400 Subject: [PATCH 6/9] chore: Update schema upgrade comment instead of overwriting (#151) * Schema upgrade * Lint --- .../check-pg_analytics-schema-upgrade.yml | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/.github/workflows/check-pg_analytics-schema-upgrade.yml b/.github/workflows/check-pg_analytics-schema-upgrade.yml index 939485e0..9600e375 100644 --- a/.github/workflows/check-pg_analytics-schema-upgrade.yml +++ b/.github/workflows/check-pg_analytics-schema-upgrade.yml @@ -142,13 +142,32 @@ jobs: fi - name: Attach Schema Diff to PR - uses: actions/github-script@v6 + uses: actions/github-script@v7 if: steps.generate_commit_message.outputs.schema_diff_detected == 'true' with: script: | - github.rest.issues.createComment({ + const comments = await github.rest.issues.listComments({ issue_number: context.issue.number, owner: context.repo.owner, - repo: context.repo.repo, - body: process.env.DIFF - }) + repo: context.repo.repo + }); + + const botComment = comments.data.find(comment => comment.user.type === 'Bot' && comment.body.includes('A schema difference was detected.')); + + if (botComment) { + // Update the existing comment + await github.rest.issues.updateComment({ + comment_id: botComment.id, + owner: context.repo.owner, + repo: context.repo.repo, + body: process.env.DIFF + }); + } else { + // Create a new comment if none exists + await github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: process.env.DIFF + }); + } From 9a218e5c282186fb27cacc721d03401c1a1c26e6 Mon Sep 17 00:00:00 2001 From: Siyuan Huang <73871299+kysshsy@users.noreply.github.com> Date: Sun, 13 Oct 2024 19:57:59 +0800 Subject: [PATCH 7/9] feat: support prepare pushdown (#140) * feat: support prepare statement * test: add prepare test * feat: add Query plan check * fix: run fmt * feat: change the way to get cached plan * fix: run fmt * fix: pg13 compile * fix: fix compile * fix: replan in Duckdb when search_path changed * fix: pg17 function name change * test: add search path test comment: add test comment * fix: change warning message url * fix: fix pg13 compile * update base on comment --- src/hooks/executor.rs | 74 +----------- src/hooks/mod.rs | 3 +- src/hooks/query.rs | 73 +++++++++++- src/hooks/utility.rs | 94 +++++++++------- src/hooks/utility/explain.rs | 61 ++++++++++ src/hooks/utility/prepare.rs | 211 +++++++++++++++++++++++++++++++++++ tests/fixtures/arrow.rs | 82 +++++++++++++- tests/scan.rs | 96 +++++++++++++++- 8 files changed, 570 insertions(+), 124 deletions(-) create mode 100644 src/hooks/utility/explain.rs create mode 100644 src/hooks/utility/prepare.rs diff --git a/src/hooks/executor.rs b/src/hooks/executor.rs index 99c73d0e..d28ded06 100644 --- a/src/hooks/executor.rs +++ b/src/hooks/executor.rs @@ -15,25 +15,17 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use anyhow::{anyhow, Result}; -use duckdb::arrow::array::RecordBatch; +use anyhow::Result; use pgrx::*; use std::ffi::CStr; use crate::duckdb::connection; -use crate::schema::cell::*; use super::query::*; #[cfg(debug_assertions)] use crate::DEBUG_GUCS; -macro_rules! fallback_warning { - ($msg:expr) => { - warning!("This query was not fully pushed down to DuckDB because DuckDB returned an error. Query times may be impacted. If you would like to see this query pushed down, please submit a request to https://github.com/paradedb/paradedb/issues with the following context:\n{}", $msg); - }; -} - #[allow(deprecated)] pub async fn executor_run( query_desc: PgBox, @@ -66,6 +58,7 @@ pub async fn executor_run( // Tech Debt: Find a less hacky way to let COPY/CREATE go through || query.to_lowercase().starts_with("copy") || query.to_lowercase().starts_with("create") + || query.to_lowercase().starts_with("prepare") { prev_hook(query_desc, direction, count, execute_once); return Ok(()); @@ -102,66 +95,3 @@ pub async fn executor_run( connection::clear_arrow(); Ok(()) } - -#[inline] -fn write_batches_to_slots( - query_desc: PgBox, - mut batches: Vec, -) -> Result<()> { - // Convert the DataFusion batches to Postgres tuples and send them to the destination - unsafe { - let tuple_desc = PgTupleDesc::from_pg(query_desc.tupDesc); - let estate = query_desc.estate; - (*estate).es_processed = 0; - - let dest = query_desc.dest; - let startup = (*dest) - .rStartup - .ok_or_else(|| anyhow!("rStartup not found"))?; - startup(dest, query_desc.operation as i32, query_desc.tupDesc); - - let receive = (*dest) - .receiveSlot - .ok_or_else(|| anyhow!("receiveSlot not found"))?; - - for batch in batches.iter_mut() { - for row_index in 0..batch.num_rows() { - let tuple_table_slot = - pg_sys::MakeTupleTableSlot(query_desc.tupDesc, &pg_sys::TTSOpsVirtual); - - pg_sys::ExecStoreVirtualTuple(tuple_table_slot); - - for (col_index, _) in tuple_desc.iter().enumerate() { - let attribute = tuple_desc - .get(col_index) - .ok_or_else(|| anyhow!("attribute at {col_index} not found in tupdesc"))?; - let column = batch.column(col_index); - let tts_value = (*tuple_table_slot).tts_values.add(col_index); - let tts_isnull = (*tuple_table_slot).tts_isnull.add(col_index); - - match column.get_cell(row_index, attribute.atttypid, attribute.name())? { - Some(cell) => { - if let Some(datum) = cell.into_datum() { - *tts_value = datum; - } - } - None => { - *tts_isnull = true; - } - }; - } - - receive(tuple_table_slot, dest); - (*estate).es_processed += 1; - pg_sys::ExecDropSingleTupleTableSlot(tuple_table_slot); - } - } - - let shutdown = (*dest) - .rShutdown - .ok_or_else(|| anyhow!("rShutdown not found"))?; - shutdown(dest); - } - - Ok(()) -} diff --git a/src/hooks/mod.rs b/src/hooks/mod.rs index 73a87ff0..c0065299 100644 --- a/src/hooks/mod.rs +++ b/src/hooks/mod.rs @@ -15,8 +15,9 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -mod executor; +#[macro_use] mod query; +mod executor; mod utility; use async_std::task::block_on; diff --git a/src/hooks/query.rs b/src/hooks/query.rs index 92e84a12..863a2f97 100644 --- a/src/hooks/query.rs +++ b/src/hooks/query.rs @@ -15,13 +15,21 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use anyhow::Result; +use anyhow::{anyhow, Result}; +use duckdb::arrow::array::RecordBatch; use pgrx::*; use std::ffi::CStr; use std::str::Utf8Error; use crate::duckdb::connection; use crate::fdw::handler::FdwHandler; +use crate::schema::cell::*; + +macro_rules! fallback_warning { + ($msg:expr) => { + warning!("This query was not fully pushed down to DuckDB because DuckDB returned an error. Query times may be impacted. If you would like to see this query pushed down, please submit a request to https://github.com/paradedb/pg_analytics/issues with the following context:\n{}", $msg); + }; +} pub fn get_current_query( planned_stmt: *mut pg_sys::PlannedStmt, @@ -121,3 +129,66 @@ pub fn is_duckdb_query(relations: &[PgRelation]) -> bool { } }) } + +#[inline] +pub fn write_batches_to_slots( + query_desc: PgBox, + mut batches: Vec, +) -> Result<()> { + // Convert the DataFusion batches to Postgres tuples and send them to the destination + unsafe { + let tuple_desc = PgTupleDesc::from_pg(query_desc.tupDesc); + let estate = query_desc.estate; + (*estate).es_processed = 0; + + let dest = query_desc.dest; + let startup = (*dest) + .rStartup + .ok_or_else(|| anyhow!("rStartup not found"))?; + startup(dest, query_desc.operation as i32, query_desc.tupDesc); + + let receive = (*dest) + .receiveSlot + .ok_or_else(|| anyhow!("receiveSlot not found"))?; + + for batch in batches.iter_mut() { + for row_index in 0..batch.num_rows() { + let tuple_table_slot = + pg_sys::MakeTupleTableSlot(query_desc.tupDesc, &pg_sys::TTSOpsVirtual); + + pg_sys::ExecStoreVirtualTuple(tuple_table_slot); + + for (col_index, _) in tuple_desc.iter().enumerate() { + let attribute = tuple_desc + .get(col_index) + .ok_or_else(|| anyhow!("attribute at {col_index} not found in tupdesc"))?; + let column = batch.column(col_index); + let tts_value = (*tuple_table_slot).tts_values.add(col_index); + let tts_isnull = (*tuple_table_slot).tts_isnull.add(col_index); + + match column.get_cell(row_index, attribute.atttypid, attribute.name())? { + Some(cell) => { + if let Some(datum) = cell.into_datum() { + *tts_value = datum; + } + } + None => { + *tts_isnull = true; + } + }; + } + + receive(tuple_table_slot, dest); + (*estate).es_processed += 1; + pg_sys::ExecDropSingleTupleTableSlot(tuple_table_slot); + } + } + + let shutdown = (*dest) + .rShutdown + .ok_or_else(|| anyhow!("rShutdown not found"))?; + shutdown(dest); + } + + Ok(()) +} diff --git a/src/hooks/utility.rs b/src/hooks/utility.rs index 70362e04..d53dbd41 100644 --- a/src/hooks/utility.rs +++ b/src/hooks/utility.rs @@ -16,17 +16,19 @@ // along with this program. If not, see . #![allow(clippy::too_many_arguments)] +#![allow(deprecated)] +mod explain; +mod prepare; -use std::ffi::CString; +use std::ptr::null_mut; use anyhow::{bail, Result}; -use pg_sys::NodeTag; -use pgrx::*; +use pgrx::{pg_sys, AllocatedByRust, HookResult, PgBox}; use sqlparser::{ast::Statement, dialect::PostgreSqlDialect, parser::Parser}; -use super::query::*; +use explain::explain_query; +use prepare::*; -#[allow(deprecated)] type ProcessUtilityHook = fn( pstmt: PgBox, query_string: &core::ffi::CStr, @@ -66,7 +68,47 @@ pub async fn process_utility_hook( return Ok(()); } + let parse_state = unsafe { + let state = pg_sys::make_parsestate(null_mut()); + (*state).p_sourcetext = query_string.as_ptr(); + (*state).p_queryEnv = query_env.as_ptr(); + state + }; + let need_exec_prev_hook = match stmt_type { + pg_sys::NodeTag::T_PrepareStmt => prepare_query( + parse_state, + pstmt.utilityStmt as *mut pg_sys::PrepareStmt, + pstmt.stmt_location, + pstmt.stmt_len, + )?, + + pg_sys::NodeTag::T_ExecuteStmt => { + let mut query_desc = unsafe { + PgBox::::from_rust(pg_sys::CreateQueryDesc( + pstmt.as_ptr(), + query_string.as_ptr(), + null_mut(), + null_mut(), + dest.as_ptr(), + null_mut(), + query_env.as_ptr(), + 0, + )) + }; + query_desc.estate = unsafe { pg_sys::CreateExecutorState() }; + + execute_query( + parse_state, + pstmt.utilityStmt as *mut pg_sys::ExecuteStmt, + query_desc, + )? + } + + pg_sys::NodeTag::T_DeallocateStmt => { + deallocate_query(pstmt.utilityStmt as *mut pg_sys::DeallocateStmt)? + } + pg_sys::NodeTag::T_ExplainStmt => explain_query( query_string, pstmt.utilityStmt as *mut pg_sys::ExplainStmt, @@ -91,45 +133,11 @@ pub async fn process_utility_hook( Ok(()) } -fn is_support_utility(stmt_type: NodeTag) -> bool { +fn is_support_utility(stmt_type: pg_sys::NodeTag) -> bool { stmt_type == pg_sys::NodeTag::T_ExplainStmt -} - -fn explain_query( - query_string: &core::ffi::CStr, - stmt: *mut pg_sys::ExplainStmt, - dest: *mut pg_sys::DestReceiver, -) -> Result { - let query = unsafe { (*stmt).query as *mut pg_sys::Query }; - - let query_relations = get_query_relations(unsafe { (*query).rtable }); - if unsafe { (*query).commandType } != pg_sys::CmdType::CMD_SELECT - || !is_duckdb_query(&query_relations) - { - return Ok(true); - } - - if unsafe { !(*stmt).options.is_null() } { - error!("the EXPLAIN options provided are not supported for DuckDB pushdown queries."); - } - - unsafe { - let tstate = pg_sys::begin_tup_output_tupdesc( - dest, - pg_sys::ExplainResultDesc(stmt), - &pg_sys::TTSOpsVirtual, - ); - let query = format!( - "DuckDB Scan: {}", - parse_query_from_utility_stmt(query_string)? - ); - let query_c_str = CString::new(query)?; - - pg_sys::do_text_output_multiline(tstate, query_c_str.as_ptr()); - pg_sys::end_tup_output(tstate); - } - - Ok(false) + || stmt_type == pg_sys::NodeTag::T_PrepareStmt + || stmt_type == pg_sys::NodeTag::T_DeallocateStmt + || stmt_type == pg_sys::NodeTag::T_ExecuteStmt } fn parse_query_from_utility_stmt(query_string: &core::ffi::CStr) -> Result { diff --git a/src/hooks/utility/explain.rs b/src/hooks/utility/explain.rs new file mode 100644 index 00000000..bdfca82b --- /dev/null +++ b/src/hooks/utility/explain.rs @@ -0,0 +1,61 @@ +// Copyright (c) 2023-2024 Retake, Inc. +// +// This file is part of ParadeDB - Postgres for Search and Analytics +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::ffi::CString; + +use anyhow::Result; +use pgrx::{error, pg_sys}; + +use super::parse_query_from_utility_stmt; +use crate::hooks::query::{get_query_relations, is_duckdb_query}; + +pub fn explain_query( + query_string: &core::ffi::CStr, + stmt: *mut pg_sys::ExplainStmt, + dest: *mut pg_sys::DestReceiver, +) -> Result { + let query = unsafe { (*stmt).query as *mut pg_sys::Query }; + + let query_relations = get_query_relations(unsafe { (*query).rtable }); + if unsafe { (*query).commandType } != pg_sys::CmdType::CMD_SELECT + || !is_duckdb_query(&query_relations) + { + return Ok(true); + } + + if unsafe { !(*stmt).options.is_null() } { + error!("the EXPLAIN options provided are not supported for DuckDB pushdown queries."); + } + + unsafe { + let tstate = pg_sys::begin_tup_output_tupdesc( + dest, + pg_sys::ExplainResultDesc(stmt), + &pg_sys::TTSOpsVirtual, + ); + let query = format!( + "DuckDB Scan: {}", + parse_query_from_utility_stmt(query_string)? + ); + let query_c_str = CString::new(query)?; + + pg_sys::do_text_output_multiline(tstate, query_c_str.as_ptr()); + pg_sys::end_tup_output(tstate); + } + + Ok(false) +} diff --git a/src/hooks/utility/prepare.rs b/src/hooks/utility/prepare.rs new file mode 100644 index 00000000..55de8c0e --- /dev/null +++ b/src/hooks/utility/prepare.rs @@ -0,0 +1,211 @@ +// Copyright (c) 2023-2024 Retake, Inc. +// +// This file is part of ParadeDB - Postgres for Search and Analytics +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::ffi::CStr; +use std::ptr::null_mut; + +use anyhow::Result; +use pgrx::{error, pg_sys, pgbox, warning, PgBox}; + +use crate::duckdb::connection; +use crate::hooks::query::*; + +pub fn prepare_query( + pstate: *mut pg_sys::ParseState, + stmt: *mut pg_sys::PrepareStmt, + stmt_location: i32, + stmt_len: i32, +) -> Result { + if unsafe { (*stmt).name }.is_null() || unsafe { *(*stmt).name } == '\0' as std::os::raw::c_char + { + return Ok(true); + } + + let prepared_stmt = unsafe { pg_sys::FetchPreparedStatement((*stmt).name, false) }; + if !prepared_stmt.is_null() { + let stmt_name = unsafe { CStr::from_ptr((*stmt).name) }; + error!( + "prepared statement \"{}\" already exists", + stmt_name.to_str()? + ); + } + + // Perform parsing and analysis to get the Query + let query = unsafe { + let mut raw_stmt = pg_sys::RawStmt { + type_: pg_sys::NodeTag::T_RawStmt, + stmt: (*stmt).query, + stmt_location, + stmt_len, + }; + + let arg_types = (*stmt).argtypes; + let mut nargs = if arg_types.is_null() { + 0 + } else { + (*arg_types).length + }; + + // Transform list of TypeNames to array of type OIDs + let mut types_oid: *mut pg_sys::Oid = if nargs > 0 { + let oid_ptr = pg_sys::palloc((nargs as usize) * std::mem::size_of::()) + as *mut pg_sys::Oid; + let type_elements = (*arg_types).elements; + for i in 0..(*arg_types).length { + let type_name = + (*type_elements.offset(i as isize)).ptr_value as *const pg_sys::TypeName; + *oid_ptr.offset(i as isize) = pg_sys::typenameTypeId(pstate, type_name) + } + oid_ptr + } else { + null_mut() + }; + + #[cfg(any(feature = "pg15", feature = "pg16", feature = "pg17"))] + { + pg_sys::parse_analyze_varparams( + &mut raw_stmt, + (*pstate).p_sourcetext, + &mut types_oid, + &mut nargs, + null_mut(), + ) + } + #[cfg(any(feature = "pg13", feature = "pg14"))] + { + pg_sys::parse_analyze_varparams( + &mut raw_stmt, + (*pstate).p_sourcetext, + &mut types_oid, + &mut nargs, + ) + } + }; + + let query_relations = get_query_relations(unsafe { (*query).rtable }); + if unsafe { (*query).commandType } != pg_sys::CmdType::CMD_SELECT + || !is_duckdb_query(&query_relations) + { + return Ok(true); + } + + // set search path according to postgres + set_search_path_by_pg()?; + + let query_sql: &CStr = unsafe { CStr::from_ptr((*pstate).p_sourcetext) }; + if let Err(e) = connection::execute(query_sql.to_str()?, []) { + fallback_warning!(e.to_string()); + return Ok(true); + } + + // It's always necessary to execute the previous hook to store a prepared statement in PostgreSQL + Ok(true) +} + +pub fn execute_query( + _psate: *mut pg_sys::ParseState, + stmt: *mut pg_sys::ExecuteStmt, + query_desc: PgBox, +) -> Result { + unsafe { + let prepared_stmt = pg_sys::FetchPreparedStatement((*stmt).name, true); + let plan_source = (*prepared_stmt).plansource; + + if plan_source.is_null() || !(*plan_source).fixed_result { + return Ok(true); + } + + // We need to ensure that DuckDB replans the PREPARE statement when the search path changes, + // in order to match PostgreSQL’s default behavior. + + #[cfg(not(feature = "pg17"))] + let need_replan = !pg_sys::OverrideSearchPathMatchesCurrent((*plan_source).search_path); + #[cfg(feature = "pg17")] + let need_replan = !pg_sys::SearchPathMatchesCurrentEnvironment((*plan_source).search_path); + + // For PostgreSQL 13 + #[cfg(feature = "pg13")] + let cached_plan = pg_sys::GetCachedPlan(plan_source, null_mut(), false, null_mut()); + // For PostgreSQL 14 and above + #[cfg(not(feature = "pg13"))] + let cached_plan = pg_sys::GetCachedPlan(plan_source, null_mut(), null_mut(), null_mut()); + + if cached_plan.is_null() || (*cached_plan).stmt_list.is_null() { + return Ok(true); + } + + let planned_stmt = + (*(*(*cached_plan).stmt_list).elements.offset(0)).ptr_value as *mut pg_sys::PlannedStmt; + let query_relations = get_query_relations((*planned_stmt).rtable); + if (*planned_stmt).commandType != pg_sys::CmdType::CMD_SELECT + || !is_duckdb_query(&query_relations) + { + return Ok(true); + } + + (*query_desc.as_ptr()).tupDesc = (*plan_source).resultDesc; + + // Note that DuckDB does not replan prepared statements when the search path changes. + // We enforce this by executing the PREPARE statement again. + set_search_path_by_pg()?; + + if need_replan { + let prepare_stmt = CStr::from_ptr((*plan_source).query_string); + if let Err(e) = connection::execute(prepare_stmt.to_str()?, []) { + error!("execute prepare replan error: {}", e.to_string()); + } + } + } + + let query = unsafe { CStr::from_ptr((*query_desc.as_ptr()).sourceText) }; + + match connection::create_arrow(query.to_str()?) { + Err(err) => { + connection::clear_arrow(); + fallback_warning!(err.to_string()); + return Ok(true); + } + Ok(false) => { + connection::clear_arrow(); + return Ok(false); + } + _ => {} + } + + match connection::get_batches() { + Ok(batches) => write_batches_to_slots(query_desc, batches)?, + Err(err) => { + connection::clear_arrow(); + fallback_warning!(err.to_string()); + return Ok(true); + } + } + + connection::clear_arrow(); + Ok(false) +} + +pub fn deallocate_query(stmt: *mut pg_sys::DeallocateStmt) -> Result { + if !unsafe { (*stmt).name }.is_null() { + let name = unsafe { CStr::from_ptr((*stmt).name) }; + // We don't care the result + // Next prepare statement will override this one. + let _ = connection::execute(&format!(r#"DEALLOCATE "{}""#, name.to_str()?), []); + } + + Ok(true) +} diff --git a/tests/fixtures/arrow.rs b/tests/fixtures/arrow.rs index 4e98dd99..56578502 100644 --- a/tests/fixtures/arrow.rs +++ b/tests/fixtures/arrow.rs @@ -43,6 +43,17 @@ fn array_data() -> ArrayData { .unwrap() } +fn single_array_data() -> ArrayData { + let values: [u8; 5] = *b"hello"; + let offsets: [i32; 2] = [0, 5]; + ArrayData::builder(DataType::Binary) + .len(1) + .add_buffer(Buffer::from_slice_ref(&offsets[..])) + .add_buffer(Buffer::from_slice_ref(&values[..])) + .build() + .unwrap() +} + // Fixed size binary is not supported yet, but this will be useful for test data when we do support. fn fixed_size_array_data() -> ArrayData { let values: [u8; 15] = *b"hellotherearrow"; // Ensure length is consistent @@ -66,6 +77,17 @@ fn binary_array_data() -> ArrayData { .unwrap() } +fn single_binary_array_data() -> ArrayData { + let values: [u8; 5] = *b"hello"; + let offsets: [i64; 2] = [0, 5]; + ArrayData::builder(DataType::LargeBinary) + .len(1) + .add_buffer(Buffer::from_slice_ref(&offsets[..])) + .add_buffer(Buffer::from_slice_ref(&values[..])) + .build() + .unwrap() +} + /// A separate version of the primitive_record_batch fixture, /// narrowed to only the types that Delta Lake supports. pub fn delta_primitive_record_batch() -> Result { @@ -119,10 +141,8 @@ pub fn record_batch_with_casing() -> Result { Ok(batch) } -// Blows up deltalake, so comment out for now. -pub fn primitive_record_batch() -> Result { - // Define the fields for each datatype - let fields = vec![ +pub fn primitive_fields() -> Vec { + vec![ Field::new("boolean_col", DataType::Boolean, true), Field::new("int8_col", DataType::Int8, false), Field::new("int16_col", DataType::Int16, false), @@ -140,7 +160,13 @@ pub fn primitive_record_batch() -> Result { Field::new("large_binary_col", DataType::LargeBinary, false), Field::new("utf8_col", DataType::Utf8, false), Field::new("large_utf8_col", DataType::LargeUtf8, false), - ]; + ] +} + +// Blows up deltalake, so comment out for now. +pub fn primitive_record_batch() -> Result { + // Define the fields for each datatype + let fields = primitive_fields(); // Create a schema from the fields let schema = Arc::new(Schema::new(fields)); @@ -186,6 +212,38 @@ pub fn primitive_record_batch() -> Result { )?) } +pub fn primitive_record_batch_single() -> Result { + // Define the fields for each datatype + let fields = primitive_fields(); + + // Create a schema from the fields + let schema = Arc::new(Schema::new(fields)); + + // Create a RecordBatch + Ok(RecordBatch::try_new( + schema, + vec![ + Arc::new(BooleanArray::from(vec![Some(true)])), + Arc::new(Int8Array::from(vec![1])), + Arc::new(Int16Array::from(vec![1])), + Arc::new(Int32Array::from(vec![1])), + Arc::new(Int64Array::from(vec![1])), + Arc::new(UInt8Array::from(vec![1])), + Arc::new(UInt16Array::from(vec![1])), + Arc::new(UInt32Array::from(vec![1])), + Arc::new(UInt64Array::from(vec![1])), + Arc::new(Float32Array::from(vec![1.0])), + Arc::new(Float64Array::from(vec![1.0])), + Arc::new(Date32Array::from(vec![18262])), + Arc::new(Date64Array::from(vec![1609459200000])), + Arc::new(BinaryArray::from(single_array_data())), + Arc::new(LargeBinaryArray::from(single_binary_array_data())), + Arc::new(StringArray::from(vec![Some("Hello")])), + Arc::new(LargeStringArray::from(vec![Some("Hello")])), + ], + )?) +} + pub fn primitive_create_foreign_data_wrapper( wrapper: &str, handler: &str, @@ -405,6 +463,20 @@ pub fn setup_local_file_listing_with_casing(local_file_path: &str, table: &str) ) } +pub fn setup_parquet_wrapper_and_server() -> String { + let create_foreign_data_wrapper = primitive_create_foreign_data_wrapper( + "parquet_wrapper", + "parquet_fdw_handler", + "parquet_fdw_validator", + ); + let create_server = primitive_create_server("parquet_server", "parquet_wrapper"); + format!( + "{create_foreign_data_wrapper}; + {create_server}; + " + ) +} + fn valid(data_type: &DataType, oid: u32) -> bool { let oid = match PgBuiltInOids::from_u32(oid) { Ok(oid) => oid, diff --git a/tests/scan.rs b/tests/scan.rs index 053c4b04..425ced84 100644 --- a/tests/scan.rs +++ b/tests/scan.rs @@ -20,8 +20,9 @@ mod fixtures; use crate::fixtures::arrow::{ delta_primitive_record_batch, primitive_create_foreign_data_wrapper, primitive_create_server, primitive_create_table, primitive_create_user_mapping_options, primitive_record_batch, - primitive_setup_fdw_local_file_delta, primitive_setup_fdw_local_file_listing, - primitive_setup_fdw_s3_delta, primitive_setup_fdw_s3_listing, + primitive_record_batch_single, primitive_setup_fdw_local_file_delta, + primitive_setup_fdw_local_file_listing, primitive_setup_fdw_s3_delta, + primitive_setup_fdw_s3_listing, setup_parquet_wrapper_and_server, }; use crate::fixtures::db::Query; use crate::fixtures::{conn, duckdb_conn, s3, tempdir, S3}; @@ -557,3 +558,94 @@ async fn test_executor_hook_search_path(mut conn: PgConnection, tempdir: TempDir Ok(()) } + +#[rstest] +async fn test_prepare_stmt_execute(#[future(awt)] s3: S3, mut conn: PgConnection) -> Result<()> { + NycTripsTable::setup().execute(&mut conn); + let rows: Vec = "SELECT * FROM nyc_trips".fetch(&mut conn); + s3.client + .create_bucket() + .bucket(S3_TRIPS_BUCKET) + .send() + .await?; + s3.create_bucket(S3_TRIPS_BUCKET).await?; + s3.put_rows(S3_TRIPS_BUCKET, S3_TRIPS_KEY, &rows).await?; + + NycTripsTable::setup_s3_listing_fdw( + &s3.url.clone(), + &format!("s3://{S3_TRIPS_BUCKET}/{S3_TRIPS_KEY}"), + ) + .execute(&mut conn); + + r#"PREPARE test_query(int) AS SELECT count(*) FROM trips WHERE "VendorID" = $1;"# + .execute(&mut conn); + + let count: (i64,) = "EXECUTE test_query(1)".fetch_one(&mut conn); + assert_eq!(count.0, 39); + + let count: (i64,) = "EXECUTE test_query(3)".fetch_one(&mut conn); + assert_eq!(count.0, 0); + + "DEALLOCATE test_query".execute(&mut conn); + + assert!("EXECUTE test_query(3)".execute_result(&mut conn).is_err()); + + Ok(()) +} + +// Note: PostgreSQL will replan the query when certain catalog changes occur, +// such as changes to the search path or when a table is deleted. +// In contrast, DuckDB does not replan when the search path is changed. +// If there are two foreign tables in different schemas and the prepared statements do not specify the schemas, +// it may lead to ambiguity or errors when referencing the tables. +#[rstest] +async fn test_prepare_search_path(mut conn: PgConnection, tempdir: TempDir) -> Result<()> { + let stored_batch = primitive_record_batch()?; + let parquet_path = tempdir.path().join("test_arrow_types.parquet"); + let parquet_file = File::create(&parquet_path)?; + + let mut writer = ArrowWriter::try_new(parquet_file, stored_batch.schema(), None).unwrap(); + writer.write(&stored_batch)?; + writer.close()?; + + let stored_batch_less = primitive_record_batch_single()?; + let less_parquet_path = tempdir.path().join("test_arrow_types_less.parquet"); + let less_parquet_file = File::create(&less_parquet_path)?; + + let mut writer = + ArrowWriter::try_new(less_parquet_file, stored_batch_less.schema(), None).unwrap(); + writer.write(&stored_batch_less)?; + writer.close()?; + + // In this example, we create two tables with identical structures and names, but in different schemas. + // We expect that when the search path is changed, the correct table (the one in the current schema) will be referenced in DuckDB. + "CREATE SCHEMA tpch1".execute(&mut conn); + "CREATE SCHEMA tpch2".execute(&mut conn); + + setup_parquet_wrapper_and_server().execute(&mut conn); + + let file_path = parquet_path.as_path().to_str().unwrap(); + let file_less_path = less_parquet_path.as_path().to_str().unwrap(); + + let create_table_t1 = primitive_create_table("parquet_server", "tpch1.t1"); + (&format!("{create_table_t1} OPTIONS (files '{file_path}');")).execute(&mut conn); + + let create_table_less_t1 = primitive_create_table("parquet_server", "tpch2.t1"); + (&format!("{create_table_less_t1} OPTIONS (files '{file_less_path}');")).execute(&mut conn); + + "SET search_path TO tpch1".execute(&mut conn); + + "PREPARE q1 AS SELECT * FROM t1 WHERE boolean_col = $1".execute(&mut conn); + + let result: Vec<(bool,)> = "EXECUTE q1(true)".fetch_collect(&mut conn); + assert_eq!(result.len(), 2); + + "SET search_path TO tpch2".execute(&mut conn); + let result: Vec<(bool,)> = "EXECUTE q1(true)".fetch_collect(&mut conn); + assert_eq!(result.len(), 1); + + "DEALLOCATE q1".execute(&mut conn); + assert!("EXECUTE q1(true)".execute_result(&mut conn).is_err()); + + Ok(()) +} From 73f092446fa028e8270e33f27b2a06a7f93f0ae0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20No=C3=ABl?= <21990816+philippemnoel@users.noreply.github.com> Date: Mon, 14 Oct 2024 14:09:44 -0400 Subject: [PATCH 8/9] Upgrade pgrx (#154) --- Cargo.lock | 32 ++++++++++++++++---------------- Cargo.toml | 6 +++--- README.md | 2 +- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dd9a35a0..d89c3546 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4042,9 +4042,9 @@ dependencies = [ [[package]] name = "pgrx" -version = "0.12.5" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39464992d5b3cdda6b390efe51c27e849d2e93c1811e7610cf88f82d5b6a1a23" +checksum = "29fd8bef6f9f963a224b6d92033aa75581ecf01c8a815eb6dea0ad7260ac4541" dependencies = [ "atomic-traits", "bitflags 2.6.0", @@ -4066,9 +4066,9 @@ dependencies = [ [[package]] name = "pgrx-bindgen" -version = "0.12.5" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84b03b74332a89c852486c0a53c33e049fb56d7911e251ddd05b7820b2963592" +checksum = "bb9cd617058bad2d78e31c55b58d2b003dcf039cba125b8e018c0f72562fb951" dependencies = [ "bindgen", "cc", @@ -4084,9 +4084,9 @@ dependencies = [ [[package]] name = "pgrx-macros" -version = "0.12.5" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "822e16af0dfc820dbd407b19538fde0def27c7d1913dea47a42c040c8bcf8b56" +checksum = "cbf074a98b59d1811b29c97e58d7ec5cab00445c7a5d7ed4153d746eb8572d55" dependencies = [ "pgrx-sql-entity-graph", "proc-macro2", @@ -4096,9 +4096,9 @@ dependencies = [ [[package]] name = "pgrx-pg-config" -version = "0.12.5" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31061cc1a9f860d1302ba952ccf3b3d0eec59be94f0a32bcc8ba858fdd44a824" +checksum = "a82140784390a8f7f8a4f36acc6c04d177f59666012fa419eb03ab90201945ef" dependencies = [ "cargo_toml", "eyre", @@ -4114,9 +4114,9 @@ dependencies = [ [[package]] name = "pgrx-pg-sys" -version = "0.12.5" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0225f2eeacf00702a705cbeb50252f4279169cd08501f73889b479d23a4250e1" +checksum = "6a0b49de7e28fdf7eb9ace7b736c2527330441efd35e55768fa26c637366d1c3" dependencies = [ "cee-scape", "libc", @@ -4129,9 +4129,9 @@ dependencies = [ [[package]] name = "pgrx-sql-entity-graph" -version = "0.12.5" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9c9dfdc4b31b381405b9735b7d674680b42584ea2ef89f3668a21be57a39310" +checksum = "dbf0cb2b4c8204678cf3dfc71bd0febe11dd4a6636b1971c3ee595d58d2db5f6" dependencies = [ "convert_case", "eyre", @@ -4145,9 +4145,9 @@ dependencies = [ [[package]] name = "pgrx-tests" -version = "0.12.5" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "992bb74e9cd178a55eab23f7a4b890a5427a411bbaf9588c86d435a3664eb4d2" +checksum = "207fa953e4634371ff00e20584e80b1faf0c381fb7bec14648ce9076494582de" dependencies = [ "clap-cargo", "eyre", @@ -5738,7 +5738,7 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "supabase-wrappers" version = "0.1.19" -source = "git+https://github.com/paradedb/wrappers.git?rev=c32abb7#c32abb70dbdfdf7b77706e2abeb1ed2324144d34" +source = "git+https://github.com/paradedb/wrappers.git?rev=f5ecb8d#f5ecb8df50e7edf834cc852392710f4f51a5f33c" dependencies = [ "pgrx", "supabase-wrappers-macros", @@ -5750,7 +5750,7 @@ dependencies = [ [[package]] name = "supabase-wrappers-macros" version = "0.1.16" -source = "git+https://github.com/paradedb/wrappers.git?rev=c32abb7#c32abb70dbdfdf7b77706e2abeb1ed2324144d34" +source = "git+https://github.com/paradedb/wrappers.git?rev=f5ecb8d#f5ecb8df50e7edf834cc852392710f4f51a5f33c" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 020fd677..47b585bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,13 +25,13 @@ duckdb = { git = "https://github.com/paradedb/duckdb-rs.git", features = [ "bundled", "extensions-full", ], rev = "e532dd6" } -pgrx = "0.12.5" +pgrx = "0.12.6" serde = "1.0.210" serde_json = "1.0.128" signal-hook = "0.3.17" sqlparser = "0.50.0" strum = { version = "0.26.3", features = ["derive"] } -supabase-wrappers = { git = "https://github.com/paradedb/wrappers.git", default-features = false, rev = "c32abb7" } +supabase-wrappers = { git = "https://github.com/paradedb/wrappers.git", default-features = false, rev = "f5ecb8d" } thiserror = "1.0.63" uuid = "1.10.0" @@ -43,7 +43,7 @@ bytes = "1.7.1" datafusion = "37.1.0" deltalake = { version = "0.17.3", features = ["datafusion"] } futures = "0.3.30" -pgrx-tests = "0.12.5" +pgrx-tests = "0.12.6" rstest = "0.19.0" serde_arrow = { version = "0.11.7", features = ["arrow-51"] } soa_derive = "0.13.0" diff --git a/README.md b/README.md index a76bbec6..e021ca96 100644 --- a/README.md +++ b/README.md @@ -199,7 +199,7 @@ Then, install and initialize `pgrx`: ```bash # Note: Replace --pg17 with your version of Postgres, if different (i.e. --pg17, --pg16, --pg15, --pg14, --pg13 etc.) -cargo install --locked cargo-pgrx --version 0.12.5 +cargo install --locked cargo-pgrx --version 0.12.6 # macOS arm64 cargo pgrx init --pg17=/opt/homebrew/opt/postgresql@17/bin/pg_config From 81c67bc4f0e814840ce9aa47bda4452e82de9a1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20No=C3=ABl?= <21990816+philippemnoel@users.noreply.github.com> Date: Mon, 14 Oct 2024 14:09:59 -0400 Subject: [PATCH 9/9] Prepare 0.2.1 (#155) --- Cargo.lock | 2 +- Cargo.toml | 2 +- sql/pg_analytics--0.2.0--0.2.1.sql | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 sql/pg_analytics--0.2.0--0.2.1.sql diff --git a/Cargo.lock b/Cargo.lock index d89c3546..30a82014 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4006,7 +4006,7 @@ dependencies = [ [[package]] name = "pg_analytics" -version = "0.2.0" +version = "0.2.1" dependencies = [ "anyhow", "async-std", diff --git a/Cargo.toml b/Cargo.toml index 47b585bd..e4923f68 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pg_analytics" description = "Postgres for analytics, powered by DuckDB" -version = "0.2.0" +version = "0.2.1" edition = "2021" license = "AGPL-3.0" diff --git a/sql/pg_analytics--0.2.0--0.2.1.sql b/sql/pg_analytics--0.2.0--0.2.1.sql new file mode 100644 index 00000000..bcb7219f --- /dev/null +++ b/sql/pg_analytics--0.2.0--0.2.1.sql @@ -0,0 +1 @@ +\echo Use "ALTER EXTENSION pg_analytics UPDATE TO '0.2.1'" to load this file. \quit