From e3af1d4ca554db1d92c729d265031c911153b0cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Milenkovi=C4=87?= Date: Wed, 16 Oct 2024 23:57:39 +0100 Subject: [PATCH] another round of cleanup ... (#1089) - ballista.plugin_dir - update to `actions/checkout@v4` relates to #1067 --- .github/workflows/comment_bot.yml | 4 ++-- .github/workflows/dev.yml | 6 +++--- .github/workflows/dev_pr.yml | 2 +- .github/workflows/docker.yml | 4 ++-- .github/workflows/rust.yml | 20 ++++++++++---------- ballista/client/src/prelude.rs | 6 +++--- ballista/core/src/config.rs | 13 +++---------- docs/source/user-guide/configs.md | 1 - 8 files changed, 24 insertions(+), 32 deletions(-) diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml index 51e53b7af..81b15939b 100644 --- a/.github/workflows/comment_bot.yml +++ b/.github/workflows/comment_bot.yml @@ -31,7 +31,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Arrow - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: apache/arrow - name: Set up Python @@ -54,7 +54,7 @@ jobs: if: startsWith(github.event.comment.body, '@github-actions rebase') runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: r-lib/actions/pr-fetch@master with: repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index b71c5d984..2c02c0b8d 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Arrow - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: apache/arrow submodules: true @@ -43,7 +43,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python uses: actions/setup-python@v4 with: @@ -55,7 +55,7 @@ jobs: name: Use prettier to check formatting of documents runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: node-version: "14" diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml index c898cbe14..b8ee4ed90 100644 --- a/.github/workflows/dev_pr.yml +++ b/.github/workflows/dev_pr.yml @@ -29,7 +29,7 @@ jobs: name: Process runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Assign GitHub labels if: | diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a50ac823d..ffeb9fa55 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -24,8 +24,8 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Installs Rust and Cargo run: curl -y --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh - name: Run script - run: ./dev/build-ballista-docker.sh \ No newline at end of file + run: ./dev/build-ballista-docker.sh diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 1ffb9e181..cbda2632b 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -38,7 +38,7 @@ jobs: # "1" means line tables only, which is useful for panic tracebacks. RUSTFLAGS: "-C debuginfo=1" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Cache Cargo uses: actions/cache@v4 with: @@ -88,7 +88,7 @@ jobs: # "1" means line tables only, which is useful for panic tracebacks. RUSTFLAGS: "-C debuginfo=1" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - name: Install protobuf compiler @@ -142,7 +142,7 @@ jobs: # "1" means line tables only, which is useful for panic tracebacks. RUSTFLAGS: "-C debuginfo=1" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - name: Install protobuf compiler @@ -192,7 +192,7 @@ jobs: os: [windows-latest, macos-latest] rust: [stable] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - name: Install protobuf macos compiler @@ -252,7 +252,7 @@ jobs: # "1" means line tables only, which is useful for panic tracebacks. RUSTFLAGS: "-C debuginfo=1" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - name: Install protobuf compiler @@ -286,7 +286,7 @@ jobs: container: image: amd64/rust steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup toolchain run: | rustup toolchain install stable @@ -306,7 +306,7 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Restore rust artifacts uses: actions/download-artifact@v4 with: @@ -343,7 +343,7 @@ jobs: # "1" means line tables only, which is useful for panic tracebacks. RUSTFLAGS: "-C debuginfo=1" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - name: Cache Cargo @@ -386,7 +386,7 @@ jobs: # "1" means line tables only, which is useful for panic tracebacks. RUSTFLAGS: "-C debuginfo=1" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - name: Cache Cargo @@ -435,7 +435,7 @@ jobs: # arch: [amd64] # rust: [stable] # steps: -# - uses: actions/checkout@v3 +# - uses: actions/checkout@v4 # with: # submodules: true # - name: Cache Cargo diff --git a/ballista/client/src/prelude.rs b/ballista/client/src/prelude.rs index 0314c1282..acab66529 100644 --- a/ballista/client/src/prelude.rs +++ b/ballista/client/src/prelude.rs @@ -21,9 +21,9 @@ pub use ballista_core::{ config::{ BallistaConfig, BALLISTA_COLLECT_STATISTICS, BALLISTA_DEFAULT_BATCH_SIZE, BALLISTA_DEFAULT_SHUFFLE_PARTITIONS, BALLISTA_GRPC_CLIENT_MAX_MESSAGE_SIZE, - BALLISTA_JOB_NAME, BALLISTA_PARQUET_PRUNING, BALLISTA_PLUGIN_DIR, - BALLISTA_REPARTITION_AGGREGATIONS, BALLISTA_REPARTITION_JOINS, - BALLISTA_REPARTITION_WINDOWS, BALLISTA_WITH_INFORMATION_SCHEMA, + BALLISTA_JOB_NAME, BALLISTA_PARQUET_PRUNING, BALLISTA_REPARTITION_AGGREGATIONS, + BALLISTA_REPARTITION_JOINS, BALLISTA_REPARTITION_WINDOWS, + BALLISTA_WITH_INFORMATION_SCHEMA, }, error::{BallistaError, Result}, }; diff --git a/ballista/core/src/config.rs b/ballista/core/src/config.rs index 746a0be92..db47a7e5b 100644 --- a/ballista/core/src/config.rs +++ b/ballista/core/src/config.rs @@ -39,8 +39,7 @@ pub const BALLISTA_PARQUET_PRUNING: &str = "ballista.parquet.pruning"; pub const BALLISTA_COLLECT_STATISTICS: &str = "ballista.collect_statistics"; pub const BALLISTA_WITH_INFORMATION_SCHEMA: &str = "ballista.with_information_schema"; -/// give a plugin files dir, and then the dynamic library files in this dir will be load when scheduler state init. -pub const BALLISTA_PLUGIN_DIR: &str = "ballista.plugin_dir"; + /// max message size for gRPC clients pub const BALLISTA_GRPC_CLIENT_MAX_MESSAGE_SIZE: &str = "ballista.grpc_client_max_message_size"; @@ -205,9 +204,7 @@ impl BallistaConfig { "Configuration for collecting statistics during scan".to_string(), DataType::Boolean, Some("false".to_string()) ), - ConfigEntry::new(BALLISTA_PLUGIN_DIR.to_string(), - "Sets the plugin dir".to_string(), - DataType::Utf8, Some("".to_string())), + ConfigEntry::new(BALLISTA_GRPC_CLIENT_MAX_MESSAGE_SIZE.to_string(), "Configuration for max message size in gRPC clients".to_string(), DataType::UInt64, @@ -227,10 +224,6 @@ impl BallistaConfig { self.get_usize_setting(BALLISTA_DEFAULT_SHUFFLE_PARTITIONS) } - pub fn default_plugin_dir(&self) -> String { - self.get_string_setting(BALLISTA_PLUGIN_DIR) - } - pub fn default_batch_size(&self) -> usize { self.get_usize_setting(BALLISTA_DEFAULT_BATCH_SIZE) } @@ -290,6 +283,7 @@ impl BallistaConfig { v.parse::().unwrap() } } + #[allow(dead_code)] fn get_string_setting(&self, key: &str) -> String { if let Some(v) = self.settings.get(key) { // infallible because we validate all configs in the constructor @@ -403,7 +397,6 @@ mod tests { fn custom_config_invalid() -> Result<()> { let config = BallistaConfig::builder() .set(BALLISTA_DEFAULT_SHUFFLE_PARTITIONS, "true") - .set(BALLISTA_PLUGIN_DIR, "test_dir") .build(); assert!(config.is_err()); assert_eq!("General(\"Failed to parse user-supplied value 'ballista.shuffle.partitions' for configuration setting 'true': ParseIntError { kind: InvalidDigit }\")", format!("{:?}", config.unwrap_err())); diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index d112e7c8c..880ee4901 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -46,7 +46,6 @@ let ctx = BallistaContext::remote("localhost", 50050, & config).await?; | ballista.repartition.windows | Boolean | true | When set to true, Ballista will repartition data using the partition keys to execute window functions in parallel using the provided `ballista.shuffle.partitions` level. | | ballista.parquet.pruning | Boolean | true | Determines whether Parquet pruning should be enabled or not. | | ballista.with_information_schema | Boolean | true | Determines whether the `information_schema` should be created in the context. This is necessary for supporting DDL commands such as `SHOW TABLES`. | -| ballista.plugin_dir | Boolean | true | Specified a path for plugin files. Dynamic library files in this directory will be loaded when scheduler state initializes. | ### DataFusion Configuration Settings