Add gorilla compression for network requests

locustdb-client/.appveyor.yml

@@ -0,0 +1,11 @@
+install:
+  - appveyor-retry appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe
+  - if not defined RUSTFLAGS rustup-init.exe -y --default-host x86_64-pc-windows-msvc --default-toolchain nightly
+  - set PATH=%PATH%;C:\Users\appveyor\.cargo\bin
+  - rustc -V
+  - cargo -V
+
+build: false
+
+test_script:
+  - cargo test --locked

locustdb-client/.github/dependabot.yml

@@ -0,0 +1,8 @@
+version: 2
+updates:
+- package-ecosystem: cargo
+  directory: "/"
+  schedule:
+    interval: daily
+    time: "08:00"
+  open-pull-requests-limit: 10

locustdb-client/.gitignore

@@ -0,0 +1,6 @@
+/target
+**/*.rs.bk
+Cargo.lock
+bin/
+pkg/
+wasm-pack.log

locustdb-client/.travis.yml

@@ -0,0 +1,69 @@
+language: rust
+sudo: false
+
+cache: cargo
+
+matrix:
+  include:
+
+  # Builds with wasm-pack.
+  - rust: beta
+    env: RUST_BACKTRACE=1
+    addons:
+      firefox: latest
+      chrome: stable
+    before_script:
+      - (test -x $HOME/.cargo/bin/cargo-install-update || cargo install cargo-update)
+      - (test -x $HOME/.cargo/bin/cargo-generate || cargo install --vers "^0.2" cargo-generate)
+      - cargo install-update -a
+      - curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh -s -- -f
+    script:
+      - cargo generate --git . --name testing
+      # Having a broken Cargo.toml (in that it has curlies in fields) anywhere
+      # in any of our parent dirs is problematic.
+      - mv Cargo.toml Cargo.toml.tmpl
+      - cd testing
+      - wasm-pack build
+      - wasm-pack test --chrome --firefox --headless
+
+  # Builds on nightly.
+  - rust: nightly
+    env: RUST_BACKTRACE=1
+    before_script:
+      - (test -x $HOME/.cargo/bin/cargo-install-update || cargo install cargo-update)
+      - (test -x $HOME/.cargo/bin/cargo-generate || cargo install --vers "^0.2" cargo-generate)
+      - cargo install-update -a
+      - rustup target add wasm32-unknown-unknown
+    script:
+      - cargo generate --git . --name testing
+      - mv Cargo.toml Cargo.toml.tmpl
+      - cd testing
+      - cargo check
+      - cargo check --target wasm32-unknown-unknown
+      - cargo check                                 --no-default-features
+      - cargo check --target wasm32-unknown-unknown --no-default-features
+      - cargo check                                 --no-default-features --features console_error_panic_hook
+      - cargo check --target wasm32-unknown-unknown --no-default-features --features console_error_panic_hook
+      - cargo check                                 --no-default-features --features "console_error_panic_hook wee_alloc"
+      - cargo check --target wasm32-unknown-unknown --no-default-features --features "console_error_panic_hook wee_alloc"
+
+  # Builds on beta.
+  - rust: beta
+    env: RUST_BACKTRACE=1
+    before_script:
+      - (test -x $HOME/.cargo/bin/cargo-install-update || cargo install cargo-update)
+      - (test -x $HOME/.cargo/bin/cargo-generate || cargo install --vers "^0.2" cargo-generate)
+      - cargo install-update -a
+      - rustup target add wasm32-unknown-unknown
+    script:
+      - cargo generate --git . --name testing
+      - mv Cargo.toml Cargo.toml.tmpl
+      - cd testing
+      - cargo check
+      - cargo check --target wasm32-unknown-unknown
+      - cargo check                                 --no-default-features
+      - cargo check --target wasm32-unknown-unknown --no-default-features
+      - cargo check                                 --no-default-features --features console_error_panic_hook
+      - cargo check --target wasm32-unknown-unknown --no-default-features --features console_error_panic_hook
+      # Note: no enabling the `wee_alloc` feature here because it requires
+      # nightly for now.

locustdb-client/Cargo.toml

@@ -0,0 +1,36 @@
+[package]
+name = "locustdb-client"
+version = "0.1.0"
+authors = ["Clemens Winter <[email protected]>"]
+edition = "2018"
+
+[lib]
+crate-type = ["cdylib", "rlib"]
+
+[features]
+default = ["console_error_panic_hook"]
+
+[dependencies]
+reqwest = { version = "0.12", default_features = false, features = ["json", "rustls-tls"] }
+serde = { version = "1.0", features = ["derive"] }
+serde-wasm-bindgen = "0.6.5"
+serde_json = "1.0"
+wasm-bindgen = "0.2.84"
+wasm-bindgen-futures = "0.4.42"
+locustdb-compression-utils = {path = "../locustdb-compression-utils", version = "0.1.0"}
+bincode = "1.3"
+
+# The `console_error_panic_hook` crate provides better debugging of panics by
+# logging them with `console.error`. This is great for development, but requires
+# all the `std::fmt` and `std::panicking` infrastructure, so isn't great for
+# code size when deploying.
+console_error_panic_hook = { version = "0.1.7", optional = true }
+web-sys = { version = "0.3.69", features = ["Performance", "Window"] }
+
+[dev-dependencies]
+wasm-bindgen-test = "0.3.34"
+
+[profile.release]
+# Tell `rustc` to optimize for small code size.
+opt-level = "s"
+debug = true

locustdb-client/README.md

@@ -0,0 +1,84 @@
+<div align="center">
+
+  <h1><code>wasm-pack-template</code></h1>
+
+  <strong>A template for kick starting a Rust and WebAssembly project using <a href="https://github.com/rustwasm/wasm-pack">wasm-pack</a>.</strong>
+
+  <p>
+    <a href="https://travis-ci.org/rustwasm/wasm-pack-template"><img src="https://img.shields.io/travis/rustwasm/wasm-pack-template.svg?style=flat-square" alt="Build Status" /></a>
+  </p>
+
+  <h3>
+    <a href="https://rustwasm.github.io/docs/wasm-pack/tutorials/npm-browser-packages/index.html">Tutorial</a>
+    <span> | </span>
+    <a href="https://discordapp.com/channels/442252698964721669/443151097398296587">Chat</a>
+  </h3>
+
+  <sub>Built with 🦀🕸 by <a href="https://rustwasm.github.io/">The Rust and WebAssembly Working Group</a></sub>
+</div>
+
+## About
+
+[**📚 Read this template tutorial! 📚**][template-docs]
+
+This template is designed for compiling Rust libraries into WebAssembly and
+publishing the resulting package to NPM.
+
+Be sure to check out [other `wasm-pack` tutorials online][tutorials] for other
+templates and usages of `wasm-pack`.
+
+[tutorials]: https://rustwasm.github.io/docs/wasm-pack/tutorials/index.html
+[template-docs]: https://rustwasm.github.io/docs/wasm-pack/tutorials/npm-browser-packages/index.html
+
+## 🚴 Usage
+
+### 🐑 Use `cargo generate` to Clone this Template
+
+[Learn more about `cargo generate` here.](https://github.com/ashleygwilliams/cargo-generate)
+
+```
+cargo generate --git https://github.com/rustwasm/wasm-pack-template.git --name my-project
+cd my-project
+```
+
+### 🛠️ Build with `wasm-pack build`
+
+```
+wasm-pack build
+```
+
+### 🔬 Test in Headless Browsers with `wasm-pack test`
+
+```
+wasm-pack test --headless --firefox
+```
+
+### 🎁 Publish to NPM with `wasm-pack publish`
+
+```
+wasm-pack publish
+```
+
+## 🔋 Batteries Included
+
+* [`wasm-bindgen`](https://github.com/rustwasm/wasm-bindgen) for communicating
+  between WebAssembly and JavaScript.
+* [`console_error_panic_hook`](https://github.com/rustwasm/console_error_panic_hook)
+  for logging panic messages to the developer console.
+* `LICENSE-APACHE` and `LICENSE-MIT`: most Rust projects are licensed this way, so these are included for you
+
+## License
+
+Licensed under either of
+
+* Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
+* MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)
+
+at your option.
+
+### Contribution
+
+Unless you explicitly state otherwise, any contribution intentionally
+submitted for inclusion in the work by you, as defined in the Apache-2.0
+license, shall be dual licensed as above, without any additional terms or
+conditions.

locustdb-client/src/client.rs

@@ -0,0 +1,137 @@
+use reqwest::header::CONTENT_TYPE;
+use serde::{Deserialize, Serialize};
+use wasm_bindgen::prelude::*;
+use locustdb_compression_utils::column;
+use std::collections::HashMap;
+use std::mem;
+
+use super::log;
+
+#[wasm_bindgen]
+pub struct Client {
+    client: reqwest::Client,
+    url: String,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+struct ColumnNameRequest {
+    tables: Vec<String>,
+    pattern: Option<String>,
+    offset: Option<usize>,
+    limit: Option<usize>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+struct ColumnNameResponse {
+    columns: Vec<String>,
+    offset: usize,
+    len: usize,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+struct MultiQueryRequest {
+    queries: Vec<String>,
+    encoding_opts: Option<EncodingOpts>,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct EncodingOpts {
+    pub xor_float_compression: bool,
+    pub mantissa: Option<u32>,
+    pub full_precision_cols: Vec<String>,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct QueryResponse {
+    pub columns: HashMap<String, column::Column>,
+}
+
+#[wasm_bindgen]
+impl Client {
+    pub fn new(url: &str) -> Client {
+        #[cfg(feature = "console_error_panic_hook")]
+        console_error_panic_hook::set_once();
+
+        Client {
+            client: reqwest::Client::new(),
+            url: url.to_string(),
+        }
+    }
+
+    pub async fn request_columns(&self, table: &str) -> Result<JsValue, JsValue> {
+        let request_body = ColumnNameRequest {
+            tables: vec![table.to_string()],
+            pattern: None,
+            offset: None,
+            limit: None,
+        };
+        //let start_time = Instant::now();
+        let response = self
+            .client
+            .post(&format!("{}/columns", self.url))
+            .header(CONTENT_TYPE, "application/json")
+            .json(&request_body)
+            .send()
+            .await
+            .unwrap();
+        let rsps: ColumnNameResponse = response.json().await.unwrap();
+        Ok(serde_wasm_bindgen::to_value(&rsps.columns).unwrap())
+    }
+
+    pub async fn multi_query(&self, queries: Vec<String>, binary: bool, compress: bool, mantissa: u32, full_precision_cols: Vec<String>) -> Result<JsValue, JsValue> {
+        let window = web_sys::window().expect("should have a window in this context");
+        let performance = window
+            .performance()
+            .expect("performance should be available");
+
+        let request_body = MultiQueryRequest {
+            queries,
+            encoding_opts: if binary {
+                Some(EncodingOpts {
+                    xor_float_compression: compress,
+                    mantissa: Some(mantissa),
+                    full_precision_cols,
+                }) } else { None },
+        };
+        let request_start_ms = performance.now();
+        let response = self
+            .client
+            .post(&format!("{}/multi_query_cols", self.url))
+            .header(CONTENT_TYPE, "application/json")
+            .json(&request_body)
+            .send()
+            .await
+            .unwrap();
+        let first_response_ms = performance.now();
+        let bytes = response.bytes().await.unwrap().to_vec();
+        let download_finished_ms = performance.now();
+
+        let rsps = if binary {
+            let mut rsps: Vec<QueryResponse> = bincode::deserialize(&bytes).unwrap();
+            rsps.iter_mut().for_each(|rsp| {
+                rsp.columns.iter_mut().for_each(|(key, col)| {
+                    let coltype = match col {
+                        column::Column::Float(_) => "float",
+                        column::Column::Int(_) => "int",
+                        column::Column::String(_) => "string",
+                        column::Column::Mixed(_) => "mixed",
+                        column::Column::Null(_) => "null",
+                        column::Column::Xor(_) => "xor",
+                    };
+                    log(&format!("decompressing column {key} of type {coltype}"));
+                    *col = mem::replace(col, column::Column::Null(0)).decompress();
+                });
+            });
+            log(&format!(
+                "waiting on server {:.2?} ms, downloading {:.2?} ms, deserializing {:.2?} ms",
+                first_response_ms - request_start_ms,
+                download_finished_ms - first_response_ms,
+                performance.now() - download_finished_ms,
+            ));
+            rsps
+        } else {
+            vec![]
+        };
+        Ok(serde_wasm_bindgen::to_value(&rsps).unwrap())
+    }
+}

locustdb-client/src/lib.rs

@@ -0,0 +1,17 @@
+use wasm_bindgen::prelude::*;
+mod client;
+
+#[wasm_bindgen]
+extern "C" {
+    fn alert(s: &str);
+
+    #[wasm_bindgen(js_namespace = console)]
+    fn log(s: &str);
+}
+
+#[wasm_bindgen]
+pub fn greet() {
+    alert("Hello, locustdb-client!");
+}
+
+pub use client::Client;

locustdb-client/tests/web.rs

@@ -0,0 +1,13 @@
+//! Test suite for the Web and headless browsers.
+
+#![cfg(target_arch = "wasm32")]
+
+extern crate wasm_bindgen_test;
+use wasm_bindgen_test::*;
+
+wasm_bindgen_test_configure!(run_in_browser);
+
+#[wasm_bindgen_test]
+fn pass() {
+    assert_eq!(1 + 1, 2);
+}