*: revise docs and stores; add mixed example

README.md

@@ -6,40 +6,32 @@
 A benchmarking framework designed for testing key-value stores with easily customizable
 workloads.
 
-## Intro
+## Introduction
 
-This Rust crate enables the execution of tailored benchmarks on various key-value stores. Users
-have the flexibility to adjust benchmark and key-value store parameters and store them in a
-TOML-formatted file. The default command line interface is capable of loading these files and
+This Rust crate enables the execution of customizable benchmarks on various key-value stores.
+Users have the flexibility to adjust benchmark and key-value store parameters and store them
+in TOML-formatted files. The built-in command line interface is capable of loading these files and
 running the benchmarks as specified.
 
-In addition to standard single-process benchmarks, kvbench seamlessly incorporates a key-value
-client/server setup that operates with a dedicated server thread/machine.
+In addition to standard single-process benchmarks, it also seamlessly incorporates a key-value
+client/server setup that operates with a dedicated server thread or machine.
 
 ## Usage
 
-Without any changes, you can run the default command line interface with three modes:
+The [Documentation](https://docs.rs/kvbench) provides detailed usage guidelines.
 
-- `kvbench bench -s <STORE_FILE> -b <BENCH_FILE>` runs a (group of) benchmark using the parameters
-stored in `<STORE_FILE>` and `<BENCH_FILE>`.
-- `kvbench server -h <HOST> -p <PORT> -s <STORE_FILE> -n <THREADS>` boots up a key-value server
-with `<THREADS>` workers, listening on `<HOST>:<PORT>`, and uses the key-value stores specified in
-`<STORE_FILE>`.
-- `kvbench list` lists all registered key-value stores that can be used.
+## Examples
 
-See [examples](examples/) for more examples.
+- [examples/your-kv-store](examples/your-kv-store): How to integrate `kvbench` into your own
+key-value store implementations.
+- [examples/readpopular](examples/readpopular): A benchmark that reads popular records in
+the store, running with different number of threads.
+- [examples/writeheavy](examples/writeheavy): A benchmark that mixes reads and writes at 1:1
+on a random record in the store, running with different number of threads.
+- [examples/mixed](examples/mixed): A benchmark that consists of multiple phases (5 seconds
+running time each), benchmarked with 32 threads.
 
-## Configuration
-
-See the documentation of the modules `stores` and `bench` for available options.
-
-## Integration
-
-You can incorporate `kvbench` into your own key-value store implementations and run it
-against the built-in stores. All you need is implementing the necessary traits, depending on the
-type of the store, and call the default command line interface provided by this crate.
-
-See [examples/your-kv-store](examples/your-kv-store) for a minimal but concrete example.
+The built-in configuration files used by the above benchmarks can be found in [presets](presets).
 
 ## Development
 

examples/mixed/plot.gpl

@@ -0,0 +1,15 @@
+set terminal pdf
+set ylabel "Throughput (MOP/s)"
+set xlabel "Time (sec)"
+set key left top
+
+set output "mixed.pdf"
+plot [0:21] [0:] \
+    "chashmap.txt" using ($1*$7+1):13 with lp ti "chashmap",\
+    "contrie.txt" using ($1*$7+1):13 with lp ti "contrie",\
+    "dashmap.txt" using ($1*$7+1):13 with lp ti "dashmap",\
+    "flurry.txt" using ($1*$7+1):13 with lp ti "flurry",\
+    "papaya.txt" using ($1*$7+1):13 with lp ti "papaya",\
+    "scchashmap.txt" using ($1*$7+1):13 with lp ti "scchashmap",\
+    "mutex_hashmap.txt" using ($1*$7+1):13 with lp ti "mutexhashmap",\
+    "rwlock_hashmap.txt" using ($1*$7+1):13 with lp ti "rwlockhashmap"

examples/mixed/run.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+
+cargo build --profile release-lto
+
+STORE_DIR=$DIR/../../presets/stores
+BENCHMARK=$DIR/../../presets/benchmarks/mixed.toml
+
+STORES="chashmap contrie dashmap flurry papaya scchashmap mutex_hashmap rwlock_hashmap"
+
+for s in $STORES; do
+    env global.threads=32 cargo run --profile release-lto -- bench -s $STORE_DIR/$s.toml -b $BENCHMARK 2>/dev/null | tee $s.txt
+done

presets/benchmarks/mixed.toml

@@ -0,0 +1,49 @@
+[global]
+threads = 1
+repeat = 5
+klen = 8
+vlen = 16
+kmin = 0
+kmax = 1000000
+report = "repeat"
+
+[[benchmark]]
+set_perc = 100
+get_perc = 0
+del_perc = 0
+repeat = 1
+dist = "incrementp"
+report = "hidden"
+
+# write-intensive, zipfian
+[[benchmark]]
+set_perc = 50
+get_perc = 50
+del_perc = 0
+timeout = 1.0
+dist = "zipfian"
+
+# write-intensive, zipfian, hotspot in middle
+[[benchmark]]
+set_perc = 50
+get_perc = 50
+del_perc = 0
+timeout = 1.0
+dist = "zipfian"
+zipf_hotspot = 0.5
+
+# read-intensive, zipfian
+[[benchmark]]
+set_perc = 5
+get_perc = 95
+del_perc = 0
+timeout = 1.0
+dist = "zipfian"
+
+# read-only, uniform
+[[benchmark]]
+set_perc = 0
+get_perc = 100
+del_perc = 0
+timeout = 1.0
+dist = "uniform"

src/cmdline.rs

@@ -9,26 +9,32 @@ use std::sync::mpsc::channel;
 struct BenchArgs {
     #[arg(short = 's')]
     #[arg(value_hint = FilePath)]
+    #[arg(help = "Path to the key-value store's TOML config file")]
     store_config: String,
 
     #[arg(short = 'b')]
     #[arg(value_hint = FilePath)]
+    #[arg(help = "Path to the benchmark's TOML config file")]
     benchmark_config: String,
 }
 
 #[derive(Args, Debug)]
 struct ServerArgs {
-    #[arg(short = 'h', default_value = "0.0.0.0")]
+    #[arg(short = 'a', default_value = "0.0.0.0")]
+    #[arg(help = "Bind address")]
     host: String,
 
     #[arg(short = 'p', default_value = "9000")]
+    #[arg(help = "Bind port")]
     port: String,
 
     #[arg(short = 's')]
     #[arg(value_hint = FilePath)]
+    #[arg(help = "Path to the key-value store's TOML config file")]
     store_config: String,
 
     #[arg(short = 'n', default_value_t = 1)]
+    #[arg(help = "Number of worker threads")]
     workers: usize,
 }
 
@@ -41,8 +47,11 @@ struct Cli {
 
 #[derive(Subcommand, Debug)]
 enum Commands {
-    Server(ServerArgs),
+    #[command(about = "Run a benchmark")]
     Bench(BenchArgs),
+    #[command(about = "Start a key-value server")]
+    Server(ServerArgs),
+    #[command(about = "List all registered key-value stores")]
     List,
 }
 
@@ -98,6 +107,53 @@ fn list_cli() {
 /// This function is public and can be called in a different crate. For example, one can integrate
 /// their own key-value stores by registering the constructor function. Then, adding this function
 /// will produce a benchmark binary the has the same usage as the one in this crate.
+///
+/// ## Usage
+///
+/// To get the usage of the command line interface, users can run:
+///
+/// ```bash
+/// kvbench -h
+/// ```
+///
+/// The interface supports three modes, `bench`, `server` and `list`.
+///
+/// ### Benchmark Mode
+///
+/// Usage:
+///
+/// ```bash
+/// kvbench bench -s <STORE_CONFIG> -b <BENCH_CONFIG>
+/// ```
+///
+/// Where `STORE_CONFIG` and `BENCH_CONFIG` are the paths to the key-value store and benchmark
+/// configuration files, respectively. For their format, you can refer to the documentations of
+/// [`crate::stores`] and [`crate::bench`].
+///
+/// ### Server mode
+///
+/// Usage:
+///
+/// ```bash
+/// kvbench server -s <STORE_CONFIG> -a <HOST> -p <PORT> -n <WORKERS>
+/// ```
+///
+/// Where `STORE_CONFIG` is the path of the key-value store configuration file. Its format is
+/// documented in [`crate::stores`].
+///
+/// The default `HOST` and `PORT` are `0.0.0.0` and `9000`. By default, the server will spawn one
+/// worker thread only for incoming connections. You can adjust the number of worker threads by
+/// specifying `-n`.
+///
+/// ### List mode
+///
+/// Usage:
+/// ``` bash
+/// kvbench list
+/// ```
+///
+/// This command lists all registered key-value stores' names.
+
 pub fn cmdline() {
     env_logger::init();
     let cli = Cli::parse();

src/server.rs

@@ -652,7 +652,6 @@ pub fn init(text: &str) -> BenchKVMap {
 #[cfg(test)]
 mod tests {
     use super::*;
-
     use crate::stores::*;
     use std::sync::atomic::{AtomicU32, Ordering};
     use std::sync::mpsc::{channel, Receiver, Sender};
@@ -711,51 +710,51 @@ mod tests {
 
     #[test]
     fn simple_mutex() {
-        let opt = MutexHashMapOpt { shards: 512 };
-        let map = BenchKVMap::Regular(Box::new(MutexHashMap::new(&opt)));
+        let opt = hashmap::MutexHashMapOpt { shards: 512 };
+        let map = BenchKVMap::Regular(Box::new(hashmap::MutexHashMap::new(&opt)));
         simple(map);
     }
 
     #[test]
     fn simple_rwlock() {
-        let opt = RwLockHashMapOpt { shards: 512 };
-        let map = BenchKVMap::Regular(Box::new(RwLockHashMap::new(&opt)));
+        let opt = hashmap::RwLockHashMapOpt { shards: 512 };
+        let map = BenchKVMap::Regular(Box::new(hashmap::RwLockHashMap::new(&opt)));
         simple(map);
     }
 
     #[test]
     fn simple_dashmap() {
-        let map = BenchKVMap::Regular(Box::new(DashMap::new()));
+        let map = BenchKVMap::Regular(Box::new(dashmap::DashMap::new()));
         simple(map);
     }
 
     #[test]
     fn simple_contrie() {
-        let map = BenchKVMap::Regular(Box::new(Contrie::new()));
+        let map = BenchKVMap::Regular(Box::new(contrie::Contrie::new()));
         simple(map);
     }
 
     #[test]
     fn simple_chashmap() {
-        let map = BenchKVMap::Regular(Box::new(CHashMap::new()));
+        let map = BenchKVMap::Regular(Box::new(chashmap::CHashMap::new()));
         simple(map);
     }
 
     #[test]
     fn simple_scchashmap() {
-        let map = BenchKVMap::Regular(Box::new(SccHashMap::new()));
+        let map = BenchKVMap::Regular(Box::new(scc::SccHashMap::new()));
         simple(map);
     }
 
     #[test]
     fn simple_flurry() {
-        let map = BenchKVMap::Regular(Box::new(Flurry::new()));
+        let map = BenchKVMap::Regular(Box::new(flurry::Flurry::new()));
         simple(map);
     }
 
     #[test]
     fn simple_papaya() {
-        let map = BenchKVMap::Regular(Box::new(Papaya::new()));
+        let map = BenchKVMap::Regular(Box::new(papaya::Papaya::new()));
         simple(map);
     }
 
@@ -849,51 +848,51 @@ mod tests {
 
     #[test]
     fn batch_mutex() {
-        let opt = MutexHashMapOpt { shards: 512 };
-        let map = BenchKVMap::Regular(Box::new(MutexHashMap::new(&opt)));
+        let opt = hashmap::MutexHashMapOpt { shards: 512 };
+        let map = BenchKVMap::Regular(Box::new(hashmap::MutexHashMap::new(&opt)));
         batch(map);
     }
 
     #[test]
     fn batch_rwlock() {
-        let opt = RwLockHashMapOpt { shards: 512 };
-        let map = BenchKVMap::Regular(Box::new(RwLockHashMap::new(&opt)));
+        let opt = hashmap::RwLockHashMapOpt { shards: 512 };
+        let map = BenchKVMap::Regular(Box::new(hashmap::RwLockHashMap::new(&opt)));
         batch(map);
     }
 
     #[test]
     fn batch_dashmap() {
-        let map = BenchKVMap::Regular(Box::new(DashMap::new()));
+        let map = BenchKVMap::Regular(Box::new(dashmap::DashMap::new()));
         batch(map);
     }
 
     #[test]
     fn batch_contrie() {
-        let map = BenchKVMap::Regular(Box::new(Contrie::new()));
+        let map = BenchKVMap::Regular(Box::new(contrie::Contrie::new()));
         batch(map);
     }
 
     #[test]
     fn batch_chashmap() {
-        let map = BenchKVMap::Regular(Box::new(CHashMap::new()));
+        let map = BenchKVMap::Regular(Box::new(chashmap::CHashMap::new()));
         batch(map);
     }
 
     #[test]
     fn batch_scchashmap() {
-        let map = BenchKVMap::Regular(Box::new(SccHashMap::new()));
+        let map = BenchKVMap::Regular(Box::new(scc::SccHashMap::new()));
         batch(map);
     }
 
     #[test]
     fn batch_flurry() {
-        let map = BenchKVMap::Regular(Box::new(Flurry::new()));
+        let map = BenchKVMap::Regular(Box::new(flurry::Flurry::new()));
         batch(map);
     }
 
     #[test]
     fn batch_papaya() {
-        let map = BenchKVMap::Regular(Box::new(Papaya::new()));
+        let map = BenchKVMap::Regular(Box::new(papaya::Papaya::new()));
         batch(map);
     }
 }

src/stores/chashmap.rs

@@ -1,3 +1,13 @@
+//! Adapter implementation of [`chashmap::CHashMap`].
+//!
+//! ## Configuration Format
+//!
+//! ``` toml
+//! [map]
+//! name = "chashmap"
+//! ```
+//! This store is [`KVMap`].
+
 use crate::stores::{BenchKVMap, Registry};
 use crate::*;