From 7518042239286c4424fff2738a4e1eaef8be0a8c Mon Sep 17 00:00:00 2001 From: Karol Sewilo Date: Mon, 13 Jan 2025 14:52:31 +0100 Subject: [PATCH] Add benchmarks commit-id:5c103759 --- .gitignore | 1 + Cargo.lock | 271 +++++++++++++++++- Cargo.toml | 1 + crates/cairo-coverage-core/Cargo.toml | 5 + .../cairo-coverage-core/benches/bench_main.rs | 7 + .../benches/benchmarks/mod.rs | 20 ++ .../benches/benchmarks/starknet_staking.rs | 25 ++ .../benches/download_bench_projects.sh | 40 +++ crates/cairo-coverage-core/src/args.rs | 2 +- crates/cairo-coverage-test-utils/src/lib.rs | 14 + .../tests/helpers/test_project.rs | 9 +- 11 files changed, 383 insertions(+), 12 deletions(-) create mode 100644 crates/cairo-coverage-core/benches/bench_main.rs create mode 100644 crates/cairo-coverage-core/benches/benchmarks/mod.rs create mode 100644 crates/cairo-coverage-core/benches/benchmarks/starknet_staking.rs create mode 100644 crates/cairo-coverage-core/benches/download_bench_projects.sh diff --git a/.gitignore b/.gitignore index 319ed95..b2e8217 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ debug target .idea +project-traces .snfoundry_cache .spr.yml Scarb.lock diff --git a/Cargo.lock b/Cargo.lock index 8945055..ebe66d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,6 +29,12 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "0.6.18" @@ -65,7 +71,7 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -75,7 +81,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" dependencies = [ "anstyle", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -178,6 +184,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + [[package]] name = "byte-slice-cast" version = "1.2.2" @@ -228,6 +240,7 @@ dependencies = [ "cairo-lang-sierra-to-casm", "cairo-lang-starknet-classes", "camino", + "criterion", "derived-deref", "ignore", "indoc", @@ -406,12 +419,45 @@ dependencies = [ "serde", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "clap" version = "4.5.23" @@ -482,6 +528,42 @@ dependencies = [ "libc", ] +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -643,7 +725,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -725,6 +807,16 @@ dependencies = [ "microlp", ] +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -760,6 +852,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" + [[package]] name = "ignore" version = "0.4.23" @@ -815,12 +913,32 @@ version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" +[[package]] +name = "is-terminal" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.11.0" @@ -854,6 +972,16 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" +[[package]] +name = "js-sys" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + [[package]] name = "keccak" version = "0.1.5" @@ -1143,6 +1271,34 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "portable-atomic" version = "1.10.0" @@ -1230,6 +1386,26 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.8" @@ -1324,7 +1500,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -1602,7 +1778,7 @@ dependencies = [ "getrandom", "once_cell", "rustix", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -1651,6 +1827,16 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "toml_datetime" version = "0.6.8" @@ -1761,6 +1947,70 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" +dependencies = [ + "cfg-if", + "once_cell", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn 2.0.95", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.95", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" + +[[package]] +name = "web-sys" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi" version = "0.3.9" @@ -1783,7 +2033,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -1792,6 +2042,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.59.0" diff --git a/Cargo.toml b/Cargo.toml index ef732af..9782041 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ anyhow = "1.0.95" assert_fs = "1.1.2" camino = "1.1.9" clap = { version = "4.5.23", features = ["derive"] } +criterion = "0.5" cairo-annotations = "0.2.1" cairo-lang-sierra = "2.9.2" cairo-lang-sierra-to-casm = "2.9.2" diff --git a/crates/cairo-coverage-core/Cargo.toml b/crates/cairo-coverage-core/Cargo.toml index 1de1bd9..69039d3 100644 --- a/crates/cairo-coverage-core/Cargo.toml +++ b/crates/cairo-coverage-core/Cargo.toml @@ -20,4 +20,9 @@ indoc.workspace = true [dev-dependencies] cairo-coverage-test-utils = { path = "../cairo-coverage-test-utils" } +criterion.workspace = true assert_fs.workspace = true + +[[bench]] +name = "bench_main" +harness = false diff --git a/crates/cairo-coverage-core/benches/bench_main.rs b/crates/cairo-coverage-core/benches/bench_main.rs new file mode 100644 index 0000000..6d54753 --- /dev/null +++ b/crates/cairo-coverage-core/benches/bench_main.rs @@ -0,0 +1,7 @@ +mod benchmarks; + +use criterion::criterion_main; + +criterion_main! { + benchmarks::starknet_staking::benches +} diff --git a/crates/cairo-coverage-core/benches/benchmarks/mod.rs b/crates/cairo-coverage-core/benches/benchmarks/mod.rs new file mode 100644 index 0000000..7cb0d33 --- /dev/null +++ b/crates/cairo-coverage-core/benches/benchmarks/mod.rs @@ -0,0 +1,20 @@ +use cairo_coverage_test_utils::read_files_from_dir; +use camino::Utf8PathBuf; +use criterion::Criterion; +use std::time::Duration; + +pub mod starknet_staking; + +/// Wrapper around `read_files_from_dir` to make it easier to use in benchmarks. +fn trace_files_for_benches(dir_name: &str) -> Vec { + read_files_from_dir(format!("benches/project-traces/{dir_name}/trace")) +} + +/// Config of [`Criterion`] that should be used for all benchmarks. +/// We lower the sample sizes so benchmarks finish faster and increase +/// the measurement time to get more accurate results and don't get warnings. +fn config() -> Criterion { + Criterion::default() + .sample_size(10) + .measurement_time(Duration::from_secs(200)) +} diff --git a/crates/cairo-coverage-core/benches/benchmarks/starknet_staking.rs b/crates/cairo-coverage-core/benches/benchmarks/starknet_staking.rs new file mode 100644 index 0000000..562b3a6 --- /dev/null +++ b/crates/cairo-coverage-core/benches/benchmarks/starknet_staking.rs @@ -0,0 +1,25 @@ +use crate::benchmarks::{config, trace_files_for_benches}; +use cairo_coverage_core::args::RunOptions; +use criterion::{criterion_group, Criterion}; +use std::hint::black_box; + +/// Benchmark of [`starknet_staking`](https://github.com/starkware-libs/starknet-staking) with default options. +/// The trace files should be generated using `download_bench_project.sh` script. +fn starknet_staking_benchmark(c: &mut Criterion) { + let trace_files = trace_files_for_benches("starknet-staking"); + let run_options = RunOptions::default(); + c.bench_function("starknet-staking default options", |b| { + b.iter(|| { + cairo_coverage_core::run( + black_box(trace_files.clone()), + black_box(run_options.clone()), + ) + }); + }); +} + +criterion_group! { + name = benches; + config = config(); + targets = starknet_staking_benchmark +} diff --git a/crates/cairo-coverage-core/benches/download_bench_projects.sh b/crates/cairo-coverage-core/benches/download_bench_projects.sh new file mode 100644 index 0000000..35ba54b --- /dev/null +++ b/crates/cairo-coverage-core/benches/download_bench_projects.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -euxo pipefail + +create_traces() { + if [[ "$#" -ne 2 ]]; then + echo "Usage: create_traces " + return 1 + fi + + local origin_link="$1" + local commit_hash="$2" + local repo_name + repo_name=$(basename "$origin_link" .git) + + mkdir -p project-traces + pushd project-traces + + # Initialize and fetch the repository + git init "$repo_name" + pushd "$repo_name" + + git fetch --depth 1 "$origin_link" "$commit_hash" + git checkout "$commit_hash" + + # Run tests and generate trace data + snforge test --save-trace-data + + # Prepare tracing directory + mkdir -p trace + find . -type d -name "snfoundry_trace" -exec cp -R {}/. ./trace \; + + # Clean up unnecessary files + find . -mindepth 1 ! \( -path "./trace/*" -o -path "./target/dev/*" -o -name "trace" \) -delete + + popd + popd +} + + +create_traces "git@github.com:starkware-libs/starknet-staking.git" "197e94c0cd10a11a44d261b27f2150c6aab3a25d" \ No newline at end of file diff --git a/crates/cairo-coverage-core/src/args.rs b/crates/cairo-coverage-core/src/args.rs index 0e61a43..03a217e 100644 --- a/crates/cairo-coverage-core/src/args.rs +++ b/crates/cairo-coverage-core/src/args.rs @@ -1,7 +1,7 @@ use camino::Utf8PathBuf; /// Options accepted by `cairo_coverage_core` `run` function. -#[derive(Default)] +#[derive(Default, Clone)] pub struct RunOptions { /// Include additional components in the coverage report. pub include: Vec, diff --git a/crates/cairo-coverage-test-utils/src/lib.rs b/crates/cairo-coverage-test-utils/src/lib.rs index a523654..093272c 100644 --- a/crates/cairo-coverage-test-utils/src/lib.rs +++ b/crates/cairo-coverage-test-utils/src/lib.rs @@ -1,7 +1,21 @@ use assert_fs::fixture::{ChildPath, FileTouch, PathChild}; use camino::Utf8PathBuf; +use std::fs; use std::path::Path; +/// Function to read all files from a directory and return their paths as [`Utf8PathBuf`]. +/// # Panics +/// - if [`fs::read_dir`] panics for the `dir_name` provided +/// - if conversion from [`PathBuf`] to [`Utf8PathBuf`] fails +pub fn read_files_from_dir(dir_name: impl AsRef) -> Vec { + fs::read_dir(dir_name) + .unwrap() + .flatten() + .map(|entry| entry.path().to_utf8_path_buf()) + .filter(|path| path.is_file()) + .collect() +} + /// Utility trait for creating files using [`assert_fs`]. pub trait CreateFile { fn create_file(&self, file_name: &str) -> ChildPath; diff --git a/crates/cairo-coverage/tests/helpers/test_project.rs b/crates/cairo-coverage/tests/helpers/test_project.rs index 4ad3261..01a15dc 100644 --- a/crates/cairo-coverage/tests/helpers/test_project.rs +++ b/crates/cairo-coverage/tests/helpers/test_project.rs @@ -1,5 +1,7 @@ use assert_fs::fixture::PathCopy; use assert_fs::TempDir; +use cairo_coverage_test_utils::read_files_from_dir; +use camino::Utf8PathBuf; use snapbox::cmd::{cargo_bin, Command as SnapboxCommand}; use std::fs; use std::path::PathBuf; @@ -60,12 +62,9 @@ impl TestProject { self } - fn find_trace_files(&self) -> Vec { + fn find_trace_files(&self) -> Vec { let trace_path = self.dir.path().join("snfoundry_trace"); - fs::read_dir(&trace_path) - .unwrap() - .map(|entry| entry.unwrap().path().display().to_string()) - .collect() + read_files_from_dir(trace_path) } fn output_lcov_path(&self) -> PathBuf {