From b5261a81e6e6a8c8d7a8bfab8f1ccda5191751af Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Fri, 20 Dec 2024 08:13:30 +0100 Subject: [PATCH 1/8] Improve benchmark timings --- exe/benchmark.cc | 63 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/exe/benchmark.cc b/exe/benchmark.cc index 661c073..49880da 100644 --- a/exe/benchmark.cc +++ b/exe/benchmark.cc @@ -1,4 +1,6 @@ +#include #include +#include #include #include @@ -26,8 +28,8 @@ class settings : public conf::configuration { explicit settings() : configuration("Options") { param(data_dir_, "data,d", "Data directory"); param(threads_, "threads,t", "Number of routing threads"); - param(n_queries_, "n", "Number of queries"); - param(max_dist_, "r", "Radius"); + param(n_queries_, ",n", "Number of queries"); + param(max_dist_, "radius,r", "Radius"); } fs::path data_dir_{"osr"}; @@ -36,6 +38,46 @@ class settings : public conf::configuration { unsigned threads_{std::thread::hardware_concurrency()}; }; +struct benchmark_result { + friend std::ostream& operator<<(std::ostream& out, + benchmark_result const& br) { + using double_milliseconds_t = + std::chrono::duration>; + out << "(duration: " << std::fixed << std::setprecision(3) << std::setw(10) + << std::chrono::duration_cast(br.duration_) + << ")"; + return out; + } + + std::chrono::microseconds duration_; +}; + +// needs sorted vector +benchmark_result quantile(std::vector const& v, double q) { + q = std::clamp(q, 0.0, 1.0); + if (q == 1.0) { + return v.back(); + } + return v[static_cast(v.size() * q)]; +} + +void print_result(std::vector const& var, + std::string const& var_name) { + std::cout << "\n--- " << var_name << " --- (n = " << var.size() << ")" + << "\n 10%: " << quantile(var, 0.1) + << "\n 20%: " << quantile(var, 0.2) + << "\n 30%: " << quantile(var, 0.3) + << "\n 40%: " << quantile(var, 0.4) + << "\n 50%: " << quantile(var, 0.5) + << "\n 60%: " << quantile(var, 0.6) + << "\n 70%: " << quantile(var, 0.7) + << "\n 80%: " << quantile(var, 0.8) + << "\n 90%: " << quantile(var, 0.9) + << "\n 99%: " << quantile(var, 0.99) + << "\n99.9%: " << quantile(var, 0.999) << "\n max: " << var.back() + << "\n-----------------------------\n"; +} + int main(int argc, char const* argv[]) { auto opt = settings{}; auto parser = conf::options_parser({&opt}); @@ -59,15 +101,18 @@ int main(int argc, char const* argv[]) { auto const w = ways{opt.data_dir_, cista::mmap::protection::READ}; - auto timer = utl::scoped_timer{"timer"}; auto threads = std::vector(std::max(1U, opt.threads_)); + auto results = std::vector{}; + results.reserve(opt.n_queries_); auto i = std::atomic_size_t{0U}; + auto m = std::mutex{}; for (auto& t : threads) { t = std::thread([&]() { auto d = dijkstra{}; auto h = cista::BASE_HASH; auto n = 0U; while (i.fetch_add(1U) < opt.n_queries_) { + auto const start_time = std::chrono::steady_clock::now(); auto const start = node_idx_t{cista::hash_combine(h, ++n, i.load()) % w.n_nodes()}; d.reset(opt.max_dist_); @@ -77,6 +122,13 @@ int main(int argc, char const* argv[]) { car::label{car::node{start, 0, direction::kBackward}, 0U}); d.run(w, *w.r_, opt.max_dist_, nullptr, nullptr); + auto const end_time = std::chrono::steady_clock::now(); + { + auto const guard = std::lock_guard{m}; + results.emplace_back( + std::chrono::duration_cast( + end_time - start_time)); + } } }); } @@ -84,4 +136,9 @@ int main(int argc, char const* argv[]) { for (auto& t : threads) { t.join(); } + + std::ranges::sort(results, std::less<>{}, + [](benchmark_result const& res) { return res.duration_; }); + + print_result(results, "duration"); } From 3dfb625d57efc950c617c2c20a4cd4a71f59e178 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Fri, 20 Dec 2024 09:58:02 +0100 Subject: [PATCH 2/8] Add average computation time --- exe/benchmark.cc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/exe/benchmark.cc b/exe/benchmark.cc index 49880da..6f119af 100644 --- a/exe/benchmark.cc +++ b/exe/benchmark.cc @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -63,7 +64,14 @@ benchmark_result quantile(std::vector const& v, double q) { void print_result(std::vector const& var, std::string const& var_name) { + auto const avg = benchmark_result{ + std::accumulate(var.begin(), var.end(), std::chrono::microseconds{0U}, + [](auto&& sum, auto const& res) { + return std::move(sum) + res.duration_; + }) / + var.size()}; std::cout << "\n--- " << var_name << " --- (n = " << var.size() << ")" + << "\n max: " << var.back() << "\n avg: " << avg << "\n----------" << "\n 10%: " << quantile(var, 0.1) << "\n 20%: " << quantile(var, 0.2) << "\n 30%: " << quantile(var, 0.3) @@ -74,7 +82,7 @@ void print_result(std::vector const& var, << "\n 80%: " << quantile(var, 0.8) << "\n 90%: " << quantile(var, 0.9) << "\n 99%: " << quantile(var, 0.99) - << "\n99.9%: " << quantile(var, 0.999) << "\n max: " << var.back() + << "\n99.9%: " << quantile(var, 0.999) << "\n-----------------------------\n"; } From 5211cb21c44dcaaf279b47a1cc5d138720591138 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Fri, 20 Dec 2024 11:20:34 +0100 Subject: [PATCH 3/8] Refactor test to run multiple profiles --- exe/benchmark.cc | 81 ++++++++++++++++++++++++++++++------------------ 1 file changed, 50 insertions(+), 31 deletions(-) diff --git a/exe/benchmark.cc b/exe/benchmark.cc index 6f119af..95d6b09 100644 --- a/exe/benchmark.cc +++ b/exe/benchmark.cc @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -13,10 +14,13 @@ #include "conf/options_parser.h" +#include "osr/routing/profile.h" +#include "osr/types.h" #include "utl/timer.h" #include "osr/lookup.h" #include "osr/routing/dijkstra.h" +#include "osr/routing/profiles/bike.h" #include "osr/routing/profiles/car.h" #include "osr/routing/route.h" #include "osr/ways.h" @@ -63,14 +67,15 @@ benchmark_result quantile(std::vector const& v, double q) { } void print_result(std::vector const& var, - std::string const& var_name) { + std::string const& profile) { auto const avg = benchmark_result{ std::accumulate(var.begin(), var.end(), std::chrono::microseconds{0U}, [](auto&& sum, auto const& res) { return std::move(sum) + res.duration_; }) / var.size()}; - std::cout << "\n--- " << var_name << " --- (n = " << var.size() << ")" + std::cout << "\n--- duration (" << profile << ") --- (n = " << var.size() + << ")" << "\n max: " << var.back() << "\n avg: " << avg << "\n----------" << "\n 10%: " << quantile(var, 0.1) << "\n 20%: " << quantile(var, 0.2) @@ -86,29 +91,19 @@ void print_result(std::vector const& var, << "\n-----------------------------\n"; } -int main(int argc, char const* argv[]) { - auto opt = settings{}; - auto parser = conf::options_parser({&opt}); - parser.read_command_line_args(argc, argv); - - if (parser.help()) { - parser.print_help(std::cout); - return 0; - } else if (parser.version()) { - return 0; - } - - parser.read_configuration_file(); - parser.print_unrecognized(std::cout); - parser.print_used(std::cout); - - if (!fs::is_directory(opt.data_dir_)) { - fmt::println("directory not found: {}", opt.data_dir_); - return 1; - } +template +void set_start(dijkstra& d, ways const& w, node_idx_t const start) { + d.add_start(w, typename T::label{typename T::node{start}, 0U}); +} - auto const w = ways{opt.data_dir_, cista::mmap::protection::READ}; +template <> +void set_start(dijkstra& d, ways const& w, node_idx_t const start) { + d.add_start(w, car::label{car::node{start, 0, direction::kForward}, 0U}); + d.add_start(w, car::label{car::node{start, 0, direction::kBackward}, 0U}); +}; +template +void run_benchmark(ways const& w, settings const& opt, const char* profile) { auto threads = std::vector(std::max(1U, opt.threads_)); auto results = std::vector{}; results.reserve(opt.n_queries_); @@ -116,7 +111,7 @@ int main(int argc, char const* argv[]) { auto m = std::mutex{}; for (auto& t : threads) { t = std::thread([&]() { - auto d = dijkstra{}; + auto d = dijkstra{}; auto h = cista::BASE_HASH; auto n = 0U; while (i.fetch_add(1U) < opt.n_queries_) { @@ -124,12 +119,9 @@ int main(int argc, char const* argv[]) { auto const start = node_idx_t{cista::hash_combine(h, ++n, i.load()) % w.n_nodes()}; d.reset(opt.max_dist_); - d.add_start(w, - car::label{car::node{start, 0, direction::kForward}, 0U}); - d.add_start(w, - car::label{car::node{start, 0, direction::kBackward}, 0U}); - d.run(w, *w.r_, opt.max_dist_, nullptr, - nullptr); + set_start(d, w, start); + d.template run(w, *w.r_, opt.max_dist_, + nullptr, nullptr); auto const end_time = std::chrono::steady_clock::now(); { auto const guard = std::lock_guard{m}; @@ -148,5 +140,32 @@ int main(int argc, char const* argv[]) { std::ranges::sort(results, std::less<>{}, [](benchmark_result const& res) { return res.duration_; }); - print_result(results, "duration"); + print_result(results, profile); +} + +int main(int argc, char const* argv[]) { + auto opt = settings{}; + auto parser = conf::options_parser({&opt}); + parser.read_command_line_args(argc, argv); + + if (parser.help()) { + parser.print_help(std::cout); + return 0; + } else if (parser.version()) { + return 0; + } + + parser.read_configuration_file(); + parser.print_unrecognized(std::cout); + parser.print_used(std::cout); + + if (!fs::is_directory(opt.data_dir_)) { + fmt::println("directory not found: {}", opt.data_dir_); + return 1; + } + + auto const w = ways{opt.data_dir_, cista::mmap::protection::READ}; + + run_benchmark(w, opt, "car"); + run_benchmark(w, opt, "bike"); } From 5d8b916f994f176a365d3e63faf184dc2ebe4959 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Fri, 20 Dec 2024 11:28:09 +0100 Subject: [PATCH 4/8] Improve output format --- exe/benchmark.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/exe/benchmark.cc b/exe/benchmark.cc index 95d6b09..3cbcfe7 100644 --- a/exe/benchmark.cc +++ b/exe/benchmark.cc @@ -74,8 +74,7 @@ void print_result(std::vector const& var, return std::move(sum) + res.duration_; }) / var.size()}; - std::cout << "\n--- duration (" << profile << ") --- (n = " << var.size() - << ")" + std::cout << "\n--- profile: " << profile << " --- (n = " << var.size() << ")" << "\n max: " << var.back() << "\n avg: " << avg << "\n----------" << "\n 10%: " << quantile(var, 0.1) << "\n 20%: " << quantile(var, 0.2) From 9676ebf85a8829839cbdddb26c979cc11e2f6ba6 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Fri, 20 Dec 2024 11:42:22 +0100 Subject: [PATCH 5/8] Move code into lambda --- exe/benchmark.cc | 88 +++++++++++++++++++++++++----------------------- 1 file changed, 45 insertions(+), 43 deletions(-) diff --git a/exe/benchmark.cc b/exe/benchmark.cc index 3cbcfe7..f356454 100644 --- a/exe/benchmark.cc +++ b/exe/benchmark.cc @@ -101,47 +101,6 @@ void set_start(dijkstra& d, ways const& w, node_idx_t const start) { d.add_start(w, car::label{car::node{start, 0, direction::kBackward}, 0U}); }; -template -void run_benchmark(ways const& w, settings const& opt, const char* profile) { - auto threads = std::vector(std::max(1U, opt.threads_)); - auto results = std::vector{}; - results.reserve(opt.n_queries_); - auto i = std::atomic_size_t{0U}; - auto m = std::mutex{}; - for (auto& t : threads) { - t = std::thread([&]() { - auto d = dijkstra{}; - auto h = cista::BASE_HASH; - auto n = 0U; - while (i.fetch_add(1U) < opt.n_queries_) { - auto const start_time = std::chrono::steady_clock::now(); - auto const start = - node_idx_t{cista::hash_combine(h, ++n, i.load()) % w.n_nodes()}; - d.reset(opt.max_dist_); - set_start(d, w, start); - d.template run(w, *w.r_, opt.max_dist_, - nullptr, nullptr); - auto const end_time = std::chrono::steady_clock::now(); - { - auto const guard = std::lock_guard{m}; - results.emplace_back( - std::chrono::duration_cast( - end_time - start_time)); - } - } - }); - } - - for (auto& t : threads) { - t.join(); - } - - std::ranges::sort(results, std::less<>{}, - [](benchmark_result const& res) { return res.duration_; }); - - print_result(results, profile); -} - int main(int argc, char const* argv[]) { auto opt = settings{}; auto parser = conf::options_parser({&opt}); @@ -165,6 +124,49 @@ int main(int argc, char const* argv[]) { auto const w = ways{opt.data_dir_, cista::mmap::protection::READ}; - run_benchmark(w, opt, "car"); - run_benchmark(w, opt, "bike"); + auto threads = std::vector(std::max(1U, opt.threads_)); + auto results = std::vector{}; + results.reserve(opt.n_queries_); + + auto const run_benchmark = [&](const char* profile) { + results.clear(); + auto i = std::atomic_size_t{0U}; + auto m = std::mutex{}; + for (auto& t : threads) { + t = std::thread([&]() { + auto d = dijkstra{}; + auto h = cista::BASE_HASH; + auto n = 0U; + while (i.fetch_add(1U) < opt.n_queries_) { + auto const start_time = std::chrono::steady_clock::now(); + auto const start = + node_idx_t{cista::hash_combine(h, ++n, i.load()) % w.n_nodes()}; + d.reset(opt.max_dist_); + set_start(d, w, start); + d.template run(w, *w.r_, opt.max_dist_, + nullptr, nullptr); + auto const end_time = std::chrono::steady_clock::now(); + { + auto const guard = std::lock_guard{m}; + results.emplace_back(std::chrono::duration_cast< + decltype(benchmark_result::duration_)>( + end_time - start_time)); + } + } + }); + } + + for (auto& t : threads) { + t.join(); + } + + std::ranges::sort(results, std::less<>{}, [](benchmark_result const& res) { + return res.duration_; + }); + + print_result(results, profile); + }; + + run_benchmark.template operator()("car"); + run_benchmark.template operator()("bike"); } From 4ce7143d1b98ac7f090057053fab566910d52017 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Fri, 20 Dec 2024 12:08:06 +0100 Subject: [PATCH 6/8] Fix compile error requiring C++20 --- exe/benchmark.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/exe/benchmark.cc b/exe/benchmark.cc index f356454..c7baf61 100644 --- a/exe/benchmark.cc +++ b/exe/benchmark.cc @@ -50,7 +50,8 @@ struct benchmark_result { std::chrono::duration>; out << "(duration: " << std::fixed << std::setprecision(3) << std::setw(10) << std::chrono::duration_cast(br.duration_) - << ")"; + .count() + << "ms)"; return out; } From 5660f54962809b84002c2c67288856aacb4a4048 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Fri, 20 Dec 2024 12:42:59 +0100 Subject: [PATCH 7/8] Attempt to fix build error --- exe/benchmark.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/exe/benchmark.cc b/exe/benchmark.cc index c7baf61..a4d52d1 100644 --- a/exe/benchmark.cc +++ b/exe/benchmark.cc @@ -149,9 +149,8 @@ int main(int argc, char const* argv[]) { auto const end_time = std::chrono::steady_clock::now(); { auto const guard = std::lock_guard{m}; - results.emplace_back(std::chrono::duration_cast< - decltype(benchmark_result::duration_)>( - end_time - start_time)); + results.emplace_back(benchmark_result{std::chrono::duration_cast< + decltype(benchmark_result::duration_)>(end_time - start_time)}); } } }); From 5dad8ed66d4699a7753e2e41f33c938e7035b427 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Fri, 20 Dec 2024 13:09:34 +0100 Subject: [PATCH 8/8] Fix header order --- exe/benchmark.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exe/benchmark.cc b/exe/benchmark.cc index a4d52d1..2b510e8 100644 --- a/exe/benchmark.cc +++ b/exe/benchmark.cc @@ -14,15 +14,15 @@ #include "conf/options_parser.h" -#include "osr/routing/profile.h" -#include "osr/types.h" #include "utl/timer.h" #include "osr/lookup.h" #include "osr/routing/dijkstra.h" +#include "osr/routing/profile.h" #include "osr/routing/profiles/bike.h" #include "osr/routing/profiles/car.h" #include "osr/routing/route.h" +#include "osr/types.h" #include "osr/ways.h" namespace fs = std::filesystem;