From 015c2b11a00e1b5e566a1d4f96d2cfbed80ad5b7 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Mon, 30 Jul 2018 18:08:02 -0400 Subject: [PATCH] Added simple benchmark for tuple_access_strategy (#37) Added basic benchmark for tuple_access_strategy. Not sure it's a valid benchmark, but I wanted a microbenchmark in order to test the framework and demonstrate how to write future microbenchmarks. --- CMakeLists.txt | 10 ++- benchmark/CMakeLists.txt | 11 +++ benchmark/storage/CMakeLists.txt | 1 + .../tuple_access_strategy_benchmark.cpp | 60 +++++++++++++ benchmark/util/benchmark_main.cpp | 26 ++++++ cmake_modules/BuildUtils.cmake | 8 ++ test/include/common/harness.h | 9 +- test/include/common/test_util.h | 43 ++++----- .../storage/tuple_access_strategy_test_util.h | 88 ++++++++----------- test/include/util/container_test_util.h | 7 +- 10 files changed, 176 insertions(+), 87 deletions(-) create mode 100644 benchmark/CMakeLists.txt create mode 100644 benchmark/storage/CMakeLists.txt create mode 100644 benchmark/storage/tuple_access_strategy_benchmark.cpp create mode 100644 benchmark/util/benchmark_main.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index e1b161e194..82b3941b96 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -308,8 +308,9 @@ set(LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}") set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}") include_directories(${PROJECT_SOURCE_DIR}/src/include) -include_directories(${PROJECT_SOURCE_DIR}/test/include) include_directories(${PROJECT_SOURCE_DIR}/third_party) +include_directories(${PROJECT_SOURCE_DIR}/test/include) +include_directories(${PROJECT_SOURCE_DIR}/benchmark/include) set(TERRIER_LINK_LIBS ${TERRIER_LINK_LIBS} @@ -322,6 +323,13 @@ set(TERRIER_TEST_LINK_LIBS gtest_main ${CMAKE_DL_LIBS}) +set(TERRIER_BENCHMARK_LINK_LIBS + terrier_shared + benchmark_shared + test_shared + gtest) + # ---[ Subdirectories add_subdirectory(src) add_subdirectory(test) +add_subdirectory(benchmark) diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt new file mode 100644 index 0000000000..97ec8d032b --- /dev/null +++ b/benchmark/CMakeLists.txt @@ -0,0 +1,11 @@ +file(GLOB_RECURSE BENCHMARK_HDRS ${PROJECT_SOURCE_DIR}/benchmark/include/*.h) +file(GLOB_RECURSE BENCHMARK_UTIL_SRCS ${PROJECT_SOURCE_DIR}/benchmark/util/*.cpp) +list(APPEND BENCHMARK_UTIL_SRCS ${BENCHMARK_HDRS}) + +############################################### +# Benchmark library +############################################### +add_library(benchmark_shared SHARED ${BENCHMARK_UTIL_SRCS}) +target_link_libraries(benchmark_shared benchmark) + +add_subdirectory(storage) diff --git a/benchmark/storage/CMakeLists.txt b/benchmark/storage/CMakeLists.txt new file mode 100644 index 0000000000..d742b54702 --- /dev/null +++ b/benchmark/storage/CMakeLists.txt @@ -0,0 +1 @@ +ADD_TERRIER_BENCHMARKS() diff --git a/benchmark/storage/tuple_access_strategy_benchmark.cpp b/benchmark/storage/tuple_access_strategy_benchmark.cpp new file mode 100644 index 0000000000..c4579ceb70 --- /dev/null +++ b/benchmark/storage/tuple_access_strategy_benchmark.cpp @@ -0,0 +1,60 @@ +#include "benchmark/benchmark.h" +#include "common/test_util.h" +#include "storage/tuple_access_strategy_test_util.h" + +namespace terrier { + +// Roughly corresponds to TEST_F(TupleAccessStrategyTests, SimpleInsertTest) +static void BM_SimpleInsert(benchmark::State &state) { + + // Get a BlockStore and then RawBlock to use for inserting into + storage::RawBlock *raw_block_ = nullptr; + storage::BlockStore block_store_{1}; + + // Number of times to repeat the Initialize and Insert loop + const uint32_t repeat = 3; + std::default_random_engine generator; + + // Tuple layout + uint16_t num_columns = 8; + uint8_t column_size = 8; + storage::BlockLayout layout(num_columns, + {column_size, column_size, column_size, column_size, column_size, column_size, + column_size, column_size}); + storage::TupleAccessStrategy tested(layout); + std::unordered_map tuples; + + while (state.KeepRunning()) { + + for (uint32_t i = 0; i < repeat; i++) { + // Get the Block, zero it, and initialize + raw_block_ = block_store_.Get(); + PELOTON_MEMSET(raw_block_, 0, sizeof(storage::RawBlock)); + storage::InitializeRawBlock(raw_block_, layout, 0); + + // Insert the maximum number of tuples into this Block + for (uint32_t j = 0; j < layout.num_slots_; j++) + testutil::TryInsertFakeTuple(layout, + tested, + raw_block_, + tuples, + generator); + + tuples.clear(); + block_store_.Release(raw_block_); + + } + + } + + // We want to approximate the amount of data processed so Google Benchmark can print stats for us + // We'll say it 2x RawBlock because we zero it, and then populate it. This is likely an underestimation + size_t bytes_per_repeat = 2 * sizeof(storage::RawBlock); + state.SetBytesProcessed(state.iterations() * repeat * bytes_per_repeat); +} + +BENCHMARK(BM_SimpleInsert) +->Repetitions(3)-> + Unit(benchmark::kMillisecond); + +} \ No newline at end of file diff --git a/benchmark/util/benchmark_main.cpp b/benchmark/util/benchmark_main.cpp new file mode 100644 index 0000000000..c353d649ea --- /dev/null +++ b/benchmark/util/benchmark_main.cpp @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// Modified from the Apache Arrow project for the Terrier project. + +#include "benchmark/benchmark.h" + +int main(int argc, char** argv) { + benchmark::Initialize(&argc, argv); + benchmark::RunSpecifiedBenchmarks(); + return 0; +} diff --git a/cmake_modules/BuildUtils.cmake b/cmake_modules/BuildUtils.cmake index abeb4d2d42..d18ae310a0 100755 --- a/cmake_modules/BuildUtils.cmake +++ b/cmake_modules/BuildUtils.cmake @@ -266,6 +266,14 @@ function(ADD_TERRIER_TESTS) endforeach() endfunction() +function(ADD_TERRIER_BENCHMARKS) + file(GLOB_RECURSE TERRIER_BENCHMARK_SOURCES "*.cpp") + foreach(FILE_FULL_PATH ${TERRIER_BENCHMARK_SOURCES}) + get_filename_component(BENCHMARK_NAME ${FILE_FULL_PATH} NAME_WE) + ADD_TERRIER_BENCHMARK(${BENCHMARK_NAME}) + endforeach() +endfunction() + ############################################################ # Fuzzing diff --git a/test/include/common/harness.h b/test/include/common/harness.h index cd86c1a04e..ac28745740 100644 --- a/test/include/common/harness.h +++ b/test/include/common/harness.h @@ -6,13 +6,10 @@ class TerrierTest : public ::testing::Test { protected: - virtual void SetUp() { - // turn on logger - terrier::Logger::InitializeLogger(); + // turn on logger + terrier::Logger::InitializeLogger(); } - virtual void TearDown() { - - } + virtual void TearDown() {} }; diff --git a/test/include/common/test_util.h b/test/include/common/test_util.h index 8e1d9229ed..02ab137242 100644 --- a/test/include/common/test_util.h +++ b/test/include/common/test_util.h @@ -1,9 +1,9 @@ #pragma once -#include #include +#include #include -#include "gtest/gtest.h" #include "common/object_pool.h" +#include "gtest/gtest.h" namespace terrier { namespace testutil { @@ -17,11 +17,9 @@ namespace testutil { * @param generator source of randomness to use * @return iterator to a randomly selected element */ -template -typename std::vector::iterator UniformRandomElement(std::vector &elems, - Random &generator) { - return elems.begin() - + std::uniform_int_distribution(0, (int) elems.size() - 1)(generator); +template +typename std::vector::iterator UniformRandomElement(std::vector &elems, Random &generator) { + return elems.begin() + std::uniform_int_distribution(0, (int)elems.size() - 1)(generator); }; /** @@ -32,15 +30,11 @@ typename std::vector::iterator UniformRandomElement(std::vector &elems, * @param workload the task the thread should run * @param repeat the number of times this should be done. */ -void RunThreadsUntilFinish(uint32_t num_threads, - const std::function &workload, - uint32_t repeat = 1) { +void RunThreadsUntilFinish(uint32_t num_threads, const std::function &workload, uint32_t repeat = 1) { for (uint32_t i = 0; i < repeat; i++) { std::vector threads; - for (uint32_t j = 0; j < num_threads; j++) - threads.emplace_back([j, &workload] { workload(j); }); - for (auto &thread : threads) - thread.join(); + for (uint32_t j = 0; j < num_threads; j++) threads.emplace_back([j, &workload] { workload(j); }); + for (auto &thread : threads) thread.join(); } } @@ -57,15 +51,12 @@ void RunThreadsUntilFinish(uint32_t num_threads, * @param generator source of randomness to use * @param repeat the number of times this should be done. */ -template -void InvokeWorkloadWithDistribution(std::vector> workloads, - std::vector probabilities, - Random &generator, - uint32_t repeat = 1) { +template +void InvokeWorkloadWithDistribution(std::vector> workloads, std::vector probabilities, + Random &generator, uint32_t repeat = 1) { PELOTON_ASSERT(probabilities.size() == workloads.size()); std::discrete_distribution dist(probabilities.begin(), probabilities.end()); - for (uint32_t i = 0; i < repeat; i++) - workloads[dist(generator)](); + for (uint32_t i = 0; i < repeat; i++) workloads[dist(generator)](); } #define TO_INT(p) reinterpret_cast(p) @@ -78,7 +69,7 @@ void InvokeWorkloadWithDistribution(std::vector> workloads * @param lower lower bound * @param upper upper bound */ -template +template void CheckInBounds(A *val, B *lower, C *upper) { EXPECT_GE(TO_INT(val), TO_INT(lower)); EXPECT_LT(TO_INT(val), TO_INT(upper)); @@ -93,7 +84,7 @@ void CheckInBounds(A *val, B *lower, C *upper) { * @param lower lower bound * @param upper upper bound */ -template +template void CheckNotInBounds(A *val, B *lower, C *upper) { EXPECT_TRUE(TO_INT(val) < TO_INT(lower) || TO_INT(val) >= TO_INT(upper)); }; @@ -104,9 +95,9 @@ void CheckNotInBounds(A *val, B *lower, C *upper) { * @param bytes bytes to advance * @return pointer that is the specified amount of bytes ahead of the given */ -template +template A *IncrementByBytes(A *ptr, uint64_t bytes) { return reinterpret_cast(reinterpret_cast(ptr) + bytes); } -} -} \ No newline at end of file +} // namespace testutil +} // namespace terrier diff --git a/test/include/storage/tuple_access_strategy_test_util.h b/test/include/storage/tuple_access_strategy_test_util.h index 36eb73a537..e7c1a25cd3 100644 --- a/test/include/storage/tuple_access_strategy_test_util.h +++ b/test/include/storage/tuple_access_strategy_test_util.h @@ -2,25 +2,22 @@ #include #include +#include "common/test_util.h" +#include "common/typedefs.h" #include "gtest/gtest.h" #include "storage/tuple_access_strategy.h" -#include "common/typedefs.h" -#include "common/test_util.h" namespace terrier { namespace testutil { // Returns a random layout that is guaranteed to be valid. -template -storage::BlockLayout RandomLayout(Random &generator, - uint16_t max_cols = UINT16_MAX) { +template +storage::BlockLayout RandomLayout(Random &generator, uint16_t max_cols = UINT16_MAX) { PELOTON_ASSERT(max_cols > 1); // We probably won't allow tables with 0 columns - uint16_t num_attrs = - std::uniform_int_distribution(1, max_cols)(generator); + uint16_t num_attrs = std::uniform_int_distribution(1, max_cols)(generator); std::vector possible_attr_sizes{1, 2, 4, 8}, attr_sizes(num_attrs); for (uint16_t i = 0; i < num_attrs; i++) - attr_sizes[i] = - *testutil::UniformRandomElement(possible_attr_sizes, generator); + attr_sizes[i] = *testutil::UniformRandomElement(possible_attr_sizes, generator); return {num_attrs, attr_sizes}; } @@ -28,10 +25,14 @@ storage::BlockLayout RandomLayout(Random &generator, // an integer of given size. (Thus only 1, 2, 4, 8 are allowed) uint64_t ReadByteValue(uint8_t attr_size, byte *pos) { switch (attr_size) { - case 1: return *reinterpret_cast(pos); - case 2: return *reinterpret_cast(pos); - case 4: return *reinterpret_cast(pos); - case 8: return *reinterpret_cast( pos); + case 1: + return *reinterpret_cast(pos); + case 2: + return *reinterpret_cast(pos); + case 4: + return *reinterpret_cast(pos); + case 8: + return *reinterpret_cast(pos); default: // Invalid attr size PELOTON_ASSERT(false); @@ -44,13 +45,17 @@ uint64_t ReadByteValue(uint8_t attr_size, byte *pos) { // Truncated if neccessary void WriteByteValue(uint8_t attr_size, uint64_t val, byte *pos) { switch (attr_size) { - case 1:*reinterpret_cast(pos) = static_cast(val); + case 1: + *reinterpret_cast(pos) = static_cast(val); return; - case 2:*reinterpret_cast(pos) = static_cast(val); + case 2: + *reinterpret_cast(pos) = static_cast(val); return; - case 4:*reinterpret_cast(pos) = static_cast(val); + case 4: + *reinterpret_cast(pos) = static_cast(val); return; - case 8:*reinterpret_cast(pos) = static_cast(val); + case 8: + *reinterpret_cast(pos) = static_cast(val); return; default: // Invalid attr size @@ -60,11 +65,10 @@ void WriteByteValue(uint8_t attr_size, uint64_t val, byte *pos) { // Fill the given location with the specified amount of random bytes, using the // given generator as a source of randomness. -template +template void FillWithRandomBytes(uint32_t num_bytes, byte *out, Random &generator) { std::uniform_int_distribution dist(0, UINT8_MAX); - for (uint32_t i = 0; i < num_bytes; i++) - out[i] = static_cast(dist(generator)); + for (uint32_t i = 0; i < num_bytes; i++) out[i] = static_cast(dist(generator)); } // This does NOT return a sensible tuple in general. This is just some filler @@ -81,16 +85,13 @@ struct FakeRawTuple { FillWithRandomBytes(layout.tuple_size_, contents_, generator); }; - ~FakeRawTuple() { - delete[] contents_; - } + ~FakeRawTuple() { delete[] contents_; } // Since all fields we store in pages are equal to or shorter than 8 bytes, // we can do equality checks on uint64_t always. // 0 return for non-primary key indexes should be treated as null. uint64_t Attribute(const storage::BlockLayout &layout, uint16_t col) { - return ReadByteValue(layout.attr_sizes_[col], - contents_ + attr_offsets_[col]); + return ReadByteValue(layout.attr_sizes_[col], contents_ + attr_offsets_[col]); } const storage::BlockLayout &layout_; @@ -100,16 +101,12 @@ struct FakeRawTuple { // Write the given fake tuple into a block using the given access strategy, // at the specified offset -void InsertTuple(FakeRawTuple &tuple, - storage::TupleAccessStrategy &tested, - const storage::BlockLayout &layout, +void InsertTuple(FakeRawTuple &tuple, storage::TupleAccessStrategy &tested, const storage::BlockLayout &layout, storage::TupleSlot slot) { for (uint16_t col = 0; col < layout.num_cols_; col++) { uint64_t col_val = tuple.Attribute(layout, col); if (col_val != 0 || col == PRIMARY_KEY_OFFSET) - WriteByteValue(layout.attr_sizes_[col], - tuple.Attribute(layout, col), - tested.AccessForceNotNull(slot, col)); + WriteByteValue(layout.attr_sizes_[col], tuple.Attribute(layout, col), tested.AccessForceNotNull(slot, col)); else tested.SetNull(slot, col); // Otherwise leave the field as null. @@ -117,9 +114,7 @@ void InsertTuple(FakeRawTuple &tuple, } // Check that the written tuple is the same as the expected one -void CheckTupleEqual(FakeRawTuple &expected, - storage::TupleAccessStrategy &tested, - const storage::BlockLayout &layout, +void CheckTupleEqual(FakeRawTuple &expected, storage::TupleAccessStrategy &tested, const storage::BlockLayout &layout, storage::TupleSlot slot) { for (uint16_t col = 0; col < layout.num_cols_; col++) { uint64_t expected_col = expected.Attribute(layout, col); @@ -128,8 +123,7 @@ void CheckTupleEqual(FakeRawTuple &expected, byte *col_slot = tested.AccessWithNullCheck(slot, col); if (!null) { EXPECT_TRUE(col_slot != nullptr); - EXPECT_EQ(expected.Attribute(layout, col), - ReadByteValue(layout.attr_sizes_[col], col_slot)); + EXPECT_EQ(expected.Attribute(layout, col), ReadByteValue(layout.attr_sizes_[col], col_slot)); } else { EXPECT_TRUE(col_slot == nullptr); } @@ -139,29 +133,23 @@ void CheckTupleEqual(FakeRawTuple &expected, // Using the given random generator, attempts to allocate a slot and write a // random tuple into it. The slot and the tuple are logged in the given map. // Checks are performed to make sure the insertion is sensible. -template +template std::pair &TryInsertFakeTuple( - const storage::BlockLayout &layout, - storage::TupleAccessStrategy &tested, - storage::RawBlock *block, - std::unordered_map &tuples, - Random &generator) { + const storage::BlockLayout &layout, storage::TupleAccessStrategy &tested, storage::RawBlock *block, + std::unordered_map &tuples, Random &generator) { storage::TupleSlot slot; // There should always be enough slots. EXPECT_TRUE(tested.Allocate(block, slot)); - EXPECT_TRUE(tested.ColumnNullBitmap(block, - PRIMARY_KEY_OFFSET)->Test(slot.GetOffset())); + EXPECT_TRUE(tested.ColumnNullBitmap(block, PRIMARY_KEY_OFFSET)->Test(slot.GetOffset())); // Construct a random tuple and associate it with the tuple slot - auto result = tuples.emplace( - std::piecewise_construct, - std::forward_as_tuple(slot), - std::forward_as_tuple(layout, generator)); + auto result = + tuples.emplace(std::piecewise_construct, std::forward_as_tuple(slot), std::forward_as_tuple(layout, generator)); // The tuple slot is not something that is already in use. EXPECT_TRUE(result.second); testutil::InsertTuple(result.first->second, tested, layout, slot); return *(result.first); } -} -} +} // namespace testutil +} // namespace terrier diff --git a/test/include/util/container_test_util.h b/test/include/util/container_test_util.h index fe242a73b5..c4fd7a38cf 100644 --- a/test/include/util/container_test_util.h +++ b/test/include/util/container_test_util.h @@ -3,11 +3,10 @@ #include namespace terrier { -template -void CheckReferenceBitmap(const Bitmap &tested, - const std::bitset &reference) { +template +void CheckReferenceBitmap(const Bitmap &tested, const std::bitset &reference) { for (uint32_t i = 0; i < num_elements; ++i) { EXPECT_EQ(reference[i], tested[i]); } } -} \ No newline at end of file +} // namespace terrier