From 10302cb82c8b0b6a2628c07dbef63beb10a6e232 Mon Sep 17 00:00:00 2001 From: sho-iizuka Date: Mon, 29 Jan 2024 05:13:23 +0000 Subject: [PATCH 1/4] Add initial_repl_sleep_delay configuration option This commit adds a new configuration option, initial_repl_sleep_delay, which allows the developer to specify the sleep delay for the GC thread during the initial replication process. This helps prevent errors related to the replication buffer being full. --- cybozu/config_parser.hpp | 18 ++++++++++++++++++ docs/usage.md | 2 ++ etc/yrmcds.conf | 6 ++++++ src/config.cpp | 6 ++++++ src/config.hpp | 4 ++++ src/constants.hpp | 1 + src/memcache/gc.cpp | 15 +++++++++++++++ test/config.cpp | 1 + test/test.conf | 1 + 9 files changed, 54 insertions(+) diff --git a/cybozu/config_parser.hpp b/cybozu/config_parser.hpp index 62dbbf1..30596cf 100644 --- a/cybozu/config_parser.hpp +++ b/cybozu/config_parser.hpp @@ -7,6 +7,7 @@ #include #include #include +#include namespace cybozu { @@ -103,6 +104,23 @@ class config_parser { } } + // Get an uint64_t integer converted from the value associated with `key`. + // @key A configuration key. + // + // Get an uint64_t integer converted from the value associated with `key`. + // Raise or . + // + // @return An uint64_t integer converted from the associated value. + std::uint64_t get_as_uint64(const std::string& key) const { + try { + return std::stoull(get(key)); + } catch(const std::invalid_argument& e) { + throw illegal_value(key); + } catch(const std::out_of_range& e) { + throw illegal_value(key); + } + } + // Get a boolean converted from the value associated with `key`. // @key A configuration key. // diff --git a/docs/usage.md b/docs/usage.md index 70ba66e..afec15f 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -51,6 +51,8 @@ These options are to configure memcache protocol: Objects larger than this will be stored in temporary files. * `repl_buffer_size` (Default: 30) The replication buffer size. Unit is MiB. +* `initial_repl_sleep_delay` (Default: 0) + Slow down the scan of the entire hash by the GC thread to prevent errors with the message "Replication buffer is full." during the initial replication. The GC thread sleeps for the time specified here for each scan of the hash bucket. Unit is microseconds. * `secure_erase` (Default: false) If `true`, object memory will be cleared as soon as the object is removed. * `lock_memory` (Default: false) diff --git a/etc/yrmcds.conf b/etc/yrmcds.conf index 603b83b..5a674e0 100644 --- a/etc/yrmcds.conf +++ b/etc/yrmcds.conf @@ -42,6 +42,12 @@ heap_data_limit = 256K # The value must be an integer > 0. Default is 30 (MiB). repl_buffer_size = 30 +# Slow down the scan of the entire hash by the GC thread to prevent +# errors with the message "Replication buffer is full." during the initial +# replication. The GC thread sleeps for the time specified here for each +# scan of the hash bucket. Unit is microseconds. +initial_repl_sleep_delay = 0 + # Clear memory used by deleted or expired objects securely. # This ensures confidential data such as crypto keys will not be # leaked after expiration. diff --git a/src/config.cpp b/src/config.cpp index a873395..445a472 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -24,6 +24,7 @@ const char MAX_DATA_SIZE[] = "max_data_size"; const char HEAP_DATA_LIMIT[] = "heap_data_limit"; const char MEMORY_LIMIT[] = "memory_limit"; const char REPL_BUFSIZE[] = "repl_buffer_size"; +const char INITIAL_REPL_SLEEP_DELAY[] = "initial_repl_sleep_delay"; const char SECURE_ERASE[] = "secure_erase"; const char LOCK_MEMORY[] = "lock_memory"; const char WORKERS[] = "workers"; @@ -206,6 +207,11 @@ void config::load(const std::string& path) { m_repl_bufsize = bufs; } + if( cp.exists(INITIAL_REPL_SLEEP_DELAY) ) { + std::uint64_t n = cp.get_as_uint64(INITIAL_REPL_SLEEP_DELAY); + m_initial_repl_sleep_delay = n; + } + if( cp.exists(SECURE_ERASE) ) { m_secure_erase = cp.get_as_bool(SECURE_ERASE); } diff --git a/src/config.hpp b/src/config.hpp index 2112b47..9977f00 100644 --- a/src/config.hpp +++ b/src/config.hpp @@ -111,6 +111,9 @@ class config { unsigned int repl_bufsize() const noexcept { return m_repl_bufsize; } + std::uint64_t initial_repl_sleep_delay() const noexcept { + return m_initial_repl_sleep_delay; + } bool secure_erase() const noexcept { return m_secure_erase; } @@ -152,6 +155,7 @@ class config { std::size_t m_heap_data_limit = DEFAULT_HEAP_DATA_LIMIT; std::size_t m_memory_limit = DEFAULT_MEMORY_LIMIT; unsigned int m_repl_bufsize = DEFAULT_REPL_BUFSIZE; + uint64_t m_initial_repl_sleep_delay = DEFAULT_INITIAL_REPL_SLEEP_DELAY; bool m_secure_erase = false; bool m_lock_memory = false; unsigned int m_workers = DEFAULT_WORKER_THREADS; diff --git a/src/constants.hpp b/src/constants.hpp index 289e248..3da4a3b 100644 --- a/src/constants.hpp +++ b/src/constants.hpp @@ -17,6 +17,7 @@ const std::size_t DEFAULT_MAX_DATA_SIZE = static_cast(1) << 20; const std::size_t DEFAULT_HEAP_DATA_LIMIT= 256 << 10; const std::size_t DEFAULT_MEMORY_LIMIT = static_cast(1) << 30; const unsigned int DEFAULT_REPL_BUFSIZE = 30; +const std::uint64_t DEFAULT_INITIAL_REPL_SLEEP_DELAY = 0; const int DEFAULT_WORKER_THREADS = 8; const unsigned int DEFAULT_GC_INTERVAL = 10; const unsigned int DEFAULT_SLAVE_TIMEOUT = 10; diff --git a/src/memcache/gc.cpp b/src/memcache/gc.cpp index 1685d98..7d69095 100644 --- a/src/memcache/gc.cpp +++ b/src/memcache/gc.cpp @@ -122,10 +122,25 @@ void gc_thread::gc() { return false; }; + // Putting the thread to sleep tens of microseconds with each loop is desired, but due to the precision of the timer, + // it's not appropriate to do sleep with each loop. The `initial_repl_sleep_delay` is accumulated until it + // exceeds 10000 microseconds (10 milliseconds), and then the thread is put to sleep all at once when this limit is exceeded. + static const std::uint64_t SLEEP_THRESHOLD = 10000; + std::uint64_t sleep_sum = 0; + for( auto it = m_hash.begin(); it != m_hash.end(); ++it ) { m_objects_in_bucket = 0; it->gc(pred); m_flushers.clear(); + + if( ! m_new_slaves.empty() ) { + sleep_sum += g_config.initial_repl_sleep_delay(); + if( sleep_sum >= SLEEP_THRESHOLD ) { + std::this_thread::sleep_for( + std::chrono::microseconds(sleep_sum)); + sleep_sum = 0; + } + } } if( flush ) diff --git a/test/config.cpp b/test/config.cpp index dfb9685..f4c7488 100644 --- a/test/config.cpp +++ b/test/config.cpp @@ -15,6 +15,7 @@ AUTOTEST(config) { cybozu_assert(g_config.group() == "nogroup"); cybozu_assert(g_config.memory_limit() == (1024 << 20)); cybozu_assert(g_config.repl_bufsize() == 100); + cybozu_assert(g_config.initial_repl_sleep_delay() == 40); cybozu_assert(g_config.secure_erase() == true); cybozu_assert(g_config.lock_memory() == true); cybozu_assert(g_config.threshold() == cybozu::severity::warning); diff --git a/test/test.conf b/test/test.conf index c3e10da..c9361c1 100644 --- a/test/test.conf +++ b/test/test.conf @@ -15,6 +15,7 @@ max_data_size = 5M heap_data_limit = 16K memory_limit = 1024M repl_buffer_size= 100 +initial_repl_sleep_delay = 40 secure_erase = true lock_memory = true workers = 10 From 36f6e2d175ee2deeae8f5e34f002ff3808bce98f Mon Sep 17 00:00:00 2001 From: sho-iizuka Date: Mon, 29 Jan 2024 08:22:54 +0000 Subject: [PATCH 2/4] 'static' does not make sense --- src/memcache/gc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/memcache/gc.cpp b/src/memcache/gc.cpp index 7d69095..7a6e49f 100644 --- a/src/memcache/gc.cpp +++ b/src/memcache/gc.cpp @@ -125,7 +125,7 @@ void gc_thread::gc() { // Putting the thread to sleep tens of microseconds with each loop is desired, but due to the precision of the timer, // it's not appropriate to do sleep with each loop. The `initial_repl_sleep_delay` is accumulated until it // exceeds 10000 microseconds (10 milliseconds), and then the thread is put to sleep all at once when this limit is exceeded. - static const std::uint64_t SLEEP_THRESHOLD = 10000; + const std::uint64_t SLEEP_THRESHOLD = 10000; std::uint64_t sleep_sum = 0; for( auto it = m_hash.begin(); it != m_hash.end(); ++it ) { From 47776d4a354f59163ff77ad8ab4ad2f4386694f8 Mon Sep 17 00:00:00 2001 From: sho-iizuka Date: Mon, 29 Jan 2024 09:06:24 +0000 Subject: [PATCH 3/4] Rename 'initial_repl_sleep_delay' to 'initial_repl_sleep_delay_usec' --- docs/usage.md | 2 +- etc/yrmcds.conf | 2 +- src/config.cpp | 8 ++++---- src/config.hpp | 6 +++--- src/constants.hpp | 2 +- src/memcache/gc.cpp | 4 ++-- test/config.cpp | 2 +- test/test.conf | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index afec15f..59fea10 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -51,7 +51,7 @@ These options are to configure memcache protocol: Objects larger than this will be stored in temporary files. * `repl_buffer_size` (Default: 30) The replication buffer size. Unit is MiB. -* `initial_repl_sleep_delay` (Default: 0) +* `initial_repl_sleep_delay_usec` (Default: 0) Slow down the scan of the entire hash by the GC thread to prevent errors with the message "Replication buffer is full." during the initial replication. The GC thread sleeps for the time specified here for each scan of the hash bucket. Unit is microseconds. * `secure_erase` (Default: false) If `true`, object memory will be cleared as soon as the object is removed. diff --git a/etc/yrmcds.conf b/etc/yrmcds.conf index 5a674e0..dea5f8b 100644 --- a/etc/yrmcds.conf +++ b/etc/yrmcds.conf @@ -46,7 +46,7 @@ repl_buffer_size = 30 # errors with the message "Replication buffer is full." during the initial # replication. The GC thread sleeps for the time specified here for each # scan of the hash bucket. Unit is microseconds. -initial_repl_sleep_delay = 0 +initial_repl_sleep_delay_usec = 0 # Clear memory used by deleted or expired objects securely. # This ensures confidential data such as crypto keys will not be diff --git a/src/config.cpp b/src/config.cpp index 445a472..2b4c12f 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -24,7 +24,7 @@ const char MAX_DATA_SIZE[] = "max_data_size"; const char HEAP_DATA_LIMIT[] = "heap_data_limit"; const char MEMORY_LIMIT[] = "memory_limit"; const char REPL_BUFSIZE[] = "repl_buffer_size"; -const char INITIAL_REPL_SLEEP_DELAY[] = "initial_repl_sleep_delay"; +const char INITIAL_REPL_SLEEP_DELAY_USEC[] = "initial_repl_sleep_delay_usec"; const char SECURE_ERASE[] = "secure_erase"; const char LOCK_MEMORY[] = "lock_memory"; const char WORKERS[] = "workers"; @@ -207,9 +207,9 @@ void config::load(const std::string& path) { m_repl_bufsize = bufs; } - if( cp.exists(INITIAL_REPL_SLEEP_DELAY) ) { - std::uint64_t n = cp.get_as_uint64(INITIAL_REPL_SLEEP_DELAY); - m_initial_repl_sleep_delay = n; + if( cp.exists(INITIAL_REPL_SLEEP_DELAY_USEC) ) { + std::uint64_t n = cp.get_as_uint64(INITIAL_REPL_SLEEP_DELAY_USEC); + m_initial_repl_sleep_delay_usec = n; } if( cp.exists(SECURE_ERASE) ) { diff --git a/src/config.hpp b/src/config.hpp index 9977f00..e79fc17 100644 --- a/src/config.hpp +++ b/src/config.hpp @@ -111,8 +111,8 @@ class config { unsigned int repl_bufsize() const noexcept { return m_repl_bufsize; } - std::uint64_t initial_repl_sleep_delay() const noexcept { - return m_initial_repl_sleep_delay; + std::uint64_t initial_repl_sleep_delay_usec() const noexcept { + return m_initial_repl_sleep_delay_usec; } bool secure_erase() const noexcept { return m_secure_erase; @@ -155,7 +155,7 @@ class config { std::size_t m_heap_data_limit = DEFAULT_HEAP_DATA_LIMIT; std::size_t m_memory_limit = DEFAULT_MEMORY_LIMIT; unsigned int m_repl_bufsize = DEFAULT_REPL_BUFSIZE; - uint64_t m_initial_repl_sleep_delay = DEFAULT_INITIAL_REPL_SLEEP_DELAY; + uint64_t m_initial_repl_sleep_delay_usec = DEFAULT_INITIAL_REPL_SLEEP_DELAY_USEC; bool m_secure_erase = false; bool m_lock_memory = false; unsigned int m_workers = DEFAULT_WORKER_THREADS; diff --git a/src/constants.hpp b/src/constants.hpp index 3da4a3b..5f4d7bf 100644 --- a/src/constants.hpp +++ b/src/constants.hpp @@ -17,7 +17,7 @@ const std::size_t DEFAULT_MAX_DATA_SIZE = static_cast(1) << 20; const std::size_t DEFAULT_HEAP_DATA_LIMIT= 256 << 10; const std::size_t DEFAULT_MEMORY_LIMIT = static_cast(1) << 30; const unsigned int DEFAULT_REPL_BUFSIZE = 30; -const std::uint64_t DEFAULT_INITIAL_REPL_SLEEP_DELAY = 0; +const std::uint64_t DEFAULT_INITIAL_REPL_SLEEP_DELAY_USEC = 0; const int DEFAULT_WORKER_THREADS = 8; const unsigned int DEFAULT_GC_INTERVAL = 10; const unsigned int DEFAULT_SLAVE_TIMEOUT = 10; diff --git a/src/memcache/gc.cpp b/src/memcache/gc.cpp index 7a6e49f..42f715f 100644 --- a/src/memcache/gc.cpp +++ b/src/memcache/gc.cpp @@ -123,7 +123,7 @@ void gc_thread::gc() { }; // Putting the thread to sleep tens of microseconds with each loop is desired, but due to the precision of the timer, - // it's not appropriate to do sleep with each loop. The `initial_repl_sleep_delay` is accumulated until it + // it's not appropriate to sleep with each loop. The `initial_repl_sleep_delay_usec` is accumulated until it // exceeds 10000 microseconds (10 milliseconds), and then the thread is put to sleep all at once when this limit is exceeded. const std::uint64_t SLEEP_THRESHOLD = 10000; std::uint64_t sleep_sum = 0; @@ -134,7 +134,7 @@ void gc_thread::gc() { m_flushers.clear(); if( ! m_new_slaves.empty() ) { - sleep_sum += g_config.initial_repl_sleep_delay(); + sleep_sum += g_config.initial_repl_sleep_delay_usec(); if( sleep_sum >= SLEEP_THRESHOLD ) { std::this_thread::sleep_for( std::chrono::microseconds(sleep_sum)); diff --git a/test/config.cpp b/test/config.cpp index f4c7488..5508b3d 100644 --- a/test/config.cpp +++ b/test/config.cpp @@ -15,7 +15,7 @@ AUTOTEST(config) { cybozu_assert(g_config.group() == "nogroup"); cybozu_assert(g_config.memory_limit() == (1024 << 20)); cybozu_assert(g_config.repl_bufsize() == 100); - cybozu_assert(g_config.initial_repl_sleep_delay() == 40); + cybozu_assert(g_config.initial_repl_sleep_delay_usec() == 40); cybozu_assert(g_config.secure_erase() == true); cybozu_assert(g_config.lock_memory() == true); cybozu_assert(g_config.threshold() == cybozu::severity::warning); diff --git a/test/test.conf b/test/test.conf index c9361c1..db44221 100644 --- a/test/test.conf +++ b/test/test.conf @@ -15,7 +15,7 @@ max_data_size = 5M heap_data_limit = 16K memory_limit = 1024M repl_buffer_size= 100 -initial_repl_sleep_delay = 40 +initial_repl_sleep_delay_usec = 40 secure_erase = true lock_memory = true workers = 10 From 8e5b2586a8ea6fb368d88ef3965eec621e6c2104 Mon Sep 17 00:00:00 2001 From: sho-iizuka Date: Mon, 29 Jan 2024 09:13:57 +0000 Subject: [PATCH 4/4] Use constexpr for constants --- src/memcache/gc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/memcache/gc.cpp b/src/memcache/gc.cpp index 42f715f..31fcbc3 100644 --- a/src/memcache/gc.cpp +++ b/src/memcache/gc.cpp @@ -125,7 +125,7 @@ void gc_thread::gc() { // Putting the thread to sleep tens of microseconds with each loop is desired, but due to the precision of the timer, // it's not appropriate to sleep with each loop. The `initial_repl_sleep_delay_usec` is accumulated until it // exceeds 10000 microseconds (10 milliseconds), and then the thread is put to sleep all at once when this limit is exceeded. - const std::uint64_t SLEEP_THRESHOLD = 10000; + constexpr std::uint64_t SLEEP_THRESHOLD = 10000; std::uint64_t sleep_sum = 0; for( auto it = m_hash.begin(); it != m_hash.end(); ++it ) {