From f0e2ab702ad7d0364f33d3885cf27f31c83332fc Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 12 Jul 2021 15:53:36 +0100 Subject: [PATCH 001/302] Major refactor of snmalloc (#343) # Pagemap The Pagemap now stores all the meta-data for the object allocation. The meta-data in the pagemap is effectively a triple of the sizeclass, the remote allocator, and a pointer to a 64 byte block of meta-data for this chunk of memory. By storing the pointer to a block, it allows the pagemap to handle multiple slab sizes without branching on the fast path. There is one entry in the pagemap per 16KiB of address space, but by using the same entry in the pagemap for 4 adjacent entries, then we can treat a 64KiB range can be treated as a single slab of allocations. This change also means there is almost no capability amplification required by the implementation on CHERI for finding meta-data. The only amplification is required, when we change the way a chunk is used to a size of object allocation. # Backend There is a second major aspect of the refactor that there is now a narrow API that abstracts the Pagemap, PAL and address space management. This should better enable the compartmentalisation and makes it easier to produce alternative backends for various research directions. This is a template parameter that can be used to specialised by the front-end in different ways. # Thread local state The thread local state has been refactored into two components, one (called 'localalloc') that is stored directly in the TLS and is constant initialised, and one that is allocated in the address space (called 'coreallloc') which is lazily created and pooled. # Difference This removes Superslabs/Medium slabs as there meta-data is now part of the pagemap. --- CMakeLists.txt | 105 +- azure-pipelines.yml | 19 +- ci/scripts/build.sh | 2 + ci/scripts/test.sh | 8 +- docs/BUILDING.md | 2 - src/aal/aal.h | 4 +- src/backend/address_space.h | 207 +++ .../address_space_core.h} | 190 +- src/backend/backend.h | 206 +++ src/backend/pagemap.h | 179 ++ src/ds/aba.h | 54 +- src/ds/address.h | 21 +- src/ds/bits.h | 19 +- src/ds/cdllist.h | 154 +- src/ds/defines.h | 39 +- src/ds/dllist.h | 1 + src/ds/helpers.h | 14 +- src/ds/invalidptr.h | 2 + src/ds/mpmcstack.h | 35 +- src/ds/mpscq.h | 11 +- src/ds/ptrwrap.h | 45 +- src/mem/alloc.h | 1563 ----------------- src/mem/allocconfig.h | 81 +- src/mem/allocslab.h | 20 - src/mem/allocstats.h | 31 +- src/mem/arenamap.h | 130 -- src/mem/baseslab.h | 32 - src/mem/chunkmap.h | 195 -- src/mem/commonconfig.h | 41 + src/mem/corealloc.h | 681 +++++++ src/mem/entropy.h | 14 +- src/mem/fixedglobalconfig.h | 78 + src/mem/freelist.h | 223 +-- src/mem/globalalloc.h | 273 ++- src/mem/globalconfig.h | 107 ++ src/mem/largealloc.h | 448 ----- src/mem/localalloc.h | 535 ++++++ src/mem/localcache.h | 112 ++ src/mem/mediumslab.h | 156 -- src/mem/metaslab.h | 234 +-- src/mem/pagemap.h | 521 ------ src/mem/pool.h | 72 +- src/mem/pooled.h | 10 +- src/mem/ptrhelpers.h | 6 +- src/mem/remoteallocator.h | 220 +-- src/mem/remotecache.h | 199 +++ src/mem/{slowalloc.h => scopedalloc.h} | 38 +- src/mem/sizeclass.h | 89 - src/mem/sizeclasstable.h | 277 ++- src/mem/slab.h | 197 --- src/mem/slaballocator.h | 163 ++ src/mem/superslab.h | 272 --- src/mem/threadalloc.h | 326 +--- src/override/malloc-extensions.cc | 9 +- src/override/malloc.cc | 150 +- src/override/new.cc | 24 +- src/override/rust.cc | 11 +- src/pal/pal.h | 11 +- src/pal/pal_noalloc.h | 3 + src/pal/pal_posix.h | 11 +- src/snmalloc.h | 12 +- src/snmalloc_core.h | 4 + src/snmalloc_front.h | 2 + .../func/external_pagemap/external_pagemap.cc | 3 +- .../func/first_operation/first_operation.cc | 152 +- src/test/func/fixed_region/fixed_region.cc | 70 +- src/test/func/malloc/malloc.cc | 71 +- src/test/func/memory/memory.cc | 214 +-- src/test/func/memory_usage/memory_usage.cc | 11 +- src/test/func/pagemap/pagemap.cc | 123 +- src/test/func/release-rounding/rounding.cc | 5 +- src/test/func/sandbox/sandbox.cc | 36 +- src/test/func/sizeclass/sizeclass.cc | 13 +- src/test/func/statistics/stats.cc | 18 +- src/test/func/teardown/teardown.cc | 209 +++ .../thread_alloc_external.cc | 31 +- src/test/func/two_alloc_types/alloc1.cc | 25 +- src/test/func/two_alloc_types/alloc2.cc | 6 +- src/test/func/two_alloc_types/main.cc | 29 +- src/test/perf/contention/contention.cc | 14 +- .../perf/external_pointer/externalpointer.cc | 16 +- src/test/perf/low_memory/low-memory.cc | 113 +- src/test/perf/singlethread/singlethread.cc | 12 +- 83 files changed, 4352 insertions(+), 5717 deletions(-) create mode 100644 src/backend/address_space.h rename src/{mem/address_space.h => backend/address_space_core.h} (52%) create mode 100644 src/backend/backend.h create mode 100644 src/backend/pagemap.h delete mode 100644 src/mem/alloc.h delete mode 100644 src/mem/allocslab.h delete mode 100644 src/mem/arenamap.h delete mode 100644 src/mem/baseslab.h delete mode 100644 src/mem/chunkmap.h create mode 100644 src/mem/commonconfig.h create mode 100644 src/mem/corealloc.h create mode 100644 src/mem/fixedglobalconfig.h create mode 100644 src/mem/globalconfig.h delete mode 100644 src/mem/largealloc.h create mode 100644 src/mem/localalloc.h create mode 100644 src/mem/localcache.h delete mode 100644 src/mem/mediumslab.h delete mode 100644 src/mem/pagemap.h create mode 100644 src/mem/remotecache.h rename src/mem/{slowalloc.h => scopedalloc.h} (67%) delete mode 100644 src/mem/sizeclass.h delete mode 100644 src/mem/slab.h create mode 100644 src/mem/slaballocator.h delete mode 100644 src/mem/superslab.h create mode 100644 src/snmalloc_core.h create mode 100644 src/snmalloc_front.h create mode 100644 src/test/func/teardown/teardown.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 34f6e53c7..878e5cbb6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,7 +18,7 @@ option(SNMALLOC_STATIC_LIBRARY "Build static libraries" ON) option(SNMALLOC_QEMU_WORKAROUND "Disable using madvise(DONT_NEED) to zero memory on Linux" Off) option(SNMALLOC_OPTIMISE_FOR_CURRENT_MACHINE "Compile for current machine architecture" Off) set(SNMALLOC_STATIC_LIBRARY_PREFIX "sn_" CACHE STRING "Static library function prefix") -option(SNMALLOC_USE_CXX20 "Build as C++20, not C++17; experimental as yet" OFF) +option(SNMALLOC_USE_CXX17 "Build as C++17 for legacy support." OFF) # malloc.h will error if you include it on FreeBSD, so this test must not # unconditionally include it. @@ -66,6 +66,7 @@ macro(warnings_high) else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4") endif() + # /Wv18 is required for the annotation to force inline a lambda. add_compile_options(/WX /wd4127 /wd4324 /wd4201) else() if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") @@ -75,10 +76,6 @@ macro(warnings_high) endif() endmacro() -macro(oe_simulate target) - target_compile_definitions(${target} PRIVATE SNMALLOC_USE_SMALL_CHUNKS) -endmacro() - macro(clangformat_targets) # The clang-format tool is installed under a variety of different names. Try # to find a sensible one. Only look for versions 9 explicitly - we don't @@ -94,7 +91,7 @@ macro(clangformat_targets) message(WARNING "Not generating clangformat target, no clang-format tool found") else () message(STATUS "Generating clangformat target using ${CLANG_FORMAT}") - file(GLOB_RECURSE ALL_SOURCE_FILES src/*.cc src/*.h src/*.hh) + file(GLOB_RECURSE ALL_SOURCE_FILES CONFIGURE_DEPENDS src/*.cc src/*.h src/*.hh) # clangformat does not yet understand concepts well; for the moment, don't # ask it to format them. See https://reviews.llvm.org/D79773 list(FILTER ALL_SOURCE_FILES EXCLUDE REGEX "src/[^/]*/[^/]*_concept\.h$") @@ -106,6 +103,13 @@ macro(clangformat_targets) endif() endmacro() +# Have to set this globally, as can't be set on an interface target. +if(SNMALLOC_USE_CXX17) + set(CMAKE_CXX_STANDARD 17) +else() + set(CMAKE_CXX_STANDARD 20) +endif() + # The main target for snmalloc add_library(snmalloc_lib INTERFACE) target_include_directories(snmalloc_lib INTERFACE src/) @@ -153,13 +157,6 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") endif() endif () -# Have to set this globally, as can't be set on an interface target. -if(SNMALLOC_USE_CXX20) - set(CMAKE_CXX_STANDARD 20) -else() - set(CMAKE_CXX_STANDARD 17) -endif() - if(USE_SNMALLOC_STATS) target_compile_definitions(snmalloc_lib INTERFACE -DUSE_SNMALLOC_STATS) endif() @@ -194,15 +191,18 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) if(MSVC) set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Zi") set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /DEBUG") + target_compile_definitions(snmalloc_lib INTERFACE -D_HAS_EXCEPTIONS=0) + add_compile_options(/std:c++latest) else() - add_compile_options(-fno-exceptions -fno-rtti -g -fomit-frame-pointer) + add_compile_options(-fno-exceptions -fno-rtti -fomit-frame-pointer -ffunction-sections) # Static TLS model is unsupported on Haiku. # All symbols are always dynamic on haiku and -rdynamic is redundant (and unsupported). if (NOT CMAKE_SYSTEM_NAME MATCHES "Haiku") add_compile_options(-ftls-model=initial-exec) if(SNMALLOC_CI_BUILD OR (${CMAKE_BUILD_TYPE} MATCHES "Debug")) - # Get better stack traces in CI and Debug. - target_link_libraries(snmalloc_lib INTERFACE "-rdynamic") + # Get better stack traces in CI and Debug. + target_link_libraries(snmalloc_lib INTERFACE "-rdynamic") + add_compile_options(-g) endif() endif() @@ -225,7 +225,7 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) endif() macro(subdirlist result curdir) - file(GLOB children LIST_DIRECTORIES true RELATIVE ${curdir} ${curdir}/*) + file(GLOB children CONFIGURE_DEPENDS LIST_DIRECTORIES true RELATIVE ${curdir} ${curdir}/* ) set(dirlist "") foreach(child ${children}) if(IS_DIRECTORY ${curdir}/${child}) @@ -235,6 +235,13 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) set(${result} ${dirlist}) endmacro() + if(CMAKE_VERSION VERSION_LESS 3.14) + set(CMAKE_REQUIRED_LIBRARIES -fuse-ld=lld) + else() + set(CMAKE_REQUIRED_LINK_OPTIONS -fuse-ld=lld) + endif() + check_cxx_source_compiles("int main() { return 1; }" LLD_WORKS) + macro(add_shim name type) add_library(${name} ${type} ${ARGN}) target_link_libraries(${name} snmalloc_lib) @@ -243,59 +250,40 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) endif() set_target_properties(${name} PROPERTIES CXX_VISIBILITY_PRESET hidden) - if(EXPOSE_EXTERNAL_PAGEMAP) - if(MSVC) - target_compile_definitions(${name} PRIVATE /DSNMALLOC_EXPOSE_PAGEMAP) - else() - target_compile_definitions(${name} PRIVATE -DSNMALLOC_EXPOSE_PAGEMAP) - endif() - endif() - - if(EXPOSE_EXTERNAL_RESERVE) - if(MSVC) - target_compile_definitions(${name} PRIVATE /DSNMALLOC_EXPOSE_RESERVE) - else() - target_compile_definitions(${name} PRIVATE -DSNMALLOC_EXPOSE_RESERVE) - endif() - endif() - # Ensure that we do not link against C++ stdlib when compiling shims. if(NOT MSVC) set_target_properties(${name} PROPERTIES LINKER_LANGUAGE C) + if (LLD_WORKS) + message("Using LLD.") + + # Only include the dependencies that actually are touched. + # Required so C++ library is not included with direct pthread access. + target_link_libraries(${name} -Wl,--as-needed) + + # Remove all the duplicate new/malloc and free/delete definitions + target_link_libraries(${name} -Wl,--icf=all -fuse-ld=lld) + endif() endif() endmacro() if (SNMALLOC_STATIC_LIBRARY) add_shim(snmallocshim-static STATIC src/override/malloc.cc) - add_shim(snmallocshim-1mib-static STATIC src/override/malloc.cc) - add_shim(snmallocshim-16mib-static STATIC src/override/malloc.cc) - target_compile_definitions(snmallocshim-16mib-static PRIVATE SNMALLOC_USE_LARGE_CHUNKS - SNMALLOC_STATIC_LIBRARY_PREFIX=${SNMALLOC_STATIC_LIBRARY_PREFIX}) target_compile_definitions(snmallocshim-static PRIVATE SNMALLOC_STATIC_LIBRARY_PREFIX=${SNMALLOC_STATIC_LIBRARY_PREFIX}) - target_compile_definitions(snmallocshim-1mib-static PRIVATE - SNMALLOC_STATIC_LIBRARY_PREFIX=${SNMALLOC_STATIC_LIBRARY_PREFIX}) endif () if(NOT WIN32) - set(SHARED_FILES src/override/new.cc src/override/malloc.cc) + set(SHARED_FILES src/override/new.cc) add_shim(snmallocshim SHARED ${SHARED_FILES}) add_shim(snmallocshim-checks SHARED ${SHARED_FILES}) - add_shim(snmallocshim-1mib SHARED ${SHARED_FILES}) - add_shim(snmallocshim-16mib SHARED ${SHARED_FILES}) - target_compile_definitions(snmallocshim-16mib PRIVATE SNMALLOC_USE_LARGE_CHUNKS) target_compile_definitions(snmallocshim-checks PRIVATE CHECK_CLIENT) - # Build a shim with some settings from oe. - add_shim(snmallocshim-oe SHARED ${SHARED_FILES}) - oe_simulate(snmallocshim-oe) endif() if(SNMALLOC_RUST_SUPPORT) add_shim(snmallocshim-rust STATIC src/override/rust.cc) - add_shim(snmallocshim-1mib-rust STATIC src/override/rust.cc) - add_shim(snmallocshim-16mib-rust STATIC src/override/rust.cc) - target_compile_definitions(snmallocshim-16mib-rust PRIVATE SNMALLOC_USE_LARGE_CHUNKS) + add_shim(snmallocshim-checks-rust STATIC src/override/rust.cc) + target_compile_definitions(snmallocshim-checks-rust PRIVATE CHECK_CLIENT) endif() enable_testing() @@ -314,9 +302,9 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) # Windows does not support aligned allocation well enough # for pass through. # NetBSD, OpenBSD and DragonFlyBSD do not support malloc*size calls. - set(FLAVOURS 1;16;oe;check) + set(FLAVOURS fast;check) else() - set(FLAVOURS 1;16;oe;malloc;check) + set(FLAVOURS fast;check) #malloc - TODO-need to add pass through back endif() foreach(FLAVOUR ${FLAVOURS}) unset(SRC) @@ -328,21 +316,12 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) # For all tests enable commit checking. target_compile_definitions(${TESTNAME} PRIVATE -DUSE_POSIX_COMMIT_CHECKS) - if (${FLAVOUR} EQUAL 16) - target_compile_definitions(${TESTNAME} PRIVATE SNMALLOC_USE_LARGE_CHUNKS) - endif() - if (${FLAVOUR} STREQUAL "oe") - oe_simulate(${TESTNAME}) - endif() if (${FLAVOUR} STREQUAL "malloc") target_compile_definitions(${TESTNAME} PRIVATE SNMALLOC_PASS_THROUGH) endif() if (${FLAVOUR} STREQUAL "check") target_compile_definitions(${TESTNAME} PRIVATE CHECK_CLIENT) endif() - if(CONST_QUALIFIED_MALLOC_USABLE_SIZE) - target_compile_definitions(${TESTNAME} PRIVATE -DMALLOC_USABLE_SIZE_QUALIFIER=const) - endif() target_link_libraries(${TESTNAME} snmalloc_lib) if (${TEST} MATCHES "release-.*") message(STATUS "Adding test: ${TESTNAME} only for release configs") @@ -371,9 +350,9 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) set_tests_properties(${TESTNAME} PROPERTIES PROCESSORS 4) endif() endif() - if (${TEST_CATEGORY} MATCHES "func") - target_compile_definitions(${TESTNAME} PRIVATE -DUSE_SNMALLOC_STATS) - endif () + # if (${TEST_CATEGORY} MATCHES "func") + # target_compile_definitions(${TESTNAME} PRIVATE -DUSE_SNMALLOC_STATS) + # endif () endforeach() endforeach() endforeach() diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6674f310c..9680a0142 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -3,6 +3,7 @@ trigger: pr: - master +- snmalloc2 jobs: - job: @@ -52,12 +53,12 @@ jobs: CMakeArgs: '' Image: snmallocciteam/build_linux_x64:latest - 64-bit Clang-10 Debug C++20: + 64-bit Clang-10 Debug C++17: CC: clang-10 CXX: clang++-10 BuildType: Debug SelfHost: false - CMakeArgs: '-DSNMALLOC_USE_CXX20=On' + CMakeArgs: '-DSNMALLOC_USE_CXX17=On' Image: snmallocciteam/build_linux_x64:latest 64-bit Clang-9 Release: @@ -97,7 +98,7 @@ jobs: CXX: clang++-9 BuildType: Debug SelfHost: false - CMakeArgs: '' + CMakeArgs: '-DSNMALLOC_USE_CXX17=On' Image: snmallocciteam/build_linux_x86:latest 32-bit Clang-9 Release: @@ -105,7 +106,7 @@ jobs: CXX: clang++-9 BuildType: Release SelfHost: false - CMakeArgs: '' + CMakeArgs: '-DSNMALLOC_USE_CXX17=On' Image: snmallocciteam/build_linux_x86:latest container: $[ variables['Image'] ] @@ -140,7 +141,7 @@ jobs: CXX: clang++-9 BuildType: Debug SelfHost: false - CMakeArgs: '-DSNMALLOC_QEMU_WORKAROUND=On' + CMakeArgs: '-DSNMALLOC_QEMU_WORKAROUND=On -DSNMALLOC_USE_CXX17=On' Image: snmallocciteam/build_linux_arm64:latest 64-bit Clang-9 Release: @@ -148,7 +149,7 @@ jobs: CXX: clang++-9 BuildType: Release SelfHost: false - CMakeArgs: '-DSNMALLOC_QEMU_WORKAROUND=On' + CMakeArgs: '-DSNMALLOC_QEMU_WORKAROUND=On -DSNMALLOC_USE_CXX17=On' Image: snmallocciteam/build_linux_arm64:latest 32-bit Clang-9 Debug: @@ -156,7 +157,7 @@ jobs: CXX: clang++-9 BuildType: Debug SelfHost: false - CMakeArgs: '-DSNMALLOC_QEMU_WORKAROUND=On' + CMakeArgs: '-DSNMALLOC_QEMU_WORKAROUND=On -DSNMALLOC_USE_CXX17=On' Image: snmallocciteam/build_linux_armhf:latest 32-bit Clang-9 Release: @@ -164,7 +165,7 @@ jobs: CXX: clang++-9 BuildType: Release SelfHost: false - CMakeArgs: '-DSNMALLOC_QEMU_WORKAROUND=On' + CMakeArgs: '-DSNMALLOC_QEMU_WORKAROUND=On -DSNMALLOC_USE_CXX17=On' Image: snmallocciteam/build_linux_armhf:latest steps: @@ -177,7 +178,7 @@ jobs: -e CC=$(CC) \ -e CXX=$(CXX) \ -e BUILD_TYPE=$(BuildType) \ - -e CMAKE_ARGS=$(CMakeArgs) \ + -e CMAKE_ARGS="$(CMakeArgs)" \ $(Image) \ ci/scripts/build.sh displayName: 'Build' diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index 7d4941706..b8ee7f71f 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -eo pipefail + mkdir build cd build diff --git a/ci/scripts/test.sh b/ci/scripts/test.sh index 02e2e6534..ac439158c 100755 --- a/ci/scripts/test.sh +++ b/ci/scripts/test.sh @@ -1,14 +1,14 @@ #!/bin/bash +set -eo pipefail + cd build if [ $SELF_HOST = false ]; then ctest -j 4 --output-on-failure -C $BUILD_TYPE else - sudo cp libsnmallocshim.so libsnmallocshim-16mib.so libsnmallocshim-oe.so /usr/local/lib/ + sudo cp libsnmallocshim.so libsnmallocshim-checks.so /usr/local/lib/ ninja clean LD_PRELOAD=/usr/local/lib/libsnmallocshim.so ninja ninja clean - LD_PRELOAD=/usr/local/lib/libsnmallocshim-16mib.so ninja - ninja clean - LD_PRELOAD=/usr/local/lib/libsnmallocshim-oe.so ninja + LD_PRELOAD=/usr/local/lib/libsnmallocshim-checks.so ninja fi \ No newline at end of file diff --git a/docs/BUILDING.md b/docs/BUILDING.md index 0a550ffc6..bad4b57a7 100644 --- a/docs/BUILDING.md +++ b/docs/BUILDING.md @@ -110,8 +110,6 @@ target_link_libraries([lib_name] PRIVATE snmalloc_lib) You will also need to compile the relevant parts of snmalloc itself. Create a new file with the following contents and compile it with the rest of your application. ```c++ -#define NO_BOOTSTRAP_ALLOCATOR - #include "snmalloc/src/override/malloc.cc" #include "snmalloc/src/override/new.cc" ``` diff --git a/src/aal/aal.h b/src/aal/aal.h index d1a71c830..3d35b6676 100644 --- a/src/aal/aal.h +++ b/src/aal/aal.h @@ -142,7 +142,7 @@ namespace snmalloc static_assert(capptr_is_bounds_refinement()); UNUSED(size); - return CapPtr(a.template as_static().unsafe_capptr); + return CapPtr(a.template as_static().unsafe_ptr()); } /** @@ -154,7 +154,7 @@ namespace snmalloc capptr_rebound(CapPtr a, CapPtr r) noexcept { UNUSED(a); - return CapPtr(r.unsafe_capptr); + return CapPtr(r.unsafe_ptr()); } }; } // namespace snmalloc diff --git a/src/backend/address_space.h b/src/backend/address_space.h new file mode 100644 index 000000000..77f45cbdb --- /dev/null +++ b/src/backend/address_space.h @@ -0,0 +1,207 @@ +#pragma once +#include "../ds/address.h" +#include "../ds/flaglock.h" +#include "../pal/pal.h" +#include "address_space_core.h" + +#include +#ifdef SNMALLOC_TRACING +# include +#endif + +namespace snmalloc +{ + /** + * Implements a power of two allocator, where all blocks are aligned to the + * same power of two as their size. This is what snmalloc uses to get + * alignment of very large sizeclasses. + * + * It cannot unreserve memory, so this does not require the + * usual complexity of a buddy allocator. + */ + template + class AddressSpaceManager + { + AddressSpaceManagerCore core; + + /** + * This is infrequently used code, a spin lock simplifies the code + * considerably, and should never be on the fast path. + */ + std::atomic_flag spin_lock = ATOMIC_FLAG_INIT; + + public: + /** + * Returns a pointer to a block of memory of the supplied size. + * The block will be committed, if specified by the template parameter. + * The returned block is guaranteed to be aligened to the size. + * + * Only request 2^n sizes, and not less than a pointer. + * + * On StrictProvenance architectures, any underlying allocations made as + * part of satisfying the request will be registered with the provided + * arena_map for use in subsequent amplification. + */ + template + CapPtr reserve(size_t size) + { +#ifdef SNMALLOC_TRACING + std::cout << "ASM reserve request:" << size << std::endl; +#endif + SNMALLOC_ASSERT(bits::is_pow2(size)); + SNMALLOC_ASSERT(size >= sizeof(void*)); + + if constexpr ((align == false) && !pal_supports) + { + if constexpr (pal_supports) + { + // TODO wasting size here. + size = bits::max(size, PAL::minimum_alloc_size); + return CapPtr( + PAL::template reserve_aligned(size)); + } + else + { + auto [block, size2] = PAL::reserve_at_least(size); + // TODO wasting size here. + UNUSED(size2); +#ifdef SNMALLOC_TRACING + std::cout << "Unaligned alloc here:" << block << " (" << size2 << ")" + << std::endl; +#endif + return CapPtr(block); + } + } + else + { + /* + * For sufficiently large allocations with platforms that support + * aligned allocations and architectures that don't require + * StrictProvenance, try asking the platform first. + */ + if constexpr ( + pal_supports && + !aal_supports) + { + if (size >= PAL::minimum_alloc_size) + return CapPtr( + PAL::template reserve_aligned(size)); + } + + CapPtr res; + { + FlagLock lock(spin_lock); + res = core.template reserve(size); + if (res == nullptr) + { + // Allocation failed ask OS for more memory + CapPtr block = nullptr; + size_t block_size = 0; + if constexpr (pal_supports) + { + /* + * We will have handled the case where size >= + * minimum_alloc_size above, so we are left to handle only small + * things here. + */ + block_size = PAL::minimum_alloc_size; + + void* block_raw = + PAL::template reserve_aligned(block_size); + + // It's a bit of a lie to convert without applying bounds, but the + // platform will have bounded block for us and it's better that + // the rest of our internals expect CBChunk bounds. + block = CapPtr(block_raw); + } + else if constexpr (!pal_supports) + { + // Need at least 2 times the space to guarantee alignment. + // Hold lock here as a race could cause additional requests to + // the PAL, and this could lead to suprious OOM. This is + // particularly bad if the PAL gives all the memory on first call. + auto block_and_size = PAL::reserve_at_least(size * 2); + block = CapPtr(block_and_size.first); + block_size = block_and_size.second; + + // Ensure block is pointer aligned. + if ( + pointer_align_up(block, sizeof(void*)) != block || + bits::align_up(block_size, sizeof(void*)) > block_size) + { + auto diff = + pointer_diff(block, pointer_align_up(block, sizeof(void*))); + block_size = block_size - diff; + block_size = bits::align_down(block_size, sizeof(void*)); + } + } + if (block == nullptr) + { + return nullptr; + } + + core.template add_range(block, block_size); + + // still holding lock so guaranteed to succeed. + res = core.template reserve(size); + } + } + + // Don't need lock while committing pages. + if constexpr (committed) + core.template commit_block(res, size); + + return res; + } + } + + /** + * Aligns block to next power of 2 above size, and unused space at the end + * of the block is retained by the address space manager. + * + * This is useful for allowing the space required for alignment to be + * used, by smaller objects. + */ + template + CapPtr reserve_with_left_over(size_t size) + { + SNMALLOC_ASSERT(size >= sizeof(void*)); + + size = bits::align_up(size, sizeof(void*)); + + size_t rsize = bits::next_pow2(size); + + auto res = reserve(rsize); + + if (res != nullptr) + { + if (rsize > size) + { + FlagLock lock(spin_lock); + core.template add_range(pointer_offset(res, size), rsize - size); + } + + if constexpr (committed) + core.commit_block(res, size); + } + return res; + } + + /** + * Default constructor. An address-space manager constructed in this way + * does not own any memory at the start and will request any that it needs + * from the PAL. + */ + AddressSpaceManager() = default; + + /** + * Add a range of memory to the address space. + * Divides blocks into power of two sizes with natural alignment + */ + void add_range(CapPtr base, size_t length) + { + FlagLock lock(spin_lock); + core.add_range(base, length); + } + }; +} // namespace snmalloc diff --git a/src/mem/address_space.h b/src/backend/address_space_core.h similarity index 52% rename from src/mem/address_space.h rename to src/backend/address_space_core.h index 6430ce708..6f0ccc5d6 100644 --- a/src/mem/address_space.h +++ b/src/backend/address_space_core.h @@ -1,12 +1,19 @@ +#pragma once #include "../ds/address.h" #include "../ds/flaglock.h" #include "../pal/pal.h" -#include "arenamap.h" #include +#ifdef SNMALLOC_TRACING +# include +#endif + namespace snmalloc { /** + * TODO all comment in this file need revisiting. Core versus locking global + * version. + * * Implements a power of two allocator, where all blocks are aligned to the * same power of two as their size. This is what snmalloc uses to get * alignment of very large sizeclasses. @@ -14,8 +21,7 @@ namespace snmalloc * It cannot unreserve memory, so this does not require the * usual complexity of a buddy allocator. */ - template - class AddressSpaceManager + class AddressSpaceManagerCore { /** * Stores the blocks of address space @@ -37,12 +43,6 @@ namespace snmalloc */ std::array, 2>, bits::BITS> ranges = {}; - /** - * This is infrequently used code, a spin lock simplifies the code - * considerably, and should never be on the fast path. - */ - std::atomic_flag spin_lock = ATOMIC_FLAG_INIT; - /** * Checks a block satisfies its invariant. */ @@ -59,6 +59,7 @@ namespace snmalloc /** * Adds a block to `ranges`. */ + template void add_block(size_t align_bits, CapPtr base) { check_block(base, align_bits); @@ -72,9 +73,12 @@ namespace snmalloc if (ranges[align_bits][1] != nullptr) { +#ifdef SNMALLOC_TRACING + std::cout << "Add range linking." << std::endl; +#endif // Add to linked list. - commit_block(base, sizeof(void*)); - *(base.template as_static>().unsafe_capptr) = + commit_block(base, sizeof(void*)); + *(base.template as_static>().unsafe_ptr()) = ranges[align_bits][1]; check_block(ranges[align_bits][1], align_bits); } @@ -88,6 +92,7 @@ namespace snmalloc * Find a block of the correct size. May split larger blocks * to satisfy this request. */ + template CapPtr remove_block(size_t align_bits) { CapPtr first = ranges[align_bits][0]; @@ -100,7 +105,7 @@ namespace snmalloc } // Look for larger block and split up recursively - CapPtr bigger = remove_block(align_bits + 1); + CapPtr bigger = remove_block(align_bits + 1); if (bigger != nullptr) { size_t left_over_size = bits::one_at_bit(align_bits); @@ -116,9 +121,9 @@ namespace snmalloc CapPtr second = ranges[align_bits][1]; if (second != nullptr) { - commit_block(second, sizeof(void*)); + commit_block(second, sizeof(void*)); auto psecond = - second.template as_static>().unsafe_capptr; + second.template as_static>().unsafe_ptr(); auto next = *psecond; ranges[align_bits][1] = next; // Zero memory. Client assumes memory contains only zeros. @@ -133,10 +138,12 @@ namespace snmalloc return first; } + public: /** * Add a range of memory to the address space. * Divides blocks into power of two sizes with natural alignment */ + template void add_range(CapPtr base, size_t length) { // Find the minimum set of maximally aligned blocks in this range. @@ -149,7 +156,7 @@ namespace snmalloc size_t align = bits::one_at_bit(align_bits); check_block(base, align_bits); - add_block(align_bits, base); + add_block(align_bits, base); base = pointer_offset(base, align); length -= align; @@ -159,6 +166,7 @@ namespace snmalloc /** * Commit a block of memory */ + template void commit_block(CapPtr base, size_t size) { // Rounding required for sub-page allocations. @@ -166,10 +174,9 @@ namespace snmalloc auto page_end = pointer_align_up(pointer_offset(base, size)); size_t using_size = pointer_diff(page_start, page_end); - PAL::template notify_using(page_start.unsafe_capptr, using_size); + PAL::template notify_using(page_start.unsafe_ptr(), using_size); } - public: /** * Returns a pointer to a block of memory of the supplied size. * The block will be committed, if specified by the template parameter. @@ -181,114 +188,17 @@ namespace snmalloc * part of satisfying the request will be registered with the provided * arena_map for use in subsequent amplification. */ - template - CapPtr reserve(size_t size, ArenaMap& arena_map) + template + CapPtr reserve(size_t size) { +#ifdef SNMALLOC_TRACING + std::cout << "ASM Core reserve request:" << size << std::endl; +#endif + SNMALLOC_ASSERT(bits::is_pow2(size)); SNMALLOC_ASSERT(size >= sizeof(void*)); - /* - * For sufficiently large allocations with platforms that support aligned - * allocations and architectures that don't require StrictProvenance, - * try asking the platform first. - */ - if constexpr ( - pal_supports && !aal_supports) - { - if (size >= PAL::minimum_alloc_size) - return CapPtr( - PAL::template reserve_aligned(size)); - } - - CapPtr res; - { - FlagLock lock(spin_lock); - res = remove_block(bits::next_pow2_bits(size)); - if (res == nullptr) - { - // Allocation failed ask OS for more memory - CapPtr block = nullptr; - size_t block_size = 0; - if constexpr (pal_supports) - { - /* - * aal_supports ends up here, too, and we ensure - * that we always allocate whole ArenaMap granules. - */ - if constexpr (aal_supports) - { - static_assert( - !aal_supports || - (ArenaMap::alloc_size >= PAL::minimum_alloc_size), - "Provenance root granule must be at least PAL's " - "minimum_alloc_size"); - block_size = bits::align_up(size, ArenaMap::alloc_size); - } - else - { - /* - * We will have handled the case where size >= minimum_alloc_size - * above, so we are left to handle only small things here. - */ - block_size = PAL::minimum_alloc_size; - } - - void* block_raw = PAL::template reserve_aligned(block_size); - - // It's a bit of a lie to convert without applying bounds, but the - // platform will have bounded block for us and it's better that the - // rest of our internals expect CBChunk bounds. - block = CapPtr(block_raw); - - if constexpr (aal_supports) - { - auto root_block = CapPtr(block_raw); - auto root_size = block_size; - do - { - arena_map.register_root(root_block); - root_block = pointer_offset(root_block, ArenaMap::alloc_size); - root_size -= ArenaMap::alloc_size; - } while (root_size > 0); - } - } - else if constexpr (!pal_supports) - { - // Need at least 2 times the space to guarantee alignment. - // Hold lock here as a race could cause additional requests to - // the PAL, and this could lead to suprious OOM. This is - // particularly bad if the PAL gives all the memory on first call. - auto block_and_size = PAL::reserve_at_least(size * 2); - block = CapPtr(block_and_size.first); - block_size = block_and_size.second; - - // Ensure block is pointer aligned. - if ( - pointer_align_up(block, sizeof(void*)) != block || - bits::align_up(block_size, sizeof(void*)) > block_size) - { - auto diff = - pointer_diff(block, pointer_align_up(block, sizeof(void*))); - block_size = block_size - diff; - block_size = bits::align_down(block_size, sizeof(void*)); - } - } - if (block == nullptr) - { - return nullptr; - } - add_range(block, block_size); - - // still holding lock so guaranteed to succeed. - res = remove_block(bits::next_pow2_bits(size)); - } - } - - // Don't need lock while committing pages. - if constexpr (committed) - commit_block(res, size); - - return res; + return remove_block(bits::next_pow2_bits(size)); } /** @@ -298,9 +208,8 @@ namespace snmalloc * This is useful for allowing the space required for alignment to be * used, by smaller objects. */ - template - CapPtr - reserve_with_left_over(size_t size, ArenaMap& arena_map) + template + CapPtr reserve_with_left_over(size_t size) { SNMALLOC_ASSERT(size >= sizeof(void*)); @@ -308,18 +217,14 @@ namespace snmalloc size_t rsize = bits::next_pow2(size); - auto res = reserve(rsize, arena_map); + auto res = reserve(rsize); if (res != nullptr) { if (rsize > size) { - FlagLock lock(spin_lock); - add_range(pointer_offset(res, size), rsize - size); + add_range(pointer_offset(res, size), rsize - size); } - - if constexpr (committed) - commit_block(res, size); } return res; } @@ -329,29 +234,6 @@ namespace snmalloc * does not own any memory at the start and will request any that it needs * from the PAL. */ - AddressSpaceManager() = default; - - /** - * Constructor that pre-initialises the address-space manager with a region - * of memory. - */ - AddressSpaceManager(CapPtr base, size_t length) - { - add_range(base, length); - } - - /** - * Move assignment operator. This should only be used during initialisation - * of the system. There should be no concurrency. - */ - AddressSpaceManager& operator=(AddressSpaceManager&& other) noexcept - { - // Lock address space manager. This will prevent it being used by - // mistake. Fails with deadlock with any subsequent caller. - if (other.spin_lock.test_and_set()) - abort(); - ranges = other.ranges; - return *this; - } + AddressSpaceManagerCore() = default; }; } // namespace snmalloc diff --git a/src/backend/backend.h b/src/backend/backend.h new file mode 100644 index 000000000..34541b303 --- /dev/null +++ b/src/backend/backend.h @@ -0,0 +1,206 @@ +#pragma once +#include "../mem/allocconfig.h" +#include "../mem/metaslab.h" +#include "../pal/pal.h" +#include "address_space.h" +#include "pagemap.h" + +namespace snmalloc +{ + /** + * This class implements the standard backend for handling allocations. + * It abstracts page table management and address space management. + */ + template + class BackendAllocator + { + public: + using Pal = PAL; + + /** + * Local state for the backend allocator. + * + * This class contains thread local structures to make the implementation + * of the backend allocator more efficient. + */ + class LocalState + { + friend BackendAllocator; + + // TODO Separate meta data and object + AddressSpaceManagerCore local_address_space; + }; + + /** + * Global state for the backend allocator + * + * This contains the various global datastructures required to store + * meta-data for each chunk of memory, and to provide well aligned chunks + * of memory. + * + * This type is required by snmalloc to exist as part of the Backend. + */ + class GlobalState + { + friend BackendAllocator; + + // TODO Separate meta data and object + AddressSpaceManager address_space; + + FlatPagemap pagemap; + + public: + void init() + { + pagemap.init(&address_space); + + if constexpr (fixed_range) + { + abort(); + } + } + + void init(CapPtr base, size_t length) + { + address_space.add_range(base, length); + pagemap.init(&address_space, address_cast(base), length); + + if constexpr (!fixed_range) + { + abort(); + } + } + }; + + private: + /** + * Internal method for acquiring state from the local and global address + * space managers. + */ + template + static CapPtr + reserve(GlobalState& h, LocalState* local_state, size_t size) + { + // TODO have two address spaces. + UNUSED(is_meta); + + CapPtr p; + if (local_state != nullptr) + { + p = + local_state->local_address_space.template reserve_with_left_over( + size); + if (p != nullptr) + local_state->local_address_space.template commit_block(p, size); + else + { + auto& a = h.address_space; + // TODO Improve heuristics and params + auto refill_size = bits::max(size, bits::one_at_bit(21)); + auto refill = a.template reserve(refill_size); + if (refill == nullptr) + return nullptr; + local_state->local_address_space.template add_range( + refill, refill_size); + // This should succeed + p = local_state->local_address_space + .template reserve_with_left_over(size); + if (p != nullptr) + local_state->local_address_space.template commit_block( + p, size); + } + } + else + { + auto& a = h.address_space; + p = a.template reserve_with_left_over(size); + } + + return p; + } + + public: + /** + * Provide a block of meta-data with size and align. + * + * Backend allocator may use guard pages and separate area of + * address space to protect this from corruption. + */ + static CapPtr + alloc_meta_data(GlobalState& h, LocalState* local_state, size_t size) + { + return reserve(h, local_state, size); + } + + /** + * Returns a chunk of memory with alignment and size of `size`, and a + * metaslab block. + * + * It additionally set the meta-data for this chunk of memory to + * be + * (remote, sizeclass, metaslab) + * where metaslab, is the second element of the pair return. + */ + static std::pair, Metaslab*> alloc_chunk( + GlobalState& h, + LocalState* local_state, + size_t size, + RemoteAllocator* remote, + sizeclass_t sizeclass) + { + SNMALLOC_ASSERT(bits::is_pow2(size)); + SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE); + + CapPtr p = reserve(h, local_state, size); + +#ifdef SNMALLOC_TRACING + std::cout << "Alloc chunk: " << p.unsafe_ptr() << " (" << size << ")" + << std::endl; +#endif + if (p == nullptr) + { +#ifdef SNMALLOC_TRACING + std::cout << "Out of memory" << std::endl; +#endif + return {p, nullptr}; + } + + auto meta = reinterpret_cast( + reserve(h, local_state, sizeof(Metaslab)).unsafe_ptr()); + + MetaEntry t(meta, remote, sizeclass); + + for (address_t a = address_cast(p); + a < address_cast(pointer_offset(p, size)); + a += MIN_CHUNK_SIZE) + { + h.pagemap.add(a, t); + } + return {p, meta}; + } + + /** + * Get the metadata associated with a chunk. + * + * Set template parameter to true if it not an error + * to access a location that is not backed by a chunk. + */ + template + static const MetaEntry& get_meta_data(GlobalState& h, address_t p) + { + return h.pagemap.template get(p); + } + + /** + * Set the metadata associated with a chunk. + */ + static void + set_meta_data(GlobalState& h, address_t p, size_t size, MetaEntry t) + { + for (address_t a = p; a < p + size; a += MIN_CHUNK_SIZE) + { + h.pagemap.set(a, t); + } + } + }; +} // namespace snmalloc diff --git a/src/backend/pagemap.h b/src/backend/pagemap.h new file mode 100644 index 000000000..688e51d36 --- /dev/null +++ b/src/backend/pagemap.h @@ -0,0 +1,179 @@ +#pragma once + +#include "../ds/bits.h" +#include "../ds/helpers.h" +#include "../pal/pal.h" + +#include +#include + +namespace snmalloc +{ + /** + * Simple pagemap that for each GRANULARITY_BITS of the address range + * stores a T. + */ + template + class FlatPagemap + { + private: + static constexpr size_t SHIFT = GRANULARITY_BITS; + + // Before init is called will contain a single entry + // that is the default value. This is needed so that + // various calls do not have to check for nullptr. + // free(nullptr) + // and + // malloc_usable_size(nullptr) + // do not require an allocation to have ocurred before + // they are called. + inline static const T default_value{}; + T* body{const_cast(&default_value)}; + + address_t base{0}; + size_t size{0}; + + /** + * Commit entry + */ + void commit_entry(void* p) + { + auto entry_size = sizeof(T); + static_assert(sizeof(T) < OS_PAGE_SIZE); + // Rounding required for sub-page allocations. + auto page_start = pointer_align_down(p); + auto page_end = + pointer_align_up(pointer_offset(p, entry_size)); + size_t using_size = pointer_diff(page_start, page_end); + PAL::template notify_using(page_start, using_size); + } + + public: + constexpr FlatPagemap() = default; + + template + void init(ASM* a, address_t b = 0, size_t s = 0) + { + if constexpr (has_bounds) + { +#ifdef SNMALLOC_TRACING + std::cout << "Pagemap.init " << (void*)b << " (" << s << ")" + << std::endl; +#endif + SNMALLOC_ASSERT(s != 0); + // Align the start and end. We won't store for the very ends as they + // are not aligned to a chunk boundary. + base = bits::align_up(b, bits::one_at_bit(GRANULARITY_BITS)); + auto end = bits::align_down(b + s, bits::one_at_bit(GRANULARITY_BITS)); + size = end - base; + body = a->template reserve( + bits::next_pow2((size >> SHIFT) * sizeof(T))) + .template as_static() + .unsafe_ptr(); + ; + } + else + { + // The parameters should not be set without has_bounds. + UNUSED(s); + UNUSED(b); + SNMALLOC_ASSERT(s == 0); + SNMALLOC_ASSERT(b == 0); + + static constexpr size_t COVERED_BITS = + bits::ADDRESS_BITS - GRANULARITY_BITS; + static constexpr size_t ENTRIES = bits::one_at_bit(COVERED_BITS); + auto new_body = (a->template reserve(ENTRIES * sizeof(T))) + .template as_static() + .unsafe_ptr(); + + // Ensure bottom page is committed + commit_entry(&new_body[0]); + + // Set up zero page + new_body[0] = body[0]; + + body = new_body; + // TODO this is pretty sparse, should we ignore huge pages for it? + // madvise(body, size, MADV_NOHUGEPAGE); + } + } + + /** + * If the location has not been used before, then + * `potentially_out_of_range` should be set to true. + * This will ensure there is a location for the + * read/write. + */ + template + const T& get(address_t p) + { + if constexpr (has_bounds) + { + if (p - base > size) + { + if constexpr (potentially_out_of_range) + { + return default_value; + } + else + { + // Out of range null should + // still return the default value. + if (p == 0) + return default_value; + PAL::error("Internal error: Pagemap read access out of range."); + } + } + p = p - base; + } + + // This means external pointer on Windows will be slow. + if constexpr (potentially_out_of_range) + { + commit_entry(&body[p >> SHIFT]); + } + + return body[p >> SHIFT]; + } + + void set(address_t p, T t) + { +#ifdef SNMALLOC_TRACING + std::cout << "Pagemap.Set " << (void*)p << std::endl; +#endif + if constexpr (has_bounds) + { + if (p - base > size) + { + PAL::error("Internal error: Pagemap write access out of range."); + } + p = p - base; + } + + body[p >> SHIFT] = t; + } + + void add(address_t p, T t) + { +#ifdef SNMALLOC_TRACING + std::cout << "Pagemap.Add " << (void*)p << std::endl; +#endif + if constexpr (has_bounds) + { + if (p - base > size) + { + PAL::error("Internal error: Pagemap new write access out of range."); + } + p = p - base; + } + + // This could be the first time this page is used + // This will potentially be expensive on Windows, + // and we should revisit the performance here. + commit_entry(&body[p >> SHIFT]); + + body[p >> SHIFT] = t; + } + }; +} // namespace snmalloc diff --git a/src/ds/aba.h b/src/ds/aba.h index c9a5d200b..e57c574ee 100644 --- a/src/ds/aba.h +++ b/src/ds/aba.h @@ -13,7 +13,7 @@ */ namespace snmalloc { -#ifndef NDEBUG +#if !defined(NDEBUG) && !defined(SNMALLOC_DISABLE_ABA_VERIFY) // LL/SC typically can only perform one operation at a time // check this on other platforms using a thread_local. inline thread_local bool operation_in_flight = false; @@ -24,24 +24,20 @@ namespace snmalloc // fall back to locked implementation. #if defined(PLATFORM_IS_X86) && \ !(defined(GCC_NOT_CLANG) && defined(OPEN_ENCLAVE)) - template< - typename T, - Construction c = RequiresInit, - template typename Ptr = Pointer, - template typename AtomicPtr = AtomicPointer> + template class ABA { public: struct alignas(2 * sizeof(std::size_t)) Linked { - Ptr ptr; - uintptr_t aba; + T* ptr{nullptr}; + uintptr_t aba{0}; }; struct Independent { - AtomicPtr ptr; - std::atomic aba; + std::atomic ptr{nullptr}; + std::atomic aba{0}; }; static_assert( @@ -59,13 +55,9 @@ namespace snmalloc }; public: - ABA() - { - if constexpr (c == RequiresInit) - init(nullptr); - } + constexpr ABA() : independent() {} - void init(Ptr x) + void init(T* x) { independent.ptr.store(x, std::memory_order_relaxed); independent.aba.store(0, std::memory_order_relaxed); @@ -75,7 +67,7 @@ namespace snmalloc Cmp read() { -# ifndef NDEBUG +# if !defined(NDEBUG) && !defined(SNMALLOC_DISABLE_ABA_VERIFY) if (operation_in_flight) error("Only one inflight ABA operation at a time is allowed."); operation_in_flight = true; @@ -97,12 +89,12 @@ namespace snmalloc */ Cmp(Linked old, ABA* parent) : old(old), parent(parent) {} - Ptr ptr() + T* ptr() { return old.ptr; } - bool store_conditional(Ptr value) + bool store_conditional(T* value) { # if defined(_MSC_VER) && defined(SNMALLOC_VA_BITS_64) auto result = _InterlockedCompareExchange128( @@ -127,7 +119,7 @@ namespace snmalloc ~Cmp() { -# ifndef NDEBUG +# if !defined(NDEBUG) && !defined(SNMALLOC_DISABLE_ABA_VERIFY) operation_in_flight = false; # endif } @@ -137,7 +129,7 @@ namespace snmalloc }; // This method is used in Verona - Ptr peek() + T* peek() { return independent.ptr.load(std::memory_order_relaxed); } @@ -146,19 +138,15 @@ namespace snmalloc /** * Naive implementation of ABA protection using a spin lock. */ - template< - typename T, - Construction c = RequiresInit, - template typename Ptr = Pointer, - template typename AtomicPtr = AtomicPointer> + template class ABA { - AtomicPtr ptr = nullptr; + std::atomic ptr = nullptr; std::atomic_flag lock = ATOMIC_FLAG_INIT; public: // This method is used in Verona - void init(Ptr x) + void init(T* x) { ptr.store(x, std::memory_order_relaxed); } @@ -170,7 +158,7 @@ namespace snmalloc while (lock.test_and_set(std::memory_order_acquire)) Aal::pause(); -# ifndef NDEBUG +# if !defined(NDEBUG) && !defined(SNMALLOC_DISABLE_ABA_VERIFY) if (operation_in_flight) error("Only one inflight ABA operation at a time is allowed."); operation_in_flight = true; @@ -184,12 +172,12 @@ namespace snmalloc ABA* parent; public: - Ptr ptr() + T* ptr() { return parent->ptr; } - bool store_conditional(Ptr t) + bool store_conditional(T* t) { parent->ptr = t; return true; @@ -198,14 +186,14 @@ namespace snmalloc ~Cmp() { parent->lock.clear(std::memory_order_release); -# ifndef NDEBUG +# if !defined(NDEBUG) && !defined(SNMALLOC_DISABLE_ABA_VERIFY) operation_in_flight = false; # endif } }; // This method is used in Verona - Ptr peek() + T* peek() { return ptr.load(std::memory_order_relaxed); } diff --git a/src/ds/address.h b/src/ds/address.h index 56fe12141..0dde50245 100644 --- a/src/ds/address.h +++ b/src/ds/address.h @@ -29,7 +29,7 @@ namespace snmalloc inline CapPtr pointer_offset(CapPtr base, size_t diff) { - return CapPtr(pointer_offset(base.unsafe_capptr, diff)); + return CapPtr(pointer_offset(base.unsafe_ptr(), diff)); } /** @@ -45,8 +45,7 @@ namespace snmalloc inline CapPtr pointer_offset_signed(CapPtr base, ptrdiff_t diff) { - return CapPtr( - pointer_offset_signed(base.unsafe_capptr, diff)); + return CapPtr(pointer_offset_signed(base.unsafe_ptr(), diff)); } /** @@ -69,7 +68,7 @@ namespace snmalloc template inline address_t address_cast(CapPtr a) { - return address_cast(a.unsafe_capptr); + return address_cast(a.unsafe_ptr()); } /** @@ -95,7 +94,7 @@ namespace snmalloc * power of two. */ template - SNMALLOC_FAST_PATH T* pointer_align_down(void* p) + inline T* pointer_align_down(void* p) { static_assert(alignment > 0); static_assert(bits::is_pow2(alignment)); @@ -115,7 +114,7 @@ namespace snmalloc template inline CapPtr pointer_align_down(CapPtr p) { - return CapPtr(pointer_align_down(p.unsafe_capptr)); + return CapPtr(pointer_align_down(p.unsafe_ptr())); } template @@ -149,7 +148,7 @@ namespace snmalloc template inline CapPtr pointer_align_up(CapPtr p) { - return CapPtr(pointer_align_up(p.unsafe_capptr)); + return CapPtr(pointer_align_up(p.unsafe_ptr())); } template @@ -163,7 +162,7 @@ namespace snmalloc * a power of two. */ template - SNMALLOC_FAST_PATH T* pointer_align_down(void* p, size_t alignment) + inline T* pointer_align_down(void* p, size_t alignment) { SNMALLOC_ASSERT(alignment > 0); SNMALLOC_ASSERT(bits::is_pow2(alignment)); @@ -196,7 +195,7 @@ namespace snmalloc inline CapPtr pointer_align_up(CapPtr p, size_t alignment) { - return CapPtr(pointer_align_up(p.unsafe_capptr, alignment)); + return CapPtr(pointer_align_up(p.unsafe_ptr(), alignment)); } /** @@ -218,7 +217,7 @@ namespace snmalloc enum capptr_bounds Ubounds> inline size_t pointer_diff(CapPtr base, CapPtr cursor) { - return pointer_diff(base.unsafe_capptr, cursor.unsafe_capptr); + return pointer_diff(base.unsafe_ptr(), cursor.unsafe_ptr()); } /** @@ -239,7 +238,7 @@ namespace snmalloc inline ptrdiff_t pointer_diff_signed(CapPtr base, CapPtr cursor) { - return pointer_diff_signed(base.unsafe_capptr, cursor.unsafe_capptr); + return pointer_diff_signed(base.unsafe_ptr(), cursor.unsafe_ptr()); } } // namespace snmalloc diff --git a/src/ds/bits.h b/src/ds/bits.h index ec32e3c77..0853b8ae3 100644 --- a/src/ds/bits.h +++ b/src/ds/bits.h @@ -53,7 +53,7 @@ namespace snmalloc static constexpr size_t ADDRESS_BITS = is64() ? 48 : 32; - SNMALLOC_FAST_PATH size_t clz(size_t x) + inline SNMALLOC_FAST_PATH size_t clz(size_t x) { SNMALLOC_ASSERT(x != 0); // Calling with 0 is UB on some implementations #if defined(_MSC_VER) @@ -219,7 +219,7 @@ namespace snmalloc return (x & (x - 1)) == 0; } - SNMALLOC_FAST_PATH size_t next_pow2(size_t x) + inline SNMALLOC_FAST_PATH size_t next_pow2(size_t x) { // Correct for numbers [0..MAX_SIZE >> 1). // Returns 1 for x > (MAX_SIZE >> 1). @@ -246,12 +246,20 @@ namespace snmalloc return one_at_bit(BITS - clz_const(x - 1)); } - constexpr size_t next_pow2_bits_const(size_t x) + constexpr size_t prev_pow2_const(size_t x) + { + if (x <= 2) + return x; + + return one_at_bit(BITS - (clz_const(x + 1) + 1)); + } + + inline constexpr size_t next_pow2_bits_const(size_t x) { return BITS - clz_const(x - 1); } - constexpr SNMALLOC_FAST_PATH size_t + inline constexpr SNMALLOC_FAST_PATH size_t align_down(size_t value, size_t alignment) { SNMALLOC_ASSERT(is_pow2(alignment)); @@ -261,7 +269,8 @@ namespace snmalloc return value; } - constexpr SNMALLOC_FAST_PATH size_t align_up(size_t value, size_t alignment) + inline constexpr SNMALLOC_FAST_PATH size_t + align_up(size_t value, size_t alignment) { SNMALLOC_ASSERT(is_pow2(alignment)); diff --git a/src/ds/cdllist.h b/src/ds/cdllist.h index f32f66197..56c601f97 100644 --- a/src/ds/cdllist.h +++ b/src/ds/cdllist.h @@ -1,5 +1,6 @@ #pragma once +#include "address.h" #include "defines.h" #include "ptrwrap.h" @@ -8,146 +9,67 @@ namespace snmalloc { - template typename Ptr = Pointer> - class CDLLNodeBase - { - /** - * to_next is used to handle a zero initialised data structure. - * This means that `is_empty` works even when the constructor hasn't - * been run. - */ - ptrdiff_t to_next = 0; - - protected: - void set_next(Ptr c) - { - to_next = pointer_diff_signed(Ptr>(this), c); - } - - public: - SNMALLOC_FAST_PATH bool is_empty() - { - return to_next == 0; - } - - SNMALLOC_FAST_PATH Ptr get_next() - { - return static_cast>(pointer_offset_signed(this, to_next)); - } - }; - - template typename Ptr = Pointer> - class CDLLNodeBaseNext - { - /** - * Like to_next in the pointer-less case, this version still works with - * zero-initialized data structure. To make `is_empty` work in this case, - * next is set to `nullptr` rather than `this` when the list is empty. - * - */ - - Ptr next = nullptr; - - protected: - void set_next(Ptr c) - { - next = address_cast(c) == address_cast(this) ? nullptr : c; - } - - public: - SNMALLOC_FAST_PATH bool is_empty() - { - return next == nullptr; - } - - SNMALLOC_FAST_PATH Ptr get_next() - { - return next == nullptr ? Ptr(static_cast(this)) : next; - } - }; - - template typename Ptr = Pointer> - using CDLLNodeParent = std::conditional_t< - aal_supports, - CDLLNodeBaseNext, - CDLLNodeBase>; - /** + * TODO Rewrite for actual use, no longer Cyclic or doubly linked. + * * Special class for cyclic doubly linked non-empty linked list * * This code assumes there is always one element in the list. The client * must ensure there is a sentinal element. */ template typename Ptr = Pointer> - class CDLLNode : public CDLLNodeParent, Ptr> + class CDLLNode { - Ptr prev = nullptr; + Ptr next{nullptr}; + + constexpr void set_next(Ptr c) + { + next = c; + } public: /** * Single element cyclic list. This is the empty case. */ - CDLLNode() + constexpr CDLLNode() { - this->set_next(Ptr(this)); - prev = Ptr(this); + this->set_next(nullptr); } - /** - * Removes this element from the cyclic list is it part of. - */ - SNMALLOC_FAST_PATH void remove() + SNMALLOC_FAST_PATH bool is_empty() { - SNMALLOC_ASSERT(!this->is_empty()); - debug_check(); - this->get_next()->prev = prev; - prev->set_next(this->get_next()); - // As this is no longer in the list, check invariant for - // neighbouring element. - this->get_next()->debug_check(); + return next == nullptr; + } -#ifndef NDEBUG - this->set_next(nullptr); - prev = nullptr; -#endif + SNMALLOC_FAST_PATH Ptr get_next() + { + return next; } /** - * Nulls the previous pointer + * Single element cyclic list. This is the uninitialised case. * - * The Meta-slab uses nullptr in prev to mean that it is not part of a - * size class list. - **/ - void null_prev() - { - prev = nullptr; - } + * This entry should never be accessed and is only used to make + * a fake metaslab. + */ + constexpr CDLLNode(bool) {} - SNMALLOC_FAST_PATH Ptr get_prev() + SNMALLOC_FAST_PATH Ptr pop() { - return prev; + SNMALLOC_ASSERT(!this->is_empty()); + auto result = get_next(); + set_next(result->get_next()); + return result; } - SNMALLOC_FAST_PATH void insert_next(Ptr item) + SNMALLOC_FAST_PATH void insert(Ptr item) { debug_check(); item->set_next(this->get_next()); - this->get_next()->prev = item; - item->prev = this; set_next(item); debug_check(); } - SNMALLOC_FAST_PATH void insert_prev(Ptr item) - { - debug_check(); - item->prev = prev; - prev->set_next(item); - item->set_next(Ptr(this)); - prev = item; - debug_check(); - } - /** * Checks the lists invariants * x->next->prev = x @@ -156,15 +78,15 @@ namespace snmalloc void debug_check() { #ifndef NDEBUG - Ptr item = this->get_next(); - auto p = Ptr(this); - - do - { - SNMALLOC_ASSERT(item->prev == p); - p = item; - item = item->get_next(); - } while (item != Ptr(this)); + // Ptr item = this->get_next(); + // auto p = Ptr(this); + + // do + // { + // SNMALLOC_ASSERT(item->prev == p); + // p = item; + // item = item->get_next(); + // } while (item != Ptr(this)); #endif } }; diff --git a/src/ds/defines.h b/src/ds/defines.h index ca2289e95..88ff5ea3c 100644 --- a/src/ds/defines.h +++ b/src/ds/defines.h @@ -7,17 +7,30 @@ # define unlikely(x) !!(x) # define SNMALLOC_SLOW_PATH NOINLINE # define SNMALLOC_FAST_PATH ALWAYSINLINE +# if _MSC_VER >= 1927 +# define SNMALLOC_FAST_PATH_LAMBDA [[msvc::forceinline]] +# else +# define SNMALLOC_FAST_PATH_LAMBDA +# endif # define SNMALLOC_PURE # define SNMALLOC_COLD +# define SNMALLOC_REQUIRE_CONSTINIT #else # define likely(x) __builtin_expect(!!(x), 1) # define unlikely(x) __builtin_expect(!!(x), 0) # define ALWAYSINLINE __attribute__((always_inline)) # define NOINLINE __attribute__((noinline)) # define SNMALLOC_SLOW_PATH NOINLINE -# define SNMALLOC_FAST_PATH inline ALWAYSINLINE +# define SNMALLOC_FAST_PATH ALWAYSINLINE +# define SNMALLOC_FAST_PATH_LAMBDA SNMALLOC_FAST_PATH # define SNMALLOC_PURE __attribute__((const)) # define SNMALLOC_COLD __attribute__((cold)) +# ifdef __clang__ +# define SNMALLOC_REQUIRE_CONSTINIT \ + [[clang::require_constant_initialization]] +# else +# define SNMALLOC_REQUIRE_CONSTINIT +# endif #endif #if defined(__cpp_constinit) && __cpp_constinit >= 201907 @@ -100,3 +113,27 @@ namespace snmalloc } while (0) # endif #endif + +// // The CHECK_CLIENT macro is used to turn on minimal checking of the client +// // calling the API correctly. +// #if !defined(NDEBUG) && !defined(CHECK_CLIENT) +// # define CHECK_CLIENT +// #endif + +inline SNMALLOC_FAST_PATH void check_client_error(const char* const str) +{ + //[[clang::musttail]] + return snmalloc::error(str); +} + +inline SNMALLOC_FAST_PATH void +check_client_impl(bool test, const char* const str) +{ + if (unlikely(!test)) + check_client_error(str); +} +#ifdef CHECK_CLIENT +# define check_client(test, str) check_client_impl(test, str) +#else +# define check_client(test, str) +#endif \ No newline at end of file diff --git a/src/ds/dllist.h b/src/ds/dllist.h index 36c669c90..d4c20e8bc 100644 --- a/src/ds/dllist.h +++ b/src/ds/dllist.h @@ -1,5 +1,6 @@ #pragma once +#include "address.h" #include "helpers.h" #include "invalidptr.h" #include "ptrwrap.h" diff --git a/src/ds/helpers.h b/src/ds/helpers.h index 49211a3b8..a3d3e8886 100644 --- a/src/ds/helpers.h +++ b/src/ds/helpers.h @@ -3,6 +3,7 @@ #include "bits.h" #include "flaglock.h" +#include #include namespace snmalloc @@ -72,6 +73,7 @@ namespace snmalloc } }; +#ifdef CHECK_CLIENT template class ModArray { @@ -84,7 +86,7 @@ namespace snmalloc }; static constexpr size_t rlength = bits::next_pow2_const(length); - TWrap array[rlength]; + std::array array; public: constexpr const T& operator[](const size_t i) const @@ -97,14 +99,22 @@ namespace snmalloc return array[i & (rlength - 1)].v; } }; +#else + template + using ModArray = std::array; +#endif /** * Helper class to execute a specified function on destruction. */ - template + template class OnDestruct { + F f; + public: + OnDestruct(F f) : f(f) {} + ~OnDestruct() { f(); diff --git a/src/ds/invalidptr.h b/src/ds/invalidptr.h index b96be083c..1a75f0b29 100644 --- a/src/ds/invalidptr.h +++ b/src/ds/invalidptr.h @@ -1,5 +1,7 @@ #pragma once +#include "address.h" + namespace snmalloc { /** diff --git a/src/ds/mpmcstack.h b/src/ds/mpmcstack.h index bd16c0872..d7a4f9259 100644 --- a/src/ds/mpmcstack.h +++ b/src/ds/mpmcstack.h @@ -1,50 +1,49 @@ #pragma once #include "aba.h" +#include "ptrwrap.h" namespace snmalloc { - template< - class T, - Construction c = RequiresInit, - template typename Ptr = Pointer, - template typename AtomicPtr = AtomicPointer> + template class MPMCStack { - using ABAT = ABA; + using ABAT = ABA; private: - static_assert( - std::is_same>::value, - "T->next must be an AtomicPtr"); - ABAT stack; public: - void push(Ptr item) + constexpr MPMCStack() = default; + + void push(T* item) { + static_assert( + std::is_same>::value, + "T->next must be an std::atomic"); + return push(item, item); } - void push(Ptr first, Ptr last) + void push(T* first, T* last) { // Pushes an item on the stack. auto cmp = stack.read(); do { - Ptr top = cmp.ptr(); + auto top = cmp.ptr(); last->next.store(top, std::memory_order_release); } while (!cmp.store_conditional(first)); } - Ptr pop() + T* pop() { // Returns the next item. If the returned value is decommitted, it is // possible for the read of top->next to segfault. auto cmp = stack.read(); - Ptr top; - Ptr next; + T* top; + T* next; do { @@ -59,11 +58,11 @@ namespace snmalloc return top; } - Ptr pop_all() + T* pop_all() { // Returns all items as a linked list, leaving an empty stack. auto cmp = stack.read(); - Ptr top; + T* top; do { diff --git a/src/ds/mpscq.h b/src/ds/mpscq.h index 683d0db53..c3e8fb993 100644 --- a/src/ds/mpscq.h +++ b/src/ds/mpscq.h @@ -18,9 +18,11 @@ namespace snmalloc "T->next must be an AtomicPtr"); AtomicPtr back{nullptr}; - Ptr front = nullptr; + Ptr front{nullptr}; public: + constexpr MPSCQ() = default; + void invariant() { SNMALLOC_ASSERT(back != nullptr); @@ -61,6 +63,11 @@ namespace snmalloc prev->next.store(first, std::memory_order_relaxed); } + Ptr peek() + { + return front; + } + std::pair, bool> dequeue() { // Returns the front message, or null if not possible to return a message. @@ -68,10 +75,10 @@ namespace snmalloc Ptr first = front; Ptr next = first->next.load(std::memory_order_relaxed); + Aal::prefetch(&(next->next)); if (next != nullptr) { front = next; - Aal::prefetch(&(next->next)); SNMALLOC_ASSERT(front != nullptr); std::atomic_thread_fence(std::memory_order_acquire); invariant(); diff --git a/src/ds/ptrwrap.h b/src/ds/ptrwrap.h index 8cf1f8109..ff073e942 100644 --- a/src/ds/ptrwrap.h +++ b/src/ds/ptrwrap.h @@ -1,5 +1,7 @@ #pragma once +#include "defines.h" + #include namespace snmalloc @@ -99,16 +101,17 @@ namespace snmalloc * summary of its StrictProvenance metadata. */ template - struct CapPtr + class CapPtr { - T* unsafe_capptr; + uintptr_t unsafe_capptr; + public: /** * nullptr is implicitly constructable at any bounds type */ - CapPtr(const std::nullptr_t n) : unsafe_capptr(n) {} + constexpr CapPtr(const std::nullptr_t) : unsafe_capptr(0) {} - CapPtr() : CapPtr(nullptr) {} + constexpr CapPtr() : CapPtr(nullptr){}; /** * all other constructions must be explicit @@ -124,18 +127,20 @@ namespace snmalloc # pragma warning(push) # pragma warning(disable : 4702) #endif - explicit CapPtr(T* p) : unsafe_capptr(p) {} + constexpr explicit CapPtr(uintptr_t p) : unsafe_capptr(p) {} #ifdef _MSC_VER # pragma warning(pop) #endif + explicit CapPtr(T* p) : unsafe_capptr(reinterpret_cast(p)) {} + /** * Allow static_cast<>-s that preserve bounds but vary the target type. */ template SNMALLOC_FAST_PATH CapPtr as_static() { - return CapPtr(static_cast(this->unsafe_capptr)); + return CapPtr(this->unsafe_capptr); } SNMALLOC_FAST_PATH CapPtr as_void() @@ -149,7 +154,7 @@ namespace snmalloc template SNMALLOC_FAST_PATH CapPtr as_reinterpret() { - return CapPtr(reinterpret_cast(this->unsafe_capptr)); + return CapPtr(this->unsafe_capptr); } SNMALLOC_FAST_PATH bool operator==(const CapPtr& rhs) const @@ -167,6 +172,16 @@ namespace snmalloc return this->unsafe_capptr < rhs.unsafe_capptr; } + [[nodiscard]] SNMALLOC_FAST_PATH T* unsafe_ptr() const + { + return reinterpret_cast(this->unsafe_capptr); + } + + [[nodiscard]] SNMALLOC_FAST_PATH uintptr_t unsafe_uintptr() const + { + return this->unsafe_capptr; + } + SNMALLOC_FAST_PATH T* operator->() const { /* @@ -174,7 +189,7 @@ namespace snmalloc * client; we should be doing nothing with them. */ static_assert(bounds != CBAllocE); - return this->unsafe_capptr; + return unsafe_ptr(); } }; @@ -198,7 +213,7 @@ namespace snmalloc * several chunks) to be the allocation. */ template - SNMALLOC_FAST_PATH CapPtr + inline SNMALLOC_FAST_PATH CapPtr capptr_chunk_is_alloc(CapPtr p) { return CapPtr(p.unsafe_capptr); @@ -208,9 +223,9 @@ namespace snmalloc * With all the bounds and constraints in place, it's safe to extract a void * pointer (to reveal to the client). */ - SNMALLOC_FAST_PATH void* capptr_reveal(CapPtr p) + inline SNMALLOC_FAST_PATH void* capptr_reveal(CapPtr p) { - return p.unsafe_capptr; + return p.unsafe_ptr(); } /** @@ -230,12 +245,12 @@ namespace snmalloc /** * nullptr is constructable at any bounds type */ - AtomicCapPtr(const std::nullptr_t n) : unsafe_capptr(n) {} + constexpr AtomicCapPtr(const std::nullptr_t n) : unsafe_capptr(n) {} /** * Interconversion with CapPtr */ - AtomicCapPtr(CapPtr p) : unsafe_capptr(p.unsafe_capptr) {} + AtomicCapPtr(CapPtr p) : unsafe_capptr(p.unsafe_ptr()) {} operator CapPtr() const noexcept { @@ -261,7 +276,7 @@ namespace snmalloc CapPtr desired, std::memory_order order = std::memory_order_seq_cst) noexcept { - this->unsafe_capptr.store(desired.unsafe_capptr, order); + this->unsafe_capptr.store(desired.unsafe_ptr(), order); } SNMALLOC_FAST_PATH CapPtr exchange( @@ -269,7 +284,7 @@ namespace snmalloc std::memory_order order = std::memory_order_seq_cst) noexcept { return CapPtr( - this->unsafe_capptr.exchange(desired.unsafe_capptr, order)); + this->unsafe_capptr.exchange(desired.unsafe_ptr(), order)); } SNMALLOC_FAST_PATH bool operator==(const AtomicCapPtr& rhs) const diff --git a/src/mem/alloc.h b/src/mem/alloc.h deleted file mode 100644 index 1ca74a9c3..000000000 --- a/src/mem/alloc.h +++ /dev/null @@ -1,1563 +0,0 @@ -#pragma once - -#ifdef _MSC_VER -# define ALLOCATOR __declspec(allocator) -#else -# define ALLOCATOR -#endif - -#include "../pal/pal_consts.h" -#include "allocstats.h" -#include "chunkmap.h" -#include "external_alloc.h" -#include "largealloc.h" -#include "mediumslab.h" -#include "pooled.h" -#include "remoteallocator.h" -#include "sizeclasstable.h" -#include "slab.h" - -#include -#include - -namespace snmalloc -{ - enum Boundary - { - /** - * The location of the first byte of this allocation. - */ - Start, - /** - * The location of the last byte of the allocation. - */ - End, - /** - * The location one past the end of the allocation. This is mostly useful - * for bounds checking, where anything less than this value is safe. - */ - OnePastEnd - }; - - // This class is just used so that the free lists are the first entry - // in the allocator and hence has better code gen. - // It contains a free list per small size class. These are used for - // allocation on the fast path. This part of the code is inspired by mimalloc. - class FastFreeLists - { - protected: - FreeListIter small_fast_free_lists[NUM_SMALL_CLASSES]; - - public: - FastFreeLists() : small_fast_free_lists() {} - }; - - /** - * Allocator. This class is parameterised on five template parameters. - * - * The first two template parameter provides a hook to allow the allocator in - * use to be dynamically modified. This is used to implement a trick from - * mimalloc that avoids a conditional branch on the fast path. We - * initialise the thread-local allocator pointer with the address of a global - * allocator, which never owns any memory. The first returns true, if is - * passed the global allocator. The second initialises the thread-local - * allocator if it is has been been initialised already. Splitting into two - * functions allows for the code to be structured into tail calls to improve - * codegen. The second template takes a function that takes the allocator - * that is initialised, and the value returned, is returned by - * `InitThreadAllocator`. This is used incase we are running during teardown - * and the thread local allocator cannot be kept alive. - * - * The `MemoryProvider` defines the source of memory for this allocator. - * Allocators try to reuse address space by allocating from existing slabs or - * reusing freed large allocations. When they need to allocate a new chunk - * of memory they request space from the `MemoryProvider`. - * - * The `ChunkMap` parameter provides the adaptor to the pagemap. This is used - * to associate metadata with large (16MiB, by default) regions, allowing an - * allocator to find the allocator responsible for that region. - * - * The final template parameter, `IsQueueInline`, defines whether the - * message queue for this allocator should be stored as a field of the - * allocator (`true`) or provided externally, allowing it to be anywhere else - * in the address space (`false`). - */ - template< - bool (*NeedsInitialisation)(void*), - void* (*InitThreadAllocator)(function_ref), - class MemoryProvider = GlobalVirtual, - class ChunkMap = SNMALLOC_DEFAULT_CHUNKMAP, - bool IsQueueInline = true> - class Allocator : public FastFreeLists, - public Pooled> - { - friend RemoteCache; - - LargeAlloc large_allocator; - ChunkMap chunk_map; - LocalEntropy entropy; - - /** - * Per size class bumpptr for building new free lists - * If aligned to a SLAB start, then it is empty, and a new - * slab is required. - */ - CapPtr bump_ptrs[NUM_SMALL_CLASSES] = {nullptr}; - - public: - Stats& stats() - { - return large_allocator.stats; - } - - template - friend class AllocPool; - - /** - * Allocate memory of a statically known size. - */ - template - SNMALLOC_FAST_PATH ALLOCATOR void* alloc() - { - static_assert(size != 0, "Size must not be zero."); -#ifdef SNMALLOC_PASS_THROUGH - // snmalloc guarantees a lot of alignment, so we can depend on this - // make pass through call aligned_alloc with the alignment snmalloc - // would guarantee. - void* result = external_alloc::aligned_alloc( - natural_alignment(size), round_size(size)); - if constexpr (zero_mem == YesZero) - memset(result, 0, size); - return result; -#else - constexpr sizeclass_t sizeclass = size_to_sizeclass_const(size); - - stats().alloc_request(size); - - if constexpr (sizeclass < NUM_SMALL_CLASSES) - { - return capptr_reveal(small_alloc(size)); - } - else if constexpr (sizeclass < NUM_SIZECLASSES) - { - handle_message_queue(); - constexpr size_t rsize = sizeclass_to_size(sizeclass); - return capptr_reveal(medium_alloc(sizeclass, rsize, size)); - } - else - { - handle_message_queue(); - return capptr_reveal(large_alloc(size)); - } -#endif - } - - /** - * Allocate memory of a dynamically known size. - */ - template - SNMALLOC_FAST_PATH ALLOCATOR void* alloc(size_t size) - { -#ifdef SNMALLOC_PASS_THROUGH - // snmalloc guarantees a lot of alignment, so we can depend on this - // make pass through call aligned_alloc with the alignment snmalloc - // would guarantee. - void* result = external_alloc::aligned_alloc( - natural_alignment(size), round_size(size)); - if constexpr (zero_mem == YesZero) - memset(result, 0, size); - return result; -#else - // Perform the - 1 on size, so that zero wraps around and ends up on - // slow path. - if (likely((size - 1) <= (sizeclass_to_size(NUM_SMALL_CLASSES - 1) - 1))) - { - // Allocations smaller than the slab size are more likely. Improve - // branch prediction by placing this case first. - return capptr_reveal(small_alloc(size)); - } - - return capptr_reveal(alloc_not_small(size)); - } - - template - SNMALLOC_SLOW_PATH CapPtr alloc_not_small(size_t size) - { - handle_message_queue(); - - if (size == 0) - { - return small_alloc(1); - } - - sizeclass_t sizeclass = size_to_sizeclass(size); - if (sizeclass < NUM_SIZECLASSES) - { - size_t rsize = sizeclass_to_size(sizeclass); - return medium_alloc(sizeclass, rsize, size); - } - - return large_alloc(size); -#endif - } - - /* - * Free memory of a statically known size. Must be called with an - * external pointer. - */ - template - void dealloc(void* p_raw) - { -#ifdef SNMALLOC_PASS_THROUGH - UNUSED(size); - return external_alloc::free(p_raw); -#else - constexpr sizeclass_t sizeclass = size_to_sizeclass_const(size); - - auto p_ret = CapPtr(p_raw); - auto p_auth = large_allocator.capptr_amplify(p_ret); - - if (sizeclass < NUM_SMALL_CLASSES) - { - auto super = Superslab::get(p_auth); - - small_dealloc_unchecked(super, p_auth, p_ret, sizeclass); - } - else if (sizeclass < NUM_SIZECLASSES) - { - auto slab = Mediumslab::get(p_auth); - - medium_dealloc_unchecked(slab, p_auth, p_ret, sizeclass); - } - else - { - large_dealloc_unchecked(p_auth, p_ret, size); - } -#endif - } - - /* - * Free memory of a dynamically known size. Must be called with an - * external pointer. - */ - SNMALLOC_FAST_PATH void dealloc(void* p_raw, size_t size) - { -#ifdef SNMALLOC_PASS_THROUGH - UNUSED(size); - return external_alloc::free(p_raw); -#else - SNMALLOC_ASSERT(p_raw != nullptr); - - auto p_ret = CapPtr(p_raw); - auto p_auth = large_allocator.capptr_amplify(p_ret); - - if (likely((size - 1) <= (sizeclass_to_size(NUM_SMALL_CLASSES - 1) - 1))) - { - auto super = Superslab::get(p_auth); - sizeclass_t sizeclass = size_to_sizeclass(size); - - small_dealloc_unchecked(super, p_auth, p_ret, sizeclass); - return; - } - dealloc_sized_slow(p_auth, p_ret, size); -#endif - } - - SNMALLOC_SLOW_PATH void dealloc_sized_slow( - CapPtr p_auth, CapPtr p_ret, size_t size) - { - if (size == 0) - return dealloc(p_ret.unsafe_capptr, 1); - - if (likely(size <= sizeclass_to_size(NUM_SIZECLASSES - 1))) - { - auto slab = Mediumslab::get(p_auth); - sizeclass_t sizeclass = size_to_sizeclass(size); - medium_dealloc_unchecked(slab, p_auth, p_ret, sizeclass); - return; - } - large_dealloc_unchecked(p_auth, p_ret, size); - } - - /* - * Free memory of an unknown size. Must be called with an external - * pointer. - */ - SNMALLOC_FAST_PATH void dealloc(void* p_raw) - { -#ifdef SNMALLOC_PASS_THROUGH - return external_alloc::free(p_raw); -#else - - uint8_t chunkmap_slab_kind = chunkmap().get(address_cast(p_raw)); - - auto p_ret = CapPtr(p_raw); - auto p_auth = large_allocator.capptr_amplify(p_ret); - - if (likely(chunkmap_slab_kind == CMSuperslab)) - { - /* - * If this is a live allocation (and not a double- or wild-free), it's - * safe to construct these Slab and Metaslab pointers and reading the - * sizeclass won't fail, since either we or the other allocator can't - * reuse the slab, as we have not yet deallocated this pointer. - * - * On the other hand, in the case of a double- or wild-free, this might - * fault or data race against reused memory. Eventually, we will come - * to rely on revocation to guard against these cases: changing the - * superslab kind will require revoking the whole superslab, as will - * changing a slab's size class. However, even then, until we get - * through the guard in small_dealloc_start(), we must treat this as - * possibly stale and suspect. - */ - auto super = Superslab::get(p_auth); - auto slab = Metaslab::get_slab(p_auth); - auto meta = super->get_meta(slab); - sizeclass_t sizeclass = meta->sizeclass(); - - small_dealloc_checked_sizeclass(super, slab, p_auth, p_ret, sizeclass); - return; - } - dealloc_not_small(p_auth, p_ret, chunkmap_slab_kind); - } - - SNMALLOC_SLOW_PATH void dealloc_not_small( - CapPtr p_auth, - CapPtr p_ret, - uint8_t chunkmap_slab_kind) - { - handle_message_queue(); - - if (p_ret == nullptr) - return; - - if (chunkmap_slab_kind == CMMediumslab) - { - /* - * The same reasoning from the fast path continues to hold here. These - * values are suspect until we complete the double-free check in - * medium_dealloc_smart(). - */ - auto slab = Mediumslab::get(p_auth); - sizeclass_t sizeclass = slab->get_sizeclass(); - - medium_dealloc_checked_sizeclass(slab, p_auth, p_ret, sizeclass); - return; - } - - if (chunkmap_slab_kind == CMNotOurs) - { - error("Not allocated by this allocator"); - } - - large_dealloc_checked_sizeclass( - p_auth, - p_ret, - bits::one_at_bit(chunkmap_slab_kind), - chunkmap_slab_kind); -#endif - } - - template - void* external_pointer(void* p_raw) - { -#ifdef SNMALLOC_PASS_THROUGH - error("Unsupported"); - UNUSED(p_raw); -#else - uint8_t chunkmap_slab_kind = chunkmap().get(address_cast(p_raw)); - auto p_ret = CapPtr(p_raw); - auto p_auth = large_allocator.capptr_amplify(p_ret); - - auto super = Superslab::get(p_auth); - if (chunkmap_slab_kind == CMSuperslab) - { - auto slab = Metaslab::get_slab(p_auth); - auto meta = super->get_meta(slab); - - sizeclass_t sc = meta->sizeclass(); - auto slab_end = - Aal::capptr_rebound(p_ret, pointer_offset(slab, SLAB_SIZE)); - - return capptr_reveal(external_pointer(p_ret, sc, slab_end)); - } - if (chunkmap_slab_kind == CMMediumslab) - { - auto slab = Mediumslab::get(p_auth); - - sizeclass_t sc = slab->get_sizeclass(); - auto slab_end = - Aal::capptr_rebound(p_ret, pointer_offset(slab, SUPERSLAB_SIZE)); - - return capptr_reveal(external_pointer(p_ret, sc, slab_end)); - } - - auto ss = super.as_void(); - - while (chunkmap_slab_kind >= CMLargeRangeMin) - { - // This is a large alloc redirect. - ss = pointer_offset_signed( - ss, - -(static_cast(1) - << (chunkmap_slab_kind - CMLargeRangeMin + SUPERSLAB_BITS))); - chunkmap_slab_kind = chunkmap().get(address_cast(ss)); - } - - if (chunkmap_slab_kind == CMNotOurs) - { - if constexpr ((location == End) || (location == OnePastEnd)) - // We don't know the End, so return MAX_PTR - return pointer_offset(nullptr, UINTPTR_MAX); - else - // We don't know the Start, so return MIN_PTR - return nullptr; - } - - SNMALLOC_ASSERT( - (chunkmap_slab_kind >= CMLargeMin) && - (chunkmap_slab_kind <= CMLargeMax)); - - CapPtr retss = Aal::capptr_rebound(p_ret, ss); - CapPtr ret; - - // This is a large alloc, mask off to the slab size. - if constexpr (location == Start) - ret = retss; - else if constexpr (location == End) - ret = pointer_offset(retss, (bits::one_at_bit(chunkmap_slab_kind)) - 1); - else - ret = pointer_offset(retss, bits::one_at_bit(chunkmap_slab_kind)); - - return capptr_reveal(ret); -#endif - } - - private: - SNMALLOC_SLOW_PATH static size_t alloc_size_error() - { - error("Not allocated by this allocator"); - } - - public: - SNMALLOC_FAST_PATH size_t alloc_size(const void* p_raw) - { -#ifdef SNMALLOC_PASS_THROUGH - return external_alloc::malloc_usable_size(const_cast(p_raw)); -#else - // This must be called on an external pointer. - size_t chunkmap_slab_kind = chunkmap().get(address_cast(p_raw)); - auto p_ret = CapPtr(const_cast(p_raw)); - auto p_auth = large_allocator.capptr_amplify(p_ret); - - if (likely(chunkmap_slab_kind == CMSuperslab)) - { - auto super = Superslab::get(p_auth); - - // Reading a remote sizeclass won't fail, since the other allocator - // can't reuse the slab, as we have no yet deallocated this pointer. - auto slab = Metaslab::get_slab(p_auth); - auto meta = super->get_meta(slab); - - return sizeclass_to_size(meta->sizeclass()); - } - - if (likely(chunkmap_slab_kind == CMMediumslab)) - { - auto slab = Mediumslab::get(p_auth); - // Reading a remote sizeclass won't fail, since the other allocator - // can't reuse the slab, as we have no yet deallocated this pointer. - return sizeclass_to_size(slab->get_sizeclass()); - } - - if (likely(chunkmap_slab_kind != CMNotOurs)) - { - SNMALLOC_ASSERT( - (chunkmap_slab_kind >= CMLargeMin) && - (chunkmap_slab_kind <= CMLargeMax)); - - return bits::one_at_bit(chunkmap_slab_kind); - } - - return alloc_size_error(); -#endif - } - - /** - * Return this allocator's "truncated" ID, an integer useful as a hash - * value of this allocator. - * - * Specifically, this is the address of this allocator's message queue - * with the least significant bits missing, masked by SIZECLASS_MASK. - * This will be unique for Allocs with inline queues; Allocs with - * out-of-line queues must ensure that no two queues' addresses collide - * under this masking. - */ - size_t get_trunc_id() - { - return public_state()->trunc_id(); - } - - private: - using alloc_id_t = typename Remote::alloc_id_t; - - SlabList small_classes[NUM_SMALL_CLASSES]; - DLList medium_classes[NUM_MEDIUM_CLASSES]; - - DLList super_available; - DLList super_only_short_available; - - RemoteCache remote_cache; - - std::conditional_t - remote_alloc; - - auto* public_state() - { - if constexpr (IsQueueInline) - { - return &remote_alloc; - } - else - { - return remote_alloc; - } - } - - auto& message_queue() - { - return public_state()->message_queue; - } - - template - friend class Pool; - - public: - Allocator( - MemoryProvider& m, - ChunkMap&& c = ChunkMap(), - RemoteAllocator* r = nullptr, - bool isFake = false) - : large_allocator(m), chunk_map(c) - { - if constexpr (IsQueueInline) - { - SNMALLOC_ASSERT(r == nullptr); - (void)r; - } - else - { - remote_alloc = r; - } - - // If this is fake, don't do any of the bits of initialisation that may - // allocate memory. - if (isFake) - return; - - // Entropy must be first, so that all data-structures can use the key - // it generates. - // This must occur before any freelists are constructed. - entropy.init(); - - init_message_queue(); - message_queue().invariant(); - -#ifndef NDEBUG - for (sizeclass_t i = 0; i < NUM_SIZECLASSES; i++) - { - size_t size = sizeclass_to_size(i); - sizeclass_t sc1 = size_to_sizeclass(size); - sizeclass_t sc2 = size_to_sizeclass_const(size); - size_t size1 = sizeclass_to_size(sc1); - size_t size2 = sizeclass_to_size(sc2); - - SNMALLOC_ASSERT(sc1 == i); - SNMALLOC_ASSERT(sc1 == sc2); - SNMALLOC_ASSERT(size1 == size); - SNMALLOC_ASSERT(size1 == size2); - } -#endif - } - - /** - * If result parameter is non-null, then false is assigned into the - * the location pointed to by result if this allocator is non-empty. - * - * If result pointer is null, then this code raises a Pal::error on the - * particular check that fails, if any do fail. - */ - void debug_is_empty(bool* result) - { - auto test = [&result](auto& queue) { - if (!queue.is_empty()) - { - if (result != nullptr) - *result = false; - else - error("debug_is_empty: found non-empty allocator"); - } - }; - - // Destroy the message queue so that it has no stub message. - { - CapPtr p = message_queue().destroy(); - - while (p != nullptr) - { - auto n = p->non_atomic_next; - handle_dealloc_remote(p); - p = n; - } - } - - // Dump bump allocators back into memory - for (size_t i = 0; i < NUM_SMALL_CLASSES; i++) - { - auto& bp = bump_ptrs[i]; - auto rsize = sizeclass_to_size(i); - FreeListIter ffl; - - CapPtr super = Superslab::get(bp); - auto super_slabd = capptr_debug_chunkd_from_chunk(super); - - CapPtr slab = Metaslab::get_slab(bp); - auto slab_slabd = capptr_debug_chunkd_from_chunk(slab); - - while (pointer_align_up(bp, SLAB_SIZE) != bp) - { - Slab::alloc_new_list(bp, ffl, rsize, entropy); - while (!ffl.empty()) - { - small_dealloc_offseted_inner( - super_slabd, slab_slabd, ffl.take(entropy), i); - } - } - } - - for (size_t i = 0; i < NUM_SMALL_CLASSES; i++) - { - if (!small_fast_free_lists[i].empty()) - { - auto head = small_fast_free_lists[i].peek(); - auto head_auth = large_allocator.capptr_amplify(head); - auto super = Superslab::get(head_auth); - auto slab = Metaslab::get_slab(head_auth); - do - { - auto curr = small_fast_free_lists[i].take(entropy); - small_dealloc_offseted_inner(super, slab, curr, i); - } while (!small_fast_free_lists[i].empty()); - - test(small_classes[i]); - } - } - - for (auto& medium_class : medium_classes) - { - test(medium_class); - } - - test(super_available); - test(super_only_short_available); - - // Place the static stub message on the queue. - init_message_queue(); - } - - template - static CapPtr external_pointer( - CapPtr p_ret, - sizeclass_t sizeclass, - CapPtr end_point) - { - size_t rsize = sizeclass_to_size(sizeclass); - - auto end_point_correction = location == End ? - pointer_offset_signed(end_point, -1) : - (location == OnePastEnd ? - end_point : - pointer_offset_signed(end_point, -static_cast(rsize))); - - size_t offset_from_end = - pointer_diff(p_ret, pointer_offset_signed(end_point, -1)); - - size_t end_to_end = round_by_sizeclass(sizeclass, offset_from_end); - - return pointer_offset_signed( - end_point_correction, -static_cast(end_to_end)); - } - - void init_message_queue() - { - // Manufacture an allocation to prime the queue - // Using an actual allocation removes a conditional from a critical path. - auto dummy = CapPtr(alloc(MIN_ALLOC_SIZE)) - .template as_static(); - if (dummy == nullptr) - { - error("Critical error: Out-of-memory during initialisation."); - } - dummy->set_info(get_trunc_id(), size_to_sizeclass_const(MIN_ALLOC_SIZE)); - message_queue().init(dummy); - } - - SNMALLOC_FAST_PATH void handle_dealloc_remote(CapPtr p) - { - auto target_id = Remote::trunc_target_id(p, &large_allocator); - if (likely(target_id == get_trunc_id())) - { - // Destined for my slabs - auto p_auth = large_allocator.template capptr_amplify(p); - auto super = Superslab::get(p_auth); - auto sizeclass = p->sizeclass(); - dealloc_not_large_local(super, Remote::clear(p), sizeclass); - } - else - { - // Merely routing; despite the cast here, p is going to be cast right - // back to a Remote. - remote_cache.dealloc( - target_id, p.template as_reinterpret(), p->sizeclass()); - } - } - - SNMALLOC_SLOW_PATH void dealloc_not_large( - RemoteAllocator* target, CapPtr p, sizeclass_t sizeclass) - { - if (likely(target->trunc_id() == get_trunc_id())) - { - auto p_auth = large_allocator.capptr_amplify(p); - auto super = Superslab::get(p_auth); - dealloc_not_large_local(super, p, sizeclass); - } - else - { - remote_dealloc_and_post(target, p, sizeclass); - } - } - - // TODO: Adjust when medium slab same as super slab. - // Second parameter should be a FreeObject. - SNMALLOC_FAST_PATH void dealloc_not_large_local( - CapPtr super, - CapPtr p, - sizeclass_t sizeclass) - { - // Guard against remote queues that have colliding IDs - SNMALLOC_ASSERT(super->get_allocator() == public_state()); - - if (likely(sizeclass < NUM_SMALL_CLASSES)) - { - SNMALLOC_ASSERT(super->get_kind() == Super); - check_client( - super->get_kind() == Super, - "Heap Corruption: Sizeclass of remote dealloc corrupt."); - auto slab = Metaslab::get_slab(Aal::capptr_rebound(super.as_void(), p)); - check_client( - super->get_meta(slab)->sizeclass() == sizeclass, - "Heap Corruption: Sizeclass of remote dealloc corrupt."); - small_dealloc_offseted(super, slab, p, sizeclass); - } - else - { - auto medium = super.template as_reinterpret(); - SNMALLOC_ASSERT(medium->get_kind() == Medium); - check_client( - medium->get_kind() == Medium, - "Heap Corruption: Sizeclass of remote dealloc corrupt."); - check_client( - medium->get_sizeclass() == sizeclass, - "Heap Corruption: Sizeclass of remote dealloc corrupt."); - medium_dealloc_local(medium, p, sizeclass); - } - } - - SNMALLOC_SLOW_PATH void handle_message_queue_inner() - { - for (size_t i = 0; i < REMOTE_BATCH; i++) - { - auto r = message_queue().dequeue(); - - if (unlikely(!r.second)) - break; - - handle_dealloc_remote(r.first); - } - - // Our remote queues may be larger due to forwarding remote frees. - if (likely(remote_cache.capacity > 0)) - return; - - stats().remote_post(); - remote_cache.post(this, get_trunc_id()); - } - - /** - * Check if this allocator has messages to deallocate blocks from another - * thread - */ - SNMALLOC_FAST_PATH bool has_messages() - { - return !(message_queue().is_empty()); - } - - SNMALLOC_FAST_PATH void handle_message_queue() - { - // Inline the empty check, but not necessarily the full queue handling. - if (likely(!has_messages())) - return; - - handle_message_queue_inner(); - } - - CapPtr get_superslab() - { - auto super = super_available.get_head(); - - if (super != nullptr) - return super; - - super = large_allocator - .template alloc(0, SUPERSLAB_SIZE, SUPERSLAB_SIZE) - .template as_reinterpret(); - - if (super == nullptr) - return super; - - super->init(public_state()); - chunkmap().set_slab(super); - super_available.insert(super); - return super; - } - - void reposition_superslab(CapPtr super) - { - switch (super->get_status()) - { - case Superslab::Full: - { - // Remove from the list of superslabs that have available slabs. - super_available.remove(super); - break; - } - - case Superslab::Available: - { - // Do nothing. - break; - } - - case Superslab::OnlyShortSlabAvailable: - { - // Move from the general list to the short slab only list. - super_available.remove(super); - super_only_short_available.insert(super); - break; - } - - case Superslab::Empty: - { - // Can't be empty since we just allocated. - error("Unreachable"); - break; - } - } - } - - SNMALLOC_SLOW_PATH CapPtr alloc_slab(sizeclass_t sizeclass) - { - stats().sizeclass_alloc_slab(sizeclass); - if (Superslab::is_short_sizeclass(sizeclass)) - { - // Pull a short slab from the list of superslabs that have only the - // short slab available. - CapPtr super = super_only_short_available.pop(); - - if (super != nullptr) - { - auto slab = Superslab::alloc_short_slab(super, sizeclass); - SNMALLOC_ASSERT(super->is_full()); - return slab; - } - - super = get_superslab(); - - if (super == nullptr) - return nullptr; - - auto slab = Superslab::alloc_short_slab(super, sizeclass); - reposition_superslab(super); - return slab; - } - - auto super = get_superslab(); - - if (super == nullptr) - return nullptr; - - auto slab = Superslab::alloc_slab(super, sizeclass); - reposition_superslab(super); - return slab; - } - - template - SNMALLOC_FAST_PATH CapPtr small_alloc(size_t size) - { - SNMALLOC_ASSUME(size <= SLAB_SIZE); - sizeclass_t sizeclass = size_to_sizeclass(size); - return small_alloc_inner(sizeclass, size); - } - - template - SNMALLOC_FAST_PATH CapPtr - small_alloc_inner(sizeclass_t sizeclass, size_t size) - { - SNMALLOC_ASSUME(sizeclass < NUM_SMALL_CLASSES); - auto& fl = small_fast_free_lists[sizeclass]; - if (likely(!fl.empty())) - { - stats().alloc_request(size); - stats().sizeclass_alloc(sizeclass); - auto p = fl.take(entropy); - if constexpr (zero_mem == YesZero) - { - pal_zero( - p, sizeclass_to_size(sizeclass)); - } - - // TODO: Should this be zeroing the next pointer? - return capptr_export(p.as_void()); - } - - if (likely(!has_messages())) - return small_alloc_next_free_list(sizeclass, size); - - return small_alloc_mq_slow(sizeclass, size); - } - - /** - * Slow path for handling message queue, before dealing with small - * allocation request. - */ - template - SNMALLOC_SLOW_PATH CapPtr - small_alloc_mq_slow(sizeclass_t sizeclass, size_t size) - { - handle_message_queue_inner(); - - return small_alloc_next_free_list(sizeclass, size); - } - - /** - * Attempt to find a new free list to allocate from - */ - template - SNMALLOC_SLOW_PATH CapPtr - small_alloc_next_free_list(sizeclass_t sizeclass, size_t size) - { - size_t rsize = sizeclass_to_size(sizeclass); - auto& sl = small_classes[sizeclass]; - - if (likely(!sl.is_empty())) - { - stats().alloc_request(size); - stats().sizeclass_alloc(sizeclass); - - auto meta = sl.get_next().template as_static(); - auto& ffl = small_fast_free_lists[sizeclass]; - return Metaslab::alloc( - meta, ffl, rsize, entropy); - } - return small_alloc_rare(sizeclass, size); - } - - /** - * Called when there are no available free list to service this request - * Could be due to using the dummy allocator, or needing to bump allocate a - * new free list. - */ - template - SNMALLOC_SLOW_PATH CapPtr - small_alloc_rare(sizeclass_t sizeclass, size_t size) - { - if (likely(!NeedsInitialisation(this))) - { - stats().alloc_request(size); - stats().sizeclass_alloc(sizeclass); - return small_alloc_new_free_list(sizeclass); - } - return small_alloc_first_alloc(sizeclass, size); - } - - /** - * Called on first allocation to set up the thread local allocator, - * then directs the allocation request to the newly created allocator. - */ - template - SNMALLOC_SLOW_PATH CapPtr - small_alloc_first_alloc(sizeclass_t sizeclass, size_t size) - { - /* - * We have to convert through void* as part of the thread allocator - * initializer API. Be a little more verbose than strictly necessary to - * demonstrate that small_alloc_inner is giving us a CBAllocE-annotated - * pointer before we just go slapping that label on a void* later. - */ - void* ret = InitThreadAllocator([sizeclass, size](void* alloc) { - CapPtr ret = - reinterpret_cast(alloc) - ->template small_alloc_inner(sizeclass, size); - return ret.unsafe_capptr; - }); - return CapPtr(ret); - } - - /** - * Called to create a new free list, and service the request from that new - * list. - */ - template - SNMALLOC_FAST_PATH CapPtr - small_alloc_new_free_list(sizeclass_t sizeclass) - { - auto& bp = bump_ptrs[sizeclass]; - if (likely(pointer_align_up(bp, SLAB_SIZE) != bp)) - { - return small_alloc_build_free_list(sizeclass); - } - // Fetch new slab - return small_alloc_new_slab(sizeclass); - } - - /** - * Creates a new free list from the thread local bump allocator and service - * the request from that new list. - */ - template - SNMALLOC_FAST_PATH CapPtr - small_alloc_build_free_list(sizeclass_t sizeclass) - { - auto& bp = bump_ptrs[sizeclass]; - auto rsize = sizeclass_to_size(sizeclass); - auto& ffl = small_fast_free_lists[sizeclass]; - SNMALLOC_ASSERT(ffl.empty()); - Slab::alloc_new_list(bp, ffl, rsize, entropy); - - auto p = ffl.take(entropy); - - if constexpr (zero_mem == YesZero) - { - pal_zero(p, sizeclass_to_size(sizeclass)); - } - - // TODO: Should this be zeroing the next pointer? - return capptr_export(p.as_void()); - } - - /** - * Allocates a new slab to allocate from, set it to be the bump allocator - * for this size class, and then builds a new free list from the thread - * local bump allocator and service the request from that new list. - */ - template - SNMALLOC_SLOW_PATH CapPtr - small_alloc_new_slab(sizeclass_t sizeclass) - { - auto& bp = bump_ptrs[sizeclass]; - // Fetch new slab - auto slab = alloc_slab(sizeclass); - if (slab == nullptr) - return nullptr; - bp = pointer_offset( - slab, get_initial_offset(sizeclass, Metaslab::is_short(slab))); - - return small_alloc_build_free_list(sizeclass); - } - - SNMALLOC_FAST_PATH void small_dealloc_unchecked( - CapPtr super, - CapPtr p_auth, - CapPtr p_ret, - sizeclass_t sizeclass) - { - check_client( - chunkmap().get(address_cast(p_ret)) == CMSuperslab, - "Claimed small deallocation is not in a Superslab"); - - small_dealloc_checked_chunkmap(super, p_auth, p_ret, sizeclass); - } - - SNMALLOC_FAST_PATH void small_dealloc_checked_chunkmap( - CapPtr super, - CapPtr p_auth, - CapPtr p_ret, - sizeclass_t sizeclass) - { - auto slab = Metaslab::get_slab(p_auth); - check_client( - sizeclass == super->get_meta(slab)->sizeclass(), - "Claimed small deallocation with mismatching size class"); - - small_dealloc_checked_sizeclass(super, slab, p_auth, p_ret, sizeclass); - } - - SNMALLOC_FAST_PATH void small_dealloc_checked_sizeclass( - CapPtr super, - CapPtr slab, - CapPtr p_auth, - CapPtr p_ret, - sizeclass_t sizeclass) - { - check_client( - Slab::get_meta(slab)->is_start_of_object(address_cast(p_ret)), - "Not deallocating start of an object"); - - small_dealloc_start(super, slab, p_auth, p_ret, sizeclass); - } - - SNMALLOC_FAST_PATH void small_dealloc_start( - CapPtr super, - CapPtr slab, - CapPtr p_auth, - CapPtr p_ret, - sizeclass_t sizeclass) - { - // TODO: with SSM/MTE, guard against double-frees - UNUSED(p_ret); - - RemoteAllocator* target = super->get_allocator(); - - auto p = - Aal::capptr_bound(p_auth, sizeclass_to_size(sizeclass)); - - if (likely(target == public_state())) - { - small_dealloc_offseted(super, slab, p, sizeclass); - } - else - remote_dealloc(target, p, sizeclass); - } - - SNMALLOC_FAST_PATH void small_dealloc_offseted( - CapPtr super, - CapPtr slab, - CapPtr p, - sizeclass_t sizeclass) - { - stats().sizeclass_dealloc(sizeclass); - - small_dealloc_offseted_inner(super, slab, FreeObject::make(p), sizeclass); - } - - SNMALLOC_FAST_PATH void small_dealloc_offseted_inner( - CapPtr super, - CapPtr slab, - CapPtr p, - sizeclass_t sizeclass) - { - if (likely(Slab::dealloc_fast(slab, super, p, entropy))) - return; - - small_dealloc_offseted_slow(super, slab, p, sizeclass); - } - - SNMALLOC_SLOW_PATH void small_dealloc_offseted_slow( - CapPtr super, - CapPtr slab, - CapPtr p, - sizeclass_t sizeclass) - { - bool was_full = super->is_full(); - SlabList* sl = &small_classes[sizeclass]; - Superslab::Action a = Slab::dealloc_slow(slab, sl, super, p, entropy); - if (likely(a == Superslab::NoSlabReturn)) - return; - stats().sizeclass_dealloc_slab(sizeclass); - - if (a == Superslab::NoStatusChange) - return; - - auto super_slab = capptr_chunk_from_chunkd(super, SUPERSLAB_SIZE); - - switch (super->get_status()) - { - case Superslab::Full: - { - error("Unreachable"); - break; - } - - case Superslab::Available: - { - if (was_full) - { - super_available.insert(super_slab); - } - else - { - super_only_short_available.remove(super_slab); - super_available.insert(super_slab); - } - break; - } - - case Superslab::OnlyShortSlabAvailable: - { - super_only_short_available.insert(super_slab); - break; - } - - case Superslab::Empty: - { - super_available.remove(super_slab); - - chunkmap().clear_slab(super_slab); - large_allocator.dealloc( - super_slab.template as_reinterpret(), 0); - stats().superslab_push(); - break; - } - } - } - - template - CapPtr - medium_alloc(sizeclass_t sizeclass, size_t rsize, size_t size) - { - sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES; - - auto sc = &medium_classes[medium_class]; - CapPtr slab = sc->get_head(); - CapPtr p; - - if (slab != nullptr) - { - p = Mediumslab::alloc( - slab, rsize); - - if (Mediumslab::full(slab)) - sc->pop(); - } - else - { - if (NeedsInitialisation(this)) - { - /* - * We have to convert through void* as part of the thread allocator - * initializer API. Be a little more verbose than strictly necessary - * to demonstrate that small_alloc_inner is giving us an annotated - * pointer before we just go slapping that label on a void* later. - */ - void* ret = - InitThreadAllocator([size, rsize, sizeclass](void* alloc) { - CapPtr ret = - reinterpret_cast(alloc)->medium_alloc( - sizeclass, rsize, size); - return ret.unsafe_capptr; - }); - return CapPtr(ret); - } - - auto newslab = - large_allocator - .template alloc(0, SUPERSLAB_SIZE, SUPERSLAB_SIZE) - .template as_reinterpret(); - - if (newslab == nullptr) - return nullptr; - - Mediumslab::init(newslab, public_state(), sizeclass, rsize); - chunkmap().set_slab(newslab); - - auto newslab_export = capptr_export(newslab); - - p = Mediumslab::alloc( - newslab_export, rsize); - - if (!Mediumslab::full(newslab)) - sc->insert(newslab_export); - } - - stats().alloc_request(size); - stats().sizeclass_alloc(sizeclass); - - return p; - } - - SNMALLOC_FAST_PATH - void medium_dealloc_unchecked( - CapPtr slab, - CapPtr p_auth, - CapPtr p_ret, - sizeclass_t sizeclass) - { - check_client( - chunkmap().get(address_cast(p_ret)) == CMMediumslab, - "Claimed medium deallocation is not in a Mediumslab"); - - medium_dealloc_checked_chunkmap(slab, p_auth, p_ret, sizeclass); - } - - SNMALLOC_FAST_PATH - void medium_dealloc_checked_chunkmap( - CapPtr slab, - CapPtr p_auth, - CapPtr p_ret, - sizeclass_t sizeclass) - { - check_client( - slab->get_sizeclass() == sizeclass, - "Claimed medium deallocation of the wrong sizeclass"); - - medium_dealloc_checked_sizeclass(slab, p_auth, p_ret, sizeclass); - } - - SNMALLOC_FAST_PATH - void medium_dealloc_checked_sizeclass( - CapPtr slab, - CapPtr p_auth, - CapPtr p_ret, - sizeclass_t sizeclass) - { - check_client( - is_multiple_of_sizeclass( - sizeclass, address_cast(slab) + SUPERSLAB_SIZE - address_cast(p_ret)), - "Not deallocating start of an object"); - - medium_dealloc_start(slab, p_auth, p_ret, sizeclass); - } - - SNMALLOC_FAST_PATH - void medium_dealloc_start( - CapPtr slab, - CapPtr p_auth, - CapPtr p_ret, - sizeclass_t sizeclass) - { - // TODO: with SSM/MTE, guard against double-frees - UNUSED(p_ret); - - RemoteAllocator* target = slab->get_allocator(); - - // TODO: This bound is perhaps superfluous in the local case, as - // mediumslabs store free objects by offset rather than pointer. - auto p = - Aal::capptr_bound(p_auth, sizeclass_to_size(sizeclass)); - - if (likely(target == public_state())) - medium_dealloc_local(slab, p, sizeclass); - else - { - remote_dealloc(target, p, sizeclass); - } - } - - SNMALLOC_FAST_PATH - void medium_dealloc_local( - CapPtr slab, - CapPtr p, - sizeclass_t sizeclass) - { - stats().sizeclass_dealloc(sizeclass); - bool was_full = Mediumslab::dealloc(slab, p); - - auto slab_bounded = capptr_chunk_from_chunkd(slab, SUPERSLAB_SIZE); - - if (Mediumslab::empty(slab)) - { - if (!was_full) - { - sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES; - auto sc = &medium_classes[medium_class]; - /* - * This unsafety lets us avoid applying platform constraints to a - * pointer we are just about to drop on the floor; remove() uses its - * argument but does not persist it. - */ - sc->remove(CapPtr(slab_bounded.unsafe_capptr)); - } - - chunkmap().clear_slab(slab_bounded); - large_allocator.dealloc( - slab_bounded.template as_reinterpret(), 0); - stats().superslab_push(); - } - else if (was_full) - { - sizeclass_t medium_class = sizeclass - NUM_SMALL_CLASSES; - auto sc = &medium_classes[medium_class]; - sc->insert(capptr_export(slab_bounded)); - } - } - - template - CapPtr large_alloc(size_t size) - { - if (NeedsInitialisation(this)) - { - // MSVC-vs-CapPtr triggering; xref CapPtr's constructor - void* ret = InitThreadAllocator([size](void* alloc) { - CapPtr ret = - reinterpret_cast(alloc)->large_alloc(size); - return ret.unsafe_capptr; - }); - return CapPtr(ret); - } - - size_t size_bits = bits::next_pow2_bits(size); - size_t large_class = size_bits - SUPERSLAB_BITS; - SNMALLOC_ASSERT(large_class < NUM_LARGE_CLASSES); - - size_t rsize = bits::one_at_bit(SUPERSLAB_BITS) << large_class; - // For superslab size, we always commit the whole range. - if (large_class == 0) - size = rsize; - - CapPtr p = - large_allocator.template alloc(large_class, rsize, size); - if (likely(p != nullptr)) - { - chunkmap().set_large_size(p, size); - - stats().alloc_request(size); - stats().large_alloc(large_class); - } - return capptr_export(Aal::capptr_bound(p, rsize)); - } - - void large_dealloc_unchecked( - CapPtr p_auth, CapPtr p_ret, size_t size) - { - uint8_t claimed_chunkmap_slab_kind = - static_cast(bits::next_pow2_bits(size)); - - // This also catches some "not deallocating start of an object" cases: if - // we're so far from the start that our actual chunkmap slab kind is not a - // legitimate large class - check_client( - chunkmap().get(address_cast(p_ret)) == claimed_chunkmap_slab_kind, - "Claimed large deallocation with wrong size class"); - - // round up as we would if we had had to look up the chunkmap_slab_kind - size_t rsize = bits::one_at_bit(claimed_chunkmap_slab_kind); - - large_dealloc_checked_sizeclass( - p_auth, p_ret, rsize, claimed_chunkmap_slab_kind); - } - - void large_dealloc_checked_sizeclass( - CapPtr p_auth, - CapPtr p_ret, - size_t size, - uint8_t chunkmap_slab_kind) - { - check_client( - address_cast(Superslab::get(p_auth)) == address_cast(p_ret), - "Not deallocating start of an object"); - SNMALLOC_ASSERT(bits::one_at_bit(chunkmap_slab_kind) >= SUPERSLAB_SIZE); - - large_dealloc_start(p_auth, p_ret, size, chunkmap_slab_kind); - } - - void large_dealloc_start( - CapPtr p_auth, - CapPtr p_ret, - size_t size, - uint8_t chunkmap_slab_kind) - { - // TODO: with SSM/MTE, guard against double-frees - - if (NeedsInitialisation(this)) - { - InitThreadAllocator( - [p_auth, p_ret, size, chunkmap_slab_kind](void* alloc) { - reinterpret_cast(alloc)->large_dealloc_start( - p_auth, p_ret, size, chunkmap_slab_kind); - return nullptr; - }); - return; - } - - size_t large_class = chunkmap_slab_kind - SUPERSLAB_BITS; - auto slab = Aal::capptr_bound(p_auth, size); - - chunkmap().clear_large_size(slab, size); - - stats().large_dealloc(large_class); - - // Initialise in order to set the correct SlabKind. - slab->init(); - large_allocator.dealloc(slab, large_class); - } - - // This is still considered the fast path as all the complex code is tail - // called in its slow path. This leads to one fewer unconditional jump in - // Clang. - SNMALLOC_FAST_PATH - void remote_dealloc( - RemoteAllocator* target, CapPtr p, sizeclass_t sizeclass) - { - SNMALLOC_ASSERT(target->trunc_id() != get_trunc_id()); - - // Check whether this will overflow the cache first. If we are a fake - // allocator, then our cache will always be full and so we will never hit - // this path. - if (remote_cache.capacity > 0) - { - stats().remote_free(sizeclass); - remote_cache.dealloc(target->trunc_id(), p, sizeclass); - return; - } - - remote_dealloc_slow(target, p, sizeclass); - } - - SNMALLOC_SLOW_PATH void remote_dealloc_slow( - RemoteAllocator* target, - CapPtr p_auth, - sizeclass_t sizeclass) - { - SNMALLOC_ASSERT(target->trunc_id() != get_trunc_id()); - - // Now that we've established that we're in the slow path (if we're a - // real allocator, we will have to empty our cache now), check if we are - // a real allocator and construct one if we aren't. - if (NeedsInitialisation(this)) - { - InitThreadAllocator([target, p_auth, sizeclass](void* alloc) { - reinterpret_cast(alloc)->dealloc_not_large( - target, p_auth, sizeclass); - return nullptr; - }); - return; - } - - remote_dealloc_and_post(target, p_auth, sizeclass); - } - - SNMALLOC_SLOW_PATH void remote_dealloc_and_post( - RemoteAllocator* target, - CapPtr p_auth, - sizeclass_t sizeclass) - { - handle_message_queue(); - - stats().remote_free(sizeclass); - remote_cache.dealloc(target->trunc_id(), p_auth, sizeclass); - - stats().remote_post(); - remote_cache.post(this, get_trunc_id()); - } - - ChunkMap& chunkmap() - { - return chunk_map; - } - }; -} // namespace snmalloc diff --git a/src/mem/allocconfig.h b/src/mem/allocconfig.h index 2b25512fa..153e05bf9 100644 --- a/src/mem/allocconfig.h +++ b/src/mem/allocconfig.h @@ -1,31 +1,10 @@ #pragma once #include "../ds/bits.h" +#include "../pal/pal.h" namespace snmalloc { -// The CHECK_CLIENT macro is used to turn on minimal checking of the client -// calling the API correctly. -#if !defined(NDEBUG) && !defined(CHECK_CLIENT) -# define CHECK_CLIENT -#endif - - SNMALLOC_FAST_PATH void check_client_impl(bool test, const char* const str) - { -#ifdef CHECK_CLIENT - if (unlikely(!test)) - error(str); -#else - UNUSED(test); - UNUSED(str); -#endif - } -#ifdef CHECK_CLIENT -# define check_client(test, str) check_client_impl(test, str) -#else -# define check_client(test, str) -#endif - // 0 intermediate bits results in power of 2 small allocs. 1 intermediate // bit gives additional sizeclasses at the midpoint between each power of 2. // 2 intermediate bits gives 3 intermediate sizeclasses, etc. @@ -55,26 +34,6 @@ namespace snmalloc #endif ; - // Specifies smaller slab and super slab sizes for address space - // constrained scenarios. - static constexpr size_t USE_LARGE_CHUNKS = -#ifdef SNMALLOC_USE_LARGE_CHUNKS - // In 32 bit uses smaller superslab. - (bits::is64()) -#else - false -#endif - ; - - // Specifies even smaller slab and super slab sizes for open enclave. - static constexpr size_t USE_SMALL_CHUNKS = -#ifdef SNMALLOC_USE_SMALL_CHUNKS - true -#else - false -#endif - ; - enum DecommitStrategy { /** @@ -116,26 +75,24 @@ namespace snmalloc static constexpr size_t MIN_ALLOC_SIZE = 2 * sizeof(void*); static constexpr size_t MIN_ALLOC_BITS = bits::ctz_const(MIN_ALLOC_SIZE); - // Slabs are 64 KiB unless constrained to 16 or even 8 KiB - static constexpr size_t SLAB_BITS = - USE_SMALL_CHUNKS ? 13 : (USE_LARGE_CHUNKS ? 16 : 14); - static constexpr size_t SLAB_SIZE = 1 << SLAB_BITS; - static constexpr size_t SLAB_MASK = ~(SLAB_SIZE - 1); + // Minimum slab size. + static constexpr size_t MIN_CHUNK_BITS = 14; + static constexpr size_t MIN_CHUNK_SIZE = bits::one_at_bit(MIN_CHUNK_BITS); - // Superslabs are composed of this many slabs. Slab offsets are encoded as - // a byte, so the maximum count is 256. This must be a power of two to - // allow fast masking to find a superslab start address. - static constexpr size_t SLAB_COUNT_BITS = - USE_SMALL_CHUNKS ? 5 : (USE_LARGE_CHUNKS ? 8 : 6); - static constexpr size_t SLAB_COUNT = 1 << SLAB_COUNT_BITS; - static constexpr size_t SUPERSLAB_SIZE = SLAB_SIZE * SLAB_COUNT; - static constexpr size_t SUPERSLAB_MASK = ~(SUPERSLAB_SIZE - 1); - static constexpr size_t SUPERSLAB_BITS = SLAB_BITS + SLAB_COUNT_BITS; + // Minimum number of objects on a slab +#ifdef CHECK_CLIENT + static constexpr size_t MIN_OBJECT_COUNT = 13; +#else + static constexpr size_t MIN_OBJECT_COUNT = 4; +#endif - static_assert((1ULL << SUPERSLAB_BITS) == SUPERSLAB_SIZE, "Sanity check"); + // Maximum size of an object that uses sizeclasses. + static constexpr size_t MAX_SIZECLASS_BITS = 16; + static constexpr size_t MAX_SIZECLASS_SIZE = + bits::one_at_bit(MAX_SIZECLASS_BITS); // Number of slots for remote deallocation. - static constexpr size_t REMOTE_SLOT_BITS = 6; + static constexpr size_t REMOTE_SLOT_BITS = 8; static constexpr size_t REMOTE_SLOTS = 1 << REMOTE_SLOT_BITS; static constexpr size_t REMOTE_MASK = REMOTE_SLOTS - 1; @@ -145,12 +102,4 @@ namespace snmalloc static_assert( MIN_ALLOC_SIZE >= (sizeof(void*) * 2), "MIN_ALLOC_SIZE must be sufficient for two pointers"); - static_assert( - SLAB_BITS <= (sizeof(uint16_t) * 8), - "SLAB_BITS must not be more than the bits in a uint16_t"); - static_assert( - SLAB_COUNT == bits::next_pow2_const(SLAB_COUNT), - "SLAB_COUNT must be a power of 2"); - static_assert( - SLAB_COUNT <= (UINT8_MAX + 1), "SLAB_COUNT must fit in a uint8_t"); } // namespace snmalloc diff --git a/src/mem/allocslab.h b/src/mem/allocslab.h deleted file mode 100644 index 579da85d0..000000000 --- a/src/mem/allocslab.h +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include "baseslab.h" - -namespace snmalloc -{ - struct RemoteAllocator; - - class Allocslab : public Baseslab - { - protected: - RemoteAllocator* allocator; - - public: - RemoteAllocator* get_allocator() - { - return allocator; - } - }; -} // namespace snmalloc diff --git a/src/mem/allocstats.h b/src/mem/allocstats.h index 488493f9b..937a3a945 100644 --- a/src/mem/allocstats.h +++ b/src/mem/allocstats.h @@ -1,13 +1,12 @@ #pragma once #include "../ds/bits.h" -#include "../mem/sizeclass.h" +#include "../mem/sizeclasstable.h" #include #ifdef USE_SNMALLOC_STATS # include "../ds/csv.h" -# include "sizeclass.h" # include # include @@ -18,11 +17,13 @@ namespace snmalloc template struct AllocStats { + constexpr AllocStats() = default; + struct CurrentMaxPair { - size_t current = 0; - size_t max = 0; - size_t used = 0; + size_t current{0}; + size_t max{0}; + size_t used{0}; void inc() { @@ -34,7 +35,9 @@ namespace snmalloc void dec() { - SNMALLOC_ASSERT(current > 0); + // Split stats means this is not true. + // TODO reestablish checks, when we sanitise the stats. + // SNMALLOC_ASSERT(current > 0); current--; } @@ -64,11 +67,13 @@ namespace snmalloc struct Stats { + constexpr Stats() = default; + CurrentMaxPair count; CurrentMaxPair slab_count; - uint64_t time = Aal::tick(); - uint64_t ticks = 0; - double online_average = 0; + uint64_t time{0}; + uint64_t ticks{0}; + double online_average{0}; bool is_empty() { @@ -413,5 +418,13 @@ namespace snmalloc << csv.endl; } #endif + + void start() + { +#ifdef USE_SNMALLOC_STATS + for (size_t i = 0; i < N; i++) + sizeclass[i].time = Aal::tick(); +#endif + } }; } // namespace snmalloc diff --git a/src/mem/arenamap.h b/src/mem/arenamap.h deleted file mode 100644 index bf12816ad..000000000 --- a/src/mem/arenamap.h +++ /dev/null @@ -1,130 +0,0 @@ -#include "../ds/ptrwrap.h" -#include "pagemap.h" - -namespace snmalloc -{ - struct default_alloc_size_t - { - /* - * Just make something up for non-StrictProvenance architectures. - * Ultimately, this is going to flow only to FlatPagemap's template argument - * for the number of bits it's covering but the whole thing will be - * discarded by the time we resolve all the conditionals behind the - * AuthPagemap type. To avoid pathologies where COVERED_BITS ends up being - * bit-width of the machine (meaning 1ULL << COVERED_BITS becomes undefined) - * and where sizeof(std::atomic[ENTRIES]) is either undefined or - * enormous, we choose a value that dodges both endpoints and still results - * in a small table. - */ - static constexpr size_t capptr_root_alloc_size = - bits::one_at_bit(bits::ADDRESS_BITS - 8); - }; - - /* - * Compute the block allocation size to use for AlignedAllocations. This - * is either PAL::capptr_root_alloc_size, on architectures that require - * StrictProvenance, or the placeholder from above. - */ - template - static constexpr size_t AUTHMAP_ALLOC_SIZE = std::conditional_t< - aal_supports, - PAL, - default_alloc_size_t>::capptr_root_alloc_size; - - template - static constexpr size_t - AUTHMAP_BITS = bits::next_pow2_bits_const(AUTHMAP_ALLOC_SIZE); - - template - static constexpr bool - AUTHMAP_USE_FLATPAGEMAP = pal_supports || - (PAGEMAP_NODE_SIZE >= sizeof(FlatPagemap, void*>)); - - struct default_auth_pagemap - { - static SNMALLOC_FAST_PATH void* get(address_t a) - { - UNUSED(a); - return nullptr; - } - }; - - template - using AuthPagemap = std::conditional_t< - aal_supports, - std::conditional_t< - AUTHMAP_USE_FLATPAGEMAP, - FlatPagemap, void*>, - Pagemap, void*, nullptr, PrimAlloc>>, - default_auth_pagemap>; - - struct ForAuthmap - {}; - template - using GlobalAuthmap = - GlobalPagemapTemplate, ForAuthmap>; - - template - struct DefaultArenaMapTemplate - { - /* - * Without AlignedAllocation, we (below) adopt a fallback mechanism that - * over-allocates and then finds an aligned region within the too-large - * region. The "trimmings" from either side are also registered in hopes - * that they can be used for later allocations. - * - * Unfortunately, that strategy does not work for this ArenaMap: trimmings - * may be smaller than the granularity of our backing PageMap, and so we - * would be unable to amplify authority. Eventually we may arrive at a need - * for an ArenaMap that is compatible with this approach, but for the moment - * it's far simpler to assume that we can always ask for memory sufficiently - * aligned to cover an entire PageMap granule. - */ - static_assert( - !aal_supports || pal_supports, - "StrictProvenance requires platform support for aligned allocation"); - - static constexpr size_t alloc_size = AUTHMAP_ALLOC_SIZE; - - /* - * Because we assume that we can `capptr_amplify` and then - * `Superslab::get()` on the result to get to the Superslab metadata - * headers, it must be the case that provenance roots cover entire - * Superslabs. - */ - static_assert( - !aal_supports || - ((alloc_size > 0) && (alloc_size % SUPERSLAB_SIZE == 0)), - "Provenance root granule must encompass whole superslabs"); - - static void register_root(CapPtr root) - { - if constexpr (aal_supports) - { - PagemapProvider::pagemap().set(address_cast(root), root.unsafe_capptr); - } - else - { - UNUSED(root); - } - } - - template - static SNMALLOC_FAST_PATH CapPtr capptr_amplify(CapPtr r) - { - static_assert( - B == CBAllocE || B == CBAlloc, - "Attempting to amplify an unexpectedly high pointer"); - return Aal::capptr_rebound( - CapPtr( - PagemapProvider::pagemap().get(address_cast(r))), - r) - .template as_static(); - } - }; - - template - using DefaultArenaMap = - DefaultArenaMapTemplate>; - -} // namespace snmalloc diff --git a/src/mem/baseslab.h b/src/mem/baseslab.h deleted file mode 100644 index 9ca661c0b..000000000 --- a/src/mem/baseslab.h +++ /dev/null @@ -1,32 +0,0 @@ -#pragma once - -#include "../ds/mpmcstack.h" -#include "allocconfig.h" - -namespace snmalloc -{ - enum SlabKind - { - Fresh = 0, - Large, - Medium, - Super, - /** - * If the decommit policy is lazy, slabs are moved to this state when all - * pages other than the first one have been decommitted. - */ - Decommitted - }; - - class Baseslab - { - protected: - SlabKind kind; - - public: - SlabKind get_kind() - { - return kind; - } - }; -} // namespace snmalloc diff --git a/src/mem/chunkmap.h b/src/mem/chunkmap.h deleted file mode 100644 index 90175583c..000000000 --- a/src/mem/chunkmap.h +++ /dev/null @@ -1,195 +0,0 @@ -#pragma once - -using namespace std; - -#include "../ds/address.h" -#include "largealloc.h" -#include "mediumslab.h" -#include "pagemap.h" -#include "slab.h" - -namespace snmalloc -{ - enum ChunkMapSuperslabKind : uint8_t - { - CMNotOurs = 0, - CMSuperslab = 1, - CMMediumslab = 2, - - /* - * Values 3 (inclusive) through SUPERSLAB_BITS (exclusive) are as yet - * unused. - * - * Values SUPERSLAB_BITS (inclusive) through 64 (exclusive, as it would - * represent the entire address space) are used for log2(size) at the - * heads of large allocations. See SuperslabMap::set_large_size. - */ - CMLargeMin = SUPERSLAB_BITS, - CMLargeMax = 63, - - /* - * Values 64 (inclusive) through 64 + SUPERSLAB_BITS (exclusive) are unused - */ - - /* - * Values 64 + SUPERSLAB_BITS (inclusive) through 128 (exclusive) are used - * for entries within a large allocation. A value of x at pagemap entry p - * indicates that there are at least 2^(x-64) (inclusive) and at most - * 2^(x+1-64) (exclusive) page map entries between p and the start of the - * allocation. See ChunkMap::set_large_size and external_address's - * handling of large reallocation redirections. - */ - CMLargeRangeMin = 64 + SUPERSLAB_BITS, - CMLargeRangeMax = 127, - - /* - * Values 128 (inclusive) through 255 (inclusive) are as yet unused. - */ - - }; - - /* - * Ensure that ChunkMapSuperslabKind values are actually disjoint, i.e., - * that large allocations don't land on CMMediumslab. - */ - static_assert( - SUPERSLAB_BITS > CMMediumslab, "Large allocations may be too small"); - -#ifndef SNMALLOC_MAX_FLATPAGEMAP_SIZE -/* - * Unless otherwise specified, use a flat pagemap for the chunkmap (1 byte per - * Superslab-sized and -aligned region of the address space) if either of the - * following hold: - * - * - the platform supports LazyCommit and the flat structure would occupy 256 - * MiB or less. 256 MiB is more than adequate for 32-bit architectures and - * is the size of the flat pagemap for a 48-bit AS with the default chunk - * size or the USE_LARGE_CHUNKS chunksize (that is, configurations other - * than USE_SMALL_CHUNKS). - * - * - the platform does not support LazyCommit but the flat structure would - * occupy less than PAGEMAP_NODE_SIZE (i.e., the backing store for an - * internal tree node in the non-flat pagemap). - */ -# define SNMALLOC_MAX_FLATPAGEMAP_SIZE \ - (pal_supports ? 256ULL * 1024 * 1024 : PAGEMAP_NODE_SIZE) -#endif - static constexpr bool CHUNKMAP_USE_FLATPAGEMAP = - SNMALLOC_MAX_FLATPAGEMAP_SIZE >= - sizeof(FlatPagemap); - - using ChunkmapPagemap = std::conditional_t< - CHUNKMAP_USE_FLATPAGEMAP, - FlatPagemap, - Pagemap>; - - struct ForChunkmap - {}; - using GlobalChunkmap = GlobalPagemapTemplate; - - /** - * Optionally exported function that accesses the global chunkmap pagemap - * provided by a shared library. - */ - extern "C" void* - snmalloc_chunkmap_global_get(snmalloc::PagemapConfig const**); - - /** - * Class that defines an interface to the pagemap. This is provided to - * `Allocator` as a template argument and so can be replaced by a compatible - * implementation (for example, to move pagemap updates to a different - * protection domain). - */ - template - struct DefaultChunkMap - { - /** - * Get the pagemap entry corresponding to a specific address. - * - * Despite the type, the return value is an enum ChunkMapSuperslabKind - * or one of the reserved values described therewith. - */ - static uint8_t get(address_t p) - { - return PagemapProvider::pagemap().get(p); - } - - /** - * Set a pagemap entry indicating that there is a superslab at the - * specified index. - */ - static void set_slab(CapPtr slab) - { - set(address_cast(slab), static_cast(CMSuperslab)); - } - /** - * Add a pagemap entry indicating that a medium slab has been allocated. - */ - static void set_slab(CapPtr slab) - { - set(address_cast(slab), static_cast(CMMediumslab)); - } - /** - * Remove an entry from the pagemap corresponding to a superslab. - */ - static void clear_slab(CapPtr slab) - { - SNMALLOC_ASSERT(get(address_cast(slab)) == CMSuperslab); - set(address_cast(slab), static_cast(CMNotOurs)); - } - /** - * Remove an entry corresponding to a medium slab. - */ - static void clear_slab(CapPtr slab) - { - SNMALLOC_ASSERT(get(address_cast(slab)) == CMMediumslab); - set(address_cast(slab), static_cast(CMNotOurs)); - } - /** - * Update the pagemap to reflect a large allocation, of `size` bytes from - * address `p`. - */ - static void set_large_size(CapPtr p, size_t size) - { - size_t size_bits = bits::next_pow2_bits(size); - set(address_cast(p), static_cast(size_bits)); - // Set redirect slide - auto ss = address_cast(p) + SUPERSLAB_SIZE; - for (size_t i = 0; i < size_bits - SUPERSLAB_BITS; i++) - { - size_t run = bits::one_at_bit(i); - PagemapProvider::pagemap().set_range( - ss, static_cast(CMLargeRangeMin + i), run); - ss = ss + SUPERSLAB_SIZE * run; - } - } - /** - * Update the pagemap to remove a large allocation, of `size` bytes from - * address `p`. - */ - static void clear_large_size(CapPtr vp, size_t size) - { - auto p = address_cast(vp); - size_t rounded_size = bits::next_pow2(size); - SNMALLOC_ASSERT(get(p) == bits::next_pow2_bits(size)); - auto count = rounded_size >> SUPERSLAB_BITS; - PagemapProvider::pagemap().set_range(p, CMNotOurs, count); - } - - private: - /** - * Helper function to set a pagemap entry. This is not part of the public - * interface and exists to make it easy to reuse the code in the public - * methods in other pagemap adaptors. - */ - static void set(address_t p, uint8_t x) - { - PagemapProvider::pagemap().set(p, x); - } - }; - -#ifndef SNMALLOC_DEFAULT_CHUNKMAP -# define SNMALLOC_DEFAULT_CHUNKMAP snmalloc::DefaultChunkMap<> -#endif - -} // namespace snmalloc diff --git a/src/mem/commonconfig.h b/src/mem/commonconfig.h new file mode 100644 index 000000000..adfdb3395 --- /dev/null +++ b/src/mem/commonconfig.h @@ -0,0 +1,41 @@ +#pragma once + +#include "../ds/defines.h" +#include "remoteallocator.h" + +namespace snmalloc +{ + // Forward reference to thread local cleanup. + void register_clean_up(); + + class CommonConfig + { + public: + /** + * Special remote that should never be used as a real remote. + * This is used to initialise allocators that should always hit the + * remote path for deallocation. Hence moving a branch off the critical + * path. + */ + SNMALLOC_REQUIRE_CONSTINIT + inline static RemoteAllocator unused_remote; + + /** + * Special remote that is used in meta-data for large allocations. + * + * nullptr is considered a large allocations for this purpose to move + * of the critical path. + * + * Bottom bits of the remote pointer are used for a sizeclass, we need + * size bits to represent the non-large sizeclasses, we can then get + * the large sizeclass by having the fake large_remote considerably + * more aligned. + */ + SNMALLOC_REQUIRE_CONSTINIT + inline static constexpr RemoteAllocator* fake_large_remote{nullptr}; + + static_assert( + &unused_remote != fake_large_remote, + "Compilation should ensure these are different"); + }; +} // namespace snmalloc diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h new file mode 100644 index 000000000..6bf140f19 --- /dev/null +++ b/src/mem/corealloc.h @@ -0,0 +1,681 @@ +#pragma once + +#include "../ds/defines.h" +#include "allocconfig.h" +#include "localcache.h" +#include "metaslab.h" +#include "pooled.h" +#include "remotecache.h" +#include "sizeclasstable.h" +#include "slaballocator.h" + +#include + +namespace snmalloc +{ + template + class CoreAllocator : public Pooled> + { + template + friend class LocalAllocator; + + /** + * Per size class list of active slabs for this allocator. + */ + MetaslabCache alloc_classes[NUM_SIZECLASSES]; + + /** + * Local entropy source and current version of keys for + * this thread + */ + LocalEntropy entropy; + + /** + * Message queue for allocations being returned to this + * allocator + */ + std::conditional_t< + SharedStateHandle::IsQueueInline, + RemoteAllocator, + RemoteAllocator*> + remote_alloc; + + /** + * A local area of address space managed by this allocator. + * Used to reduce calls on the global address space. + */ + typename SharedStateHandle::Backend::LocalState backend_state; + + /** + * This is the thread local structure associated to this + * allocator. + */ + LocalCache* attached_cache; + + /** + * This contains the way to access all the global state and + * configuration for the system setup. + */ + SharedStateHandle handle; + + /** + * The message queue needs to be accessible from other threads + * + * In the cross trust domain version this is the minimum amount + * of allocator state that must be accessible to other threads. + */ + auto* public_state() + { + if constexpr (SharedStateHandle::IsQueueInline) + { + return &remote_alloc; + } + else + { + return remote_alloc; + } + } + + /** + * Return this allocator's "truncated" ID, an integer useful as a hash + * value of this allocator. + * + * Specifically, this is the address of this allocator's message queue + * with the least significant bits missing, masked by SIZECLASS_MASK. + * This will be unique for Allocs with inline queues; Allocs with + * out-of-line queues must ensure that no two queues' addresses collide + * under this masking. + */ + size_t get_trunc_id() + { + return public_state()->trunc_id(); + } + + /** + * Abstracts access to the message queue to handle different + * layout configurations of the allocator. + */ + auto& message_queue() + { + return public_state()->message_queue; + } + + /** + * The message queue has non-trivial initialisation as it needs to + * be non-empty, so we prime it with a single allocation. + */ + void init_message_queue() + { + // Manufacture an allocation to prime the queue + // Using an actual allocation removes a conditional from a critical path. + auto dummy = + CapPtr(small_alloc_one(sizeof(MIN_ALLOC_SIZE))) + .template as_static(); + if (dummy == nullptr) + { + error("Critical error: Out-of-memory during initialisation."); + } + message_queue().init(dummy); + } + + /** + * There are a few internal corner cases where we need to allocate + * a small object. These are not on the fast path, + * - Allocating stub in the message queue + * Note this is not performance critical as very infrequently called. + */ + void* small_alloc_one(size_t size) + { + SNMALLOC_ASSERT(attached_cache != nullptr); + // Use attached cache, and fill it if it is empty. + return attached_cache->template alloc( + size, [&](sizeclass_t sizeclass, FreeListIter* fl) { + return small_alloc(sizeclass, *fl); + }); + } + + static SNMALLOC_FAST_PATH void alloc_new_list( + CapPtr& bumpptr, + FreeListIter& fast_free_list, + size_t rsize, + size_t slab_size, + LocalEntropy& entropy) + { + auto slab_end = pointer_offset(bumpptr, slab_size + 1 - rsize); + + FreeListBuilder b; + SNMALLOC_ASSERT(b.empty()); + +#ifdef CHECK_CLIENT + // Structure to represent the temporary list elements + struct PreAllocObject + { + CapPtr next; + }; + // The following code implements Sattolo's algorithm for generating + // random cyclic permutations. This implementation is in the opposite + // direction, so that the original space does not need initialising. This + // is described as outside-in without citation on Wikipedia, appears to be + // Folklore algorithm. + + // Note the wide bounds on curr relative to each of the ->next fields; + // curr is not persisted once the list is built. + CapPtr curr = + pointer_offset(bumpptr, 0).template as_static(); + curr->next = Aal::capptr_bound(curr, rsize); + + uint16_t count = 1; + for (curr = + pointer_offset(curr, rsize).template as_static(); + curr.as_void() < slab_end; + curr = + pointer_offset(curr, rsize).template as_static()) + { + size_t insert_index = entropy.sample(count); + curr->next = std::exchange( + pointer_offset(bumpptr, insert_index * rsize) + .template as_static() + ->next, + Aal::capptr_bound(curr, rsize)); + count++; + } + + // Pick entry into space, and then build linked list by traversing cycle + // to the start. Use ->next to jump from CBArena to CBAlloc. + auto start_index = entropy.sample(count); + auto start_ptr = pointer_offset(bumpptr, start_index * rsize) + .template as_static() + ->next; + auto curr_ptr = start_ptr; + do + { + b.add(FreeObject::make(curr_ptr.as_void()), entropy); + curr_ptr = curr_ptr->next; + } while (curr_ptr != start_ptr); +#else + auto p = bumpptr; + do + { + b.add(Aal::capptr_bound(p, rsize), entropy); + p = pointer_offset(p, rsize); + } while (p < slab_end); +#endif + // This code consumes everything up to slab_end. + bumpptr = slab_end; + + SNMALLOC_ASSERT(!b.empty()); + b.close(fast_free_list, entropy); + } + + ChunkRecord* clear_slab(Metaslab* meta, sizeclass_t sizeclass) + { + FreeListIter fl; + meta->free_queue.close(fl, entropy); + void* p = finish_alloc_no_zero(fl.take(entropy), sizeclass); + +#ifdef CHECK_CLIENT + // Check free list is well-formed on platforms with + // integers as pointers. + size_t count = 1; // Already taken one above. + while (!fl.empty()) + { + fl.take(entropy); + count++; + } + // Check the list contains all the elements + SNMALLOC_ASSERT( + count == snmalloc::sizeclass_to_slab_object_count(sizeclass)); +#endif + ChunkRecord* chunk_record = reinterpret_cast(meta); + // TODO: This is a capability amplification as we are saying we + // have the whole chunk. + auto start_of_slab = pointer_align_down( + p, snmalloc::sizeclass_to_slab_size(sizeclass)); + // TODO Add bounds correctly here + chunk_record->chunk = CapPtr(start_of_slab); + +#ifdef SNMALLOC_TRACING + std::cout << "Slab " << start_of_slab << " is unused, Object sizeclass " + << sizeclass << std::endl; +#endif + return chunk_record; + } + + SNMALLOC_SLOW_PATH void dealloc_local_slabs(sizeclass_t sizeclass) + { + // Return unused slabs of sizeclass_t back to global allocator + SlabLink* prev = &alloc_classes[sizeclass]; + auto curr = prev->get_next(); + while (curr != nullptr) + { + auto nxt = curr->get_next(); + auto meta = reinterpret_cast(curr); + if (meta->needed() == 0) + { + prev->pop(); + alloc_classes[sizeclass].length--; + alloc_classes[sizeclass].unused--; + + // TODO delay the clear to the next user of the slab, or teardown so + // don't touch the cache lines at this point in check_client. + auto chunk_record = clear_slab(meta, sizeclass); + ChunkAllocator::dealloc( + handle, chunk_record, sizeclass_to_slab_sizeclass(sizeclass)); + } + else + { + prev = curr; + } + curr = nxt; + } + } + + /** + * Slow path for deallocating an object locally. + * This is either waking up a slab that was not actively being used + * by this thread, or handling the final deallocation onto a slab, + * so it can be reused by other threads. + */ + SNMALLOC_SLOW_PATH void dealloc_local_object_slow(const MetaEntry& entry) + { + // TODO: Handle message queue on this path? + + Metaslab* meta = entry.get_metaslab(); + sizeclass_t sizeclass = entry.get_sizeclass(); + + UNUSED(entropy); + if (meta->is_sleeping()) + { + // Slab has been woken up add this to the list of slabs with free space. + + // Wake slab up. + meta->set_not_sleeping(sizeclass); + + alloc_classes[sizeclass].insert(meta); + alloc_classes[sizeclass].length++; + +#ifdef SNMALLOC_TRACING + std::cout << "Slab is woken up" << std::endl; +#endif + + return; + } + + alloc_classes[sizeclass].unused++; + + // If we have several slabs, and it isn't too expensive as a proportion + // return to the global pool. + if ( + (alloc_classes[sizeclass].unused > 2) && + (alloc_classes[sizeclass].unused > + (alloc_classes[sizeclass].length >> 2))) + { + dealloc_local_slabs(sizeclass); + } + } + + /** + * Check if this allocator has messages to deallocate blocks from another + * thread + */ + SNMALLOC_FAST_PATH bool has_messages() + { + return !(message_queue().is_empty()); + } + + /** + * Process remote frees into this allocator. + */ + template + SNMALLOC_SLOW_PATH decltype(auto) + handle_message_queue_inner(Action action, Args... args) + { + bool need_post = false; + for (size_t i = 0; i < REMOTE_BATCH; i++) + { + auto p = message_queue().peek(); + auto& entry = SharedStateHandle::Backend::get_meta_data( + handle.get_backend_state(), snmalloc::address_cast(p)); + + auto r = message_queue().dequeue(); + + if (unlikely(!r.second)) + break; +#ifdef SNMALLOC_TRACING + std::cout << "Handling remote" << std::endl; +#endif + handle_dealloc_remote(entry, p, need_post); + } + + if (need_post) + { + post(); + } + + return action(args...); + } + + /** + * Dealloc a message either by putting for a forward, or + * deallocating locally. + * + * need_post will be set to true, if capacity is exceeded. + */ + void handle_dealloc_remote( + const MetaEntry& entry, CapPtr p, bool& need_post) + { + // TODO this needs to not double count stats + // TODO this needs to not double revoke if using MTE + // TODO thread capabilities? + + if (likely(entry.get_remote() == public_state())) + { + if (likely(dealloc_local_object_fast(entry, p.unsafe_ptr(), entropy))) + return; + + dealloc_local_object_slow(entry); + } + else + { + if ( + !need_post && + !attached_cache->remote_dealloc_cache.reserve_space(entry)) + need_post = true; + attached_cache->remote_dealloc_cache + .template dealloc( + entry.get_remote()->trunc_id(), p.as_void()); + } + } + + public: + CoreAllocator(LocalCache* cache, SharedStateHandle handle) + : attached_cache(cache), handle(handle) + { +#ifdef SNMALLOC_TRACING + std::cout << "Making an allocator." << std::endl; +#endif + // Entropy must be first, so that all data-structures can use the key + // it generates. + // This must occur before any freelists are constructed. + entropy.init(); + + // Ignoring stats for now. + // stats().start(); + + init_message_queue(); + message_queue().invariant(); + +#ifndef NDEBUG + for (sizeclass_t i = 0; i < NUM_SIZECLASSES; i++) + { + size_t size = sizeclass_to_size(i); + sizeclass_t sc1 = size_to_sizeclass(size); + sizeclass_t sc2 = size_to_sizeclass_const(size); + size_t size1 = sizeclass_to_size(sc1); + size_t size2 = sizeclass_to_size(sc2); + + SNMALLOC_ASSERT(sc1 == i); + SNMALLOC_ASSERT(sc1 == sc2); + SNMALLOC_ASSERT(size1 == size); + SNMALLOC_ASSERT(size1 == size2); + } +#endif + } + + /** + * Post deallocations onto other threads. + * + * Returns true if it actually performed a post, + * and false otherwise. + */ + SNMALLOC_FAST_PATH bool post() + { + // stats().remote_post(); // TODO queue not in line! + bool sent_something = + attached_cache->remote_dealloc_cache.post( + handle, public_state()->trunc_id()); + + return sent_something; + } + + template + SNMALLOC_FAST_PATH decltype(auto) + handle_message_queue(Action action, Args... args) + { + // Inline the empty check, but not necessarily the full queue handling. + if (likely(!has_messages())) + { + return action(args...); + } + + return handle_message_queue_inner(action, args...); + } + + SNMALLOC_FAST_PATH void dealloc_local_object(void* p) + { + auto entry = SharedStateHandle::Backend::get_meta_data( + handle.get_backend_state(), snmalloc::address_cast(p)); + if (likely(dealloc_local_object_fast(entry, p, entropy))) + return; + + dealloc_local_object_slow(entry); + } + + SNMALLOC_FAST_PATH static bool dealloc_local_object_fast( + const MetaEntry& entry, void* p, LocalEntropy& entropy) + { + auto meta = entry.get_metaslab(); + + SNMALLOC_ASSERT(!meta->is_unused()); + + check_client( + Metaslab::is_start_of_object(entry.get_sizeclass(), address_cast(p)), + "Not deallocating start of an object"); + + auto cp = CapPtr(reinterpret_cast(p)); + + // Update the head and the next pointer in the free list. + meta->free_queue.add(cp, entropy); + + return likely(!meta->return_object()); + } + + template + SNMALLOC_SLOW_PATH void* + small_alloc(sizeclass_t sizeclass, FreeListIter& fast_free_list) + { + size_t rsize = sizeclass_to_size(sizeclass); + + // Look to see if we can grab a free list. + auto& sl = alloc_classes[sizeclass]; + if (likely(!(sl.is_empty()))) + { + auto meta = reinterpret_cast(sl.pop()); + // Drop length of sl, and empty count if it was empty. + alloc_classes[sizeclass].length--; + if (meta->needed() == 0) + alloc_classes[sizeclass].unused--; + + auto p = Metaslab::alloc(meta, fast_free_list, entropy, sizeclass); + + return finish_alloc(p, sizeclass); + } + return small_alloc_slow(sizeclass, fast_free_list, rsize); + } + + template + SNMALLOC_SLOW_PATH void* small_alloc_slow( + sizeclass_t sizeclass, FreeListIter& fast_free_list, size_t rsize) + { + // No existing free list get a new slab. + size_t slab_size = sizeclass_to_slab_size(sizeclass); + size_t slab_sizeclass = sizeclass_to_slab_sizeclass(sizeclass); + +#ifdef SNMALLOC_TRACING + std::cout << "rsize " << rsize << std::endl; + std::cout << "slab size " << slab_size << std::endl; +#endif + + auto [slab, meta] = snmalloc::ChunkAllocator::alloc_chunk( + handle, + backend_state, + sizeclass, + slab_sizeclass, + slab_size, + public_state()); + + if (slab == nullptr) + { + return nullptr; + } + + // Build a free list for the slab + alloc_new_list(slab, fast_free_list, rsize, slab_size, entropy); + + // Set meta slab to empty. + meta->initialise(sizeclass); + + // take an allocation from the free list + auto p = fast_free_list.take(entropy); + + return finish_alloc(p, sizeclass); + } + + /** + * Flush the cached state and delayed deallocations + * + * Returns true if messages are sent to other threads. + */ + bool flush(bool destroy_queue = false) + { + SNMALLOC_ASSERT(attached_cache != nullptr); + + if (destroy_queue) + { + CapPtr p = message_queue().destroy(); + + while (p != nullptr) + { + bool need_post = true; // Always going to post, so ignore. + auto n = p->non_atomic_next; + auto& entry = SharedStateHandle::Backend::get_meta_data( + handle.get_backend_state(), snmalloc::address_cast(p)); + handle_dealloc_remote(entry, p, need_post); + p = n; + } + } + else + { + // Process incoming message queue + // Loop as normally only processes a batch + while (has_messages()) + handle_message_queue([]() {}); + } + + auto posted = attached_cache->flush( + [&](auto p) { dealloc_local_object(p); }, handle); + + // We may now have unused slabs, return to the global allocator. + for (sizeclass_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++) + { + dealloc_local_slabs(sizeclass); + } + + return posted; + } + + // This allows the caching layer to be attached to an underlying + // allocator instance. + void attach(LocalCache* c) + { +#ifdef SNMALLOC_TRACING + std::cout << "Attach cache to " << this << std::endl; +#endif + attached_cache = c; + + // Set up secrets. + c->entropy = entropy; + + // Set up remote allocator. + c->remote_allocator = public_state(); + + // Set up remote cache. + c->remote_dealloc_cache.init(); + } + + /** + * Performs the work of checking if empty under the assumption that + * a local cache has been attached. + */ + bool debug_is_empty_impl(bool* result) + { + auto test = [&result](auto& queue) { + if (!queue.is_empty()) + { + auto curr = reinterpret_cast(queue.get_next()); + while (curr != nullptr) + { + if (curr->needed() != 0) + { + if (result != nullptr) + *result = false; + else + error("debug_is_empty: found non-empty allocator"); + } + curr = reinterpret_cast(curr->get_next()); + } + } + }; + + bool sent_something = flush(true); + + for (auto& alloc_class : alloc_classes) + { + test(alloc_class); + } + + // Place the static stub message on the queue. + init_message_queue(); + +#ifdef SNMALLOC_TRACING + std::cout << "debug_is_empty - done" << std::endl; +#endif + return sent_something; + } + + /** + * If result parameter is non-null, then false is assigned into the + * the location pointed to by result if this allocator is non-empty. + * + * If result pointer is null, then this code raises a Pal::error on the + * particular check that fails, if any do fail. + * + * Do not run this while other thread could be deallocating as the + * message queue invariant is temporarily broken. + */ + bool debug_is_empty(bool* result) + { +#ifdef SNMALLOC_TRACING + std::cout << "debug_is_empty" << std::endl; +#endif + if (attached_cache == nullptr) + { + // We need a cache to perform some operations, so set one up + // temporarily + LocalCache temp(public_state()); + attach(&temp); +#ifdef SNMALLOC_TRACING + std::cout << "debug_is_empty - attach a cache" << std::endl; +#endif + auto sent_something = debug_is_empty_impl(result); + + // Remove cache from the allocator + flush(); + attached_cache = nullptr; + return sent_something; + } + + return debug_is_empty_impl(result); + } + }; +} // namespace snmalloc diff --git a/src/mem/entropy.h b/src/mem/entropy.h index fa1b4ec27..8654532e6 100644 --- a/src/mem/entropy.h +++ b/src/mem/entropy.h @@ -23,14 +23,16 @@ namespace snmalloc class LocalEntropy { - uint64_t bit_source; - uint64_t local_key; - uint64_t local_counter; - address_t constant_key; - uint64_t fresh_bits; - uint64_t count; + uint64_t bit_source{0}; + uint64_t local_key{0}; + uint64_t local_counter{0}; + address_t constant_key{0}; + uint64_t fresh_bits{0}; + uint64_t count{0}; public: + constexpr LocalEntropy() = default; + template void init() { diff --git a/src/mem/fixedglobalconfig.h b/src/mem/fixedglobalconfig.h new file mode 100644 index 000000000..146993d5b --- /dev/null +++ b/src/mem/fixedglobalconfig.h @@ -0,0 +1,78 @@ +#pragma once + +#include "../backend/backend.h" +#include "../mem/corealloc.h" +#include "../mem/pool.h" +#include "../mem/slaballocator.h" +#include "commonconfig.h" + +namespace snmalloc +{ + /** + * A single fixed address range allocator configuration + */ + class FixedGlobals : public CommonConfig + { + public: + using Backend = BackendAllocator, true>; + + private: + inline static Backend::GlobalState backend_state; + + inline static ChunkAllocatorState slab_allocator_state; + + inline static PoolState> alloc_pool; + + public: + static Backend::GlobalState& get_backend_state() + { + return backend_state; + } + + ChunkAllocatorState& get_slab_allocator_state() + { + return slab_allocator_state; + } + + PoolState>& pool() + { + return alloc_pool; + } + + static constexpr bool IsQueueInline = true; + + // Performs initialisation for this configuration + // of allocators. Will be called at most once + // before any other datastructures are accessed. + void ensure_init() noexcept + { +#ifdef SNMALLOC_TRACING + std::cout << "Run init_impl" << std::endl; +#endif + } + + static bool is_initialised() + { + return true; + } + + // This needs to be a forward reference as the + // thread local state will need to know about this. + // This may allocate, so must be called once a thread + // local allocator exists. + static void register_clean_up() + { + snmalloc::register_clean_up(); + } + + static void init(CapPtr base, size_t length) + { + get_backend_state().init(base, length); + } + + constexpr static FixedGlobals get_handle() + { + return {}; + } + }; +} diff --git a/src/mem/freelist.h b/src/mem/freelist.h index c8766c8cd..37809b418 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -5,9 +5,6 @@ */ #include "../ds/address.h" -#include "../ds/cdllist.h" -#include "../ds/dllist.h" -#include "../ds/helpers.h" #include "allocconfig.h" #include "entropy.h" @@ -15,36 +12,11 @@ namespace snmalloc { -#ifdef CHECK_CLIENT - static constexpr std::size_t PRESERVE_BOTTOM_BITS = 16; -#endif - - /** - * Used to turn a location into a key. This is currently - * just the slab address truncated to 16bits and offset by 1. - */ - template - inline static address_t initial_key(CapPtr slab) - { -#ifdef CHECK_CLIENT - /** - * This file assumes that SLAB_BITS is smaller than 16. In multiple - * places it uses uint16_t to represent the offset into a slab. - */ - static_assert( - SLAB_BITS <= 16, - "Encoding requires slab offset representable in 16bits."); - - return (address_cast(slab) & SLAB_MASK) + 1; -#else - UNUSED(slab); - return 0; -#endif - } + static constexpr std::size_t PRESERVE_BOTTOM_BITS = 30; static inline bool different_slab(address_t p1, address_t p2) { - return ((p1 ^ p2) >= SLAB_SIZE); + return (p1 ^ p2) >= bits::one_at_bit(PRESERVE_BOTTOM_BITS); } template @@ -83,7 +55,7 @@ namespace snmalloc #ifdef CHECK_CLIENT template static std::enable_if_t> encode( - uint16_t local_key, CapPtr next_object, LocalEntropy& entropy) + uint32_t local_key, CapPtr next_object, LocalEntropy& entropy) { // Simple involutional encoding. The bottom half of each word is // multiplied by a function of both global and local keys (the latter, @@ -93,16 +65,16 @@ namespace snmalloc auto next = address_cast(next_object); constexpr address_t MASK = bits::one_at_bit(PRESERVE_BOTTOM_BITS) - 1; // Mix in local_key - address_t key = (local_key + 1) * entropy.get_constant_key(); - next ^= (((next & MASK) + 1) * key) & - ~(bits::one_at_bit(PRESERVE_BOTTOM_BITS) - 1); + address_t p1 = local_key + entropy.get_constant_key(); + address_t p2 = (next & MASK) - entropy.get_constant_key(); + next ^= (p1 * p2) & ~MASK; return CapPtr(reinterpret_cast(next)); } #endif template static std::enable_if_t> encode( - uint16_t local_key, CapPtr next_object, LocalEntropy& entropy) + uint32_t local_key, CapPtr next_object, LocalEntropy& entropy) { UNUSED(local_key); UNUSED(entropy); @@ -111,13 +83,13 @@ namespace snmalloc void store( CapPtr value, - uint16_t local_key, + uint32_t local_key, LocalEntropy& entropy) { reference = encode(local_key, value, entropy); } - CapPtr read(uint16_t local_key, LocalEntropy& entropy) + CapPtr read(uint32_t local_key, LocalEntropy& entropy) { return encode(local_key, reference, entropy); } @@ -151,7 +123,7 @@ namespace snmalloc /** * Read the next pointer handling any required decoding of the pointer */ - CapPtr read_next(uint16_t key, LocalEntropy& entropy) + CapPtr read_next(uint32_t key, LocalEntropy& entropy) { return next_object.read(key, entropy); } @@ -164,61 +136,41 @@ namespace snmalloc */ class FreeListIter { - CapPtr curr = nullptr; + CapPtr curr{1}; #ifdef CHECK_CLIENT - address_t prev = 0; + address_t prev{0}; #endif - uint16_t get_prev() + uint32_t get_prev() { #ifdef CHECK_CLIENT - return prev & 0xffff; + return prev & 0xffff'ffff; #else return 0; #endif } - /** - * Updates the cursor to the new value, - * importantly this updates the key being used. - * Currently this is just the value of current before this call. - * Other schemes could be used. - */ - void update_cursor(CapPtr next) - { -#ifdef CHECK_CLIENT -# ifndef NDEBUG - if (next != nullptr) - { - check_client( - !different_slab(prev, next), - "Heap corruption - free list corrupted!"); - } -# endif - prev = address_cast(curr); -#endif - curr = next; - } - public: - FreeListIter(CapPtr head) + constexpr FreeListIter( + CapPtr head, address_t prev_value) : curr(head) #ifdef CHECK_CLIENT , - prev(initial_key(head)) + prev(prev_value) #endif { - SNMALLOC_ASSERT(head != nullptr); + // SNMALLOC_ASSERT(head != nullptr); + UNUSED(prev_value); } - FreeListIter() = default; + constexpr FreeListIter() = default; /** * Checks if there are any more values to iterate. */ bool empty() { - return curr == nullptr; + return (address_cast(curr) & 1) == 1; } /** @@ -234,12 +186,17 @@ namespace snmalloc */ CapPtr take(LocalEntropy& entropy) { + auto c = curr; + auto next = curr->read_next(get_prev(), entropy); + #ifdef CHECK_CLIENT check_client( - !different_slab(prev, curr), "Heap corruption - free list corrupted!"); + !different_slab(curr, next), "Heap corruption - free list corrupted!"); + prev = address_cast(curr); #endif - auto c = curr; - update_cursor(curr->read_next(get_prev(), entropy)); + curr = next; + + Aal::prefetch(next.unsafe_ptr()); return c; } }; @@ -247,20 +204,18 @@ namespace snmalloc /** * Used to build a free list in object space. * - * Adds signing of pointers + * Adds signing of pointers in the CHECK_CLIENT mode * - * On 64bit ptr architectures this data structure has - * 44 bytes of data - * and has an alignment of - * 8 bytes - * This unfortunately means its sizeof is 48bytes. We - * use the template parameter, so that an enclosing - * class can make use of the remaining four bytes. + * We use the template parameter, so that an enclosing + * class can make use of the remaining bytes, which may not + * be aligned. On 64bit ptr architectures, this structure + * is a multiple of 8 bytes in the checked and random more. + * But on 128bit ptr architectures this may be a benefit. * - * The builder uses two queues, and "randomly" decides to - * add to one of the two queues. This means that we will - * maintain a randomisation of the order between - * allocations. + * If RANDOM is enabled, the builder uses two queues, and + * "randomly" decides to add to one of the two queues. This + * means that we will maintain a randomisation of the order + * between allocations. * * The fields are paired up to give better codegen as then they are offset * by a power of 2, and the bit extract from the interleaving seed can @@ -281,17 +236,13 @@ namespace snmalloc // This enables branch free enqueuing. EncodeFreeObjectReference* end[LENGTH]; #ifdef CHECK_CLIENT - // The bottom 16 bits of the previous pointer - uint16_t prev[LENGTH]; - // The bottom 16 bits of the current pointer - // This needs to be stored for the empty case - // where it is `initial_key()` for the slab. - uint16_t curr[LENGTH]; + // The bottom 32 bits of the previous pointer + uint32_t prev[LENGTH]; #endif public: S s; - uint16_t get_prev(uint32_t index) + uint32_t get_prev(uint32_t index) { #ifdef CHECK_CLIENT return prev[index]; @@ -301,43 +252,24 @@ namespace snmalloc #endif } - uint16_t get_curr(uint32_t index) + uint32_t get_curr(uint32_t index) { #ifdef CHECK_CLIENT - return curr[index]; + return address_cast(end[index]) & 0xffff'ffff; #else UNUSED(index); return 0; #endif } - static constexpr uint16_t HEAD_KEY = 1; + static constexpr uint32_t HEAD_KEY = 1; public: - FreeListBuilder() + constexpr FreeListBuilder() { init(); } - /** - * Start building a new free list. - * Provide pointer to the slab to initialise the system. - */ - void open(CapPtr p) - { - SNMALLOC_ASSERT(empty()); - for (size_t i = 0; i < LENGTH; i++) - { -#ifdef CHECK_CLIENT - prev[i] = HEAD_KEY; - curr[i] = initial_key(p) & 0xffff; -#else - UNUSED(p); -#endif - end[i] = &head[i]; - } - } - /** * Checks if the builder contains any elements. */ @@ -368,14 +300,17 @@ namespace snmalloc { SNMALLOC_ASSERT(!debug_different_slab(n) || empty()); - auto index = RANDOM ? entropy.next_bit() : 0; + uint32_t index; + if constexpr (RANDOM) + index = entropy.next_bit(); + else + index = 0; end[index]->store(n, get_prev(index), entropy); - end[index] = &(n->next_object); #ifdef CHECK_CLIENT - prev[index] = curr[index]; - curr[index] = address_cast(n) & 0xffff; + prev[index] = get_curr(index); #endif + end[index] = &(n->next_object); } /** @@ -389,16 +324,17 @@ namespace snmalloc size_t count = 0; for (size_t i = 0; i < LENGTH; i++) { - uint16_t local_prev = HEAD_KEY; + uint32_t local_prev = HEAD_KEY; EncodeFreeObjectReference* iter = &head[i]; CapPtr prev_obj = iter->read(local_prev, entropy); - uint16_t local_curr = initial_key(prev_obj) & 0xffff; + UNUSED(prev_obj); + uint32_t local_curr = address_cast(&head[i]) & 0xffff'ffff; while (end[i] != iter) { CapPtr next = iter->read(local_prev, entropy); check_client(!different_slab(next, prev_obj), "Heap corruption"); local_prev = local_curr; - local_curr = address_cast(next) & 0xffff; + local_curr = address_cast(next) & 0xffff'ffff; count++; iter = &next->next_object; } @@ -406,6 +342,20 @@ namespace snmalloc return count; } + /** + * Makes a terminator to a free list. + * + * Termination uses the bottom bit, this allows the next pointer + * to always be to the same slab. + */ + SNMALLOC_FAST_PATH void + terminate_list(uint32_t index, LocalEntropy& entropy) + { + auto term = CapPtr( + reinterpret_cast(end[index]) | 1); + end[index]->store(term, get_prev(index), entropy); + } + /** * Adds a terminator at the end of a free list, * but does not close the builder. Thus new elements @@ -422,7 +372,8 @@ namespace snmalloc * * It is used with preserve_queue disabled by close. */ - FreeListIter terminate(LocalEntropy& entropy, bool preserve_queue = true) + SNMALLOC_FAST_PATH void terminate( + FreeListIter& fl, LocalEntropy& entropy, bool preserve_queue = true) { if constexpr (RANDOM) { @@ -432,17 +383,19 @@ namespace snmalloc // If second list is non-empty, perform append. if (end[1] != &head[1]) { - end[1]->store(nullptr, get_prev(1), entropy); + terminate_list(1, entropy); // Append 1 to 0 auto mid = head[1].read(HEAD_KEY, entropy); end[0]->store(mid, get_prev(0), entropy); // Re-code first link in second list (if there is one). - // The first link in the second list will be encoded with initial_key, - // But that needs to be changed to the curr of the first list. + // The first link in the second list will be encoded with initial key + // of the head, But that needs to be changed to the curr of the first + // list. if (mid != nullptr) { - auto mid_next = mid->read_next(initial_key(mid) & 0xffff, entropy); + auto mid_next = + mid->read_next(address_cast(&head[1]) & 0xffff'ffff, entropy); mid->next_object.store(mid_next, get_curr(0), entropy); } @@ -455,12 +408,10 @@ namespace snmalloc { #ifdef CHECK_CLIENT prev[0] = prev[1]; - curr[0] = curr[1]; #endif end[0] = end[1]; #ifdef CHECK_CLIENT prev[1] = HEAD_KEY; - curr[1] = initial_key(h) & 0xffff; #endif end[1] = &(head[1]); } @@ -468,7 +419,8 @@ namespace snmalloc SNMALLOC_ASSERT(end[1] != &head[0]); SNMALLOC_ASSERT(end[0] != &head[1]); - return {h}; + fl = {h, address_cast(&head[0])}; + return; } } else @@ -476,28 +428,31 @@ namespace snmalloc UNUSED(preserve_queue); } - end[0]->store(nullptr, get_prev(0), entropy); - return {head[0].read(HEAD_KEY, entropy)}; + terminate_list(0, entropy); + fl = {head[0].read(HEAD_KEY, entropy), address_cast(&head[0])}; } /** * Close a free list, and set the iterator parameter * to iterate it. */ - void close(FreeListIter& dst, LocalEntropy& entropy) + SNMALLOC_FAST_PATH void close(FreeListIter& dst, LocalEntropy& entropy) { - dst = terminate(entropy, false); + terminate(dst, entropy, false); init(); } /** * Set the builder to a not building state. */ - void init() + constexpr void init() { for (size_t i = 0; i < LENGTH; i++) { end[i] = &head[i]; +#ifdef CHECK_CLIENT + prev[i] = HEAD_KEY; +#endif } } }; diff --git a/src/mem/globalalloc.h b/src/mem/globalalloc.h index 7ce318990..00265989e 100644 --- a/src/mem/globalalloc.h +++ b/src/mem/globalalloc.h @@ -1,200 +1,171 @@ #pragma once #include "../ds/helpers.h" -#include "alloc.h" -#include "pool.h" +#include "localalloc.h" namespace snmalloc { - inline bool needs_initialisation(void*); - void* init_thread_allocator(function_ref); - - template - class AllocPool : Pool + template + inline static void aggregate_stats(SharedStateHandle handle, Stats& stats) { - using Parent = Pool; + auto* alloc = Pool>::iterate(handle); - public: - static AllocPool* make(MemoryProvider& mp) + while (alloc != nullptr) { - static_assert( - sizeof(AllocPool) == sizeof(Parent), - "You cannot add fields to this class."); - // This cast is safe due to the static assert. - return static_cast(Parent::make(mp)); - } - - static AllocPool* make() noexcept - { - return make(default_memory_provider()); + auto a = alloc->attached_stats(); + if (a != nullptr) + stats.add(*a); + stats.add(alloc->stats()); + alloc = Pool>::iterate(handle, alloc); } + } - Alloc* acquire() - { - return Parent::acquire(Parent::memory_provider); - } +#ifdef USE_SNMALLOC_STATS + template + inline static void print_all_stats( + SharedStateHandle handle, std::ostream& o, uint64_t dumpid = 0) + { + auto alloc = Pool>::iterate(handle); - void release(Alloc* a) + while (alloc != nullptr) { - Parent::release(a); + auto stats = alloc->stats(); + if (stats != nullptr) + stats->template print(o, dumpid, alloc->id()); + alloc = Pool>::iterate(handle, alloc); } + } +#else + template + inline static void + print_all_stats(SharedStateHandle handle, void*& o, uint64_t dumpid = 0) + { + UNUSED(o); + UNUSED(dumpid); + UNUSED(handle); + } +#endif - public: - void aggregate_stats(Stats& stats) + template + inline static void cleanup_unused(SharedStateHandle handle) + { +#ifndef SNMALLOC_PASS_THROUGH + // Call this periodically to free and coalesce memory allocated by + // allocators that are not currently in use by any thread. + // One atomic operation to extract the stack, another to restore it. + // Handling the message queue for each stack is non-atomic. + auto* first = Pool>::extract(handle); + auto* alloc = first; + decltype(alloc) last; + + if (alloc != nullptr) { - auto* alloc = Parent::iterate(); - while (alloc != nullptr) { - stats.add(alloc->stats()); - alloc = Parent::iterate(alloc); + alloc->flush(); + last = alloc; + alloc = Pool>::extract(handle, alloc); } - } - -#ifdef USE_SNMALLOC_STATS - void print_all_stats(std::ostream& o, uint64_t dumpid = 0) - { - auto alloc = Parent::iterate(); - while (alloc != nullptr) - { - alloc->stats().template print(o, dumpid, alloc->id()); - alloc = Parent::iterate(alloc); - } - } -#else - void print_all_stats(void*& o, uint64_t dumpid = 0) - { - UNUSED(o); - UNUSED(dumpid); + Pool>::restore(handle, first, last); } #endif + } - void cleanup_unused() - { + /** + If you pass a pointer to a bool, then it returns whether all the + allocators are empty. If you don't pass a pointer to a bool, then will + raise an error all the allocators are not empty. + */ + template + inline static void + debug_check_empty(SharedStateHandle handle, bool* result = nullptr) + { #ifndef SNMALLOC_PASS_THROUGH - // Call this periodically to free and coalesce memory allocated by - // allocators that are not currently in use by any thread. - // One atomic operation to extract the stack, another to restore it. - // Handling the message queue for each stack is non-atomic. - auto* first = Parent::extract(); - auto* alloc = first; - decltype(alloc) last; - - if (alloc != nullptr) - { - while (alloc != nullptr) - { - alloc->handle_message_queue(); - last = alloc; - alloc = Parent::extract(alloc); - } + // This is a debugging function. It checks that all memory from all + // allocators has been freed. + auto* alloc = Pool>::iterate(handle); - restore(first, last); - } -#endif - } +# ifdef SNMALLOC_TRACING + std::cout << "debug check empty: first " << alloc << std::endl; +# endif + bool done = false; + bool okay = true; - /** - If you pass a pointer to a bool, then it returns whether all the - allocators are empty. If you don't pass a pointer to a bool, then will - raise an error all the allocators are not empty. - */ - void debug_check_empty(bool* result = nullptr) + while (!done) { -#ifndef SNMALLOC_PASS_THROUGH - // This is a debugging function. It checks that all memory from all - // allocators has been freed. - auto* alloc = Parent::iterate(); +# ifdef SNMALLOC_TRACING + std::cout << "debug_check_empty: Check all allocators!" << std::endl; +# endif + done = true; + alloc = Pool>::iterate(handle); + okay = true; - bool done = false; - bool okay = true; - - while (!done) + while (alloc != nullptr) { - done = true; - alloc = Parent::iterate(); - okay = true; - - while (alloc != nullptr) +# ifdef SNMALLOC_TRACING + std::cout << "debug check empty: " << alloc << std::endl; +# endif + // Check that the allocator has freed all memory. + // repeat the loop if empty caused message sends. + if (alloc->debug_is_empty(&okay)) { - // Check that the allocator has freed all memory. - alloc->debug_is_empty(&okay); - - // Post all remotes, including forwarded ones. If any allocator posts, - // repeat the loop. - if (alloc->remote_cache.capacity < REMOTE_CACHE) - { - alloc->stats().remote_post(); - alloc->remote_cache.post(alloc, alloc->get_trunc_id()); - done = false; - } - - alloc = Parent::iterate(alloc); + done = false; +# ifdef SNMALLOC_TRACING + std::cout << "debug check empty: sent messages " << alloc + << std::endl; +# endif } - } - if (result != nullptr) - { - *result = okay; - return; +# ifdef SNMALLOC_TRACING + std::cout << "debug check empty: okay = " << okay << std::endl; +# endif + alloc = Pool>::iterate(handle, alloc); } + } + + if (result != nullptr) + { + *result = okay; + return; + } - if (!okay) + // Redo check so abort is on allocator with allocation left. + if (!okay) + { + alloc = Pool>::iterate(handle); + while (alloc != nullptr) { - alloc = Parent::iterate(); - while (alloc != nullptr) - { - alloc->debug_is_empty(nullptr); - alloc = Parent::iterate(alloc); - } + alloc->debug_is_empty(nullptr); + alloc = Pool>::iterate(handle, alloc); } + } #else - UNUSED(result); + UNUSED(result); #endif - } + } - void debug_in_use(size_t count) + template + inline static void debug_in_use(SharedStateHandle handle, size_t count) + { + auto alloc = Pool>::iterate(handle); + while (alloc != nullptr) { - auto alloc = Parent::iterate(); - while (alloc != nullptr) + if (alloc->debug_is_in_use()) { - if (alloc->debug_is_in_use()) + if (count == 0) { - if (count == 0) - { - error("ERROR: allocator in use."); - } - count--; + error("ERROR: allocator in use."); } - alloc = Parent::iterate(alloc); + count--; + } + alloc = Pool>::iterate(handle, alloc); - if (count != 0) - { - error("Error: two few allocators in use."); - } + if (count != 0) + { + error("Error: two few allocators in use."); } } - }; - - using Alloc = Allocator< - needs_initialisation, - init_thread_allocator, - GlobalVirtual, - SNMALLOC_DEFAULT_CHUNKMAP, - true>; - - inline AllocPool*& current_alloc_pool() - { - return Singleton< - AllocPool*, - AllocPool::make>::get(); - } - - template - inline AllocPool* make_alloc_pool(MemoryProvider& mp) - { - return AllocPool::make(mp); } } // namespace snmalloc diff --git a/src/mem/globalconfig.h b/src/mem/globalconfig.h new file mode 100644 index 000000000..d87e75a2e --- /dev/null +++ b/src/mem/globalconfig.h @@ -0,0 +1,107 @@ +#pragma once + +#include "../backend/backend.h" +#include "../mem/corealloc.h" +#include "../mem/pool.h" +#include "../mem/slaballocator.h" +#include "commonconfig.h" + +namespace snmalloc +{ + // Forward reference to thread local cleanup. + void register_clean_up(); + +#ifdef USE_SNMALLOC_STATS + inline static void print_stats() + { + printf("No Stats yet!"); + // Stats s; + // current_alloc_pool()->aggregate_stats(s); + // s.print(std::cout); + } +#endif + + class Globals : public CommonConfig + { + public: + using Backend = BackendAllocator; + + private: + SNMALLOC_REQUIRE_CONSTINIT + inline static Backend::GlobalState backend_state; + + SNMALLOC_REQUIRE_CONSTINIT + inline static ChunkAllocatorState slab_allocator_state; + + SNMALLOC_REQUIRE_CONSTINIT + inline static PoolState> alloc_pool; + + SNMALLOC_REQUIRE_CONSTINIT + inline static std::atomic initialised{false}; + + SNMALLOC_REQUIRE_CONSTINIT + inline static std::atomic_flag initialisation_lock{}; + + public: + Backend::GlobalState& get_backend_state() + { + return backend_state; + } + + ChunkAllocatorState& get_slab_allocator_state() + { + return slab_allocator_state; + } + + PoolState>& pool() + { + return alloc_pool; + } + + static constexpr bool IsQueueInline = true; + + // Performs initialisation for this configuration + // of allocators. Needs to be idempotent, + // and concurrency safe. + void ensure_init() + { + FlagLock lock{initialisation_lock}; +#ifdef SNMALLOC_TRACING + std::cout << "Run init_impl" << std::endl; +#endif + + if (initialised) + return; + + // Need to initialise pagemap. + backend_state.init(); + +#ifdef USE_SNMALLOC_STATS + atexit(snmalloc::print_stats); +#endif + + initialised = true; + } + + bool is_initialised() + { + return initialised; + } + + // This needs to be a forward reference as the + // thread local state will need to know about this. + // This may allocate, so should only be called once + // a thread local allocator is available. + void register_clean_up() + { + snmalloc::register_clean_up(); + } + + // This is an empty structure as all the state is global + // for this allocator configuration. + static constexpr Globals get_handle() + { + return {}; + } + }; +} // namespace snmalloc diff --git a/src/mem/largealloc.h b/src/mem/largealloc.h deleted file mode 100644 index 0d870fcdd..000000000 --- a/src/mem/largealloc.h +++ /dev/null @@ -1,448 +0,0 @@ -#pragma once - -#include "../ds/flaglock.h" -#include "../ds/helpers.h" -#include "../ds/mpmcstack.h" -#include "../pal/pal.h" -#include "address_space.h" -#include "allocstats.h" -#include "baseslab.h" -#include "sizeclass.h" - -#include -#include - -namespace snmalloc -{ - template - class MemoryProviderStateMixin; - - class Largeslab : public Baseslab - { - // This is the view of a contiguous memory area when it is being kept - // in the global size-classed caches of available contiguous memory areas. - private: - template< - class a, - Construction c, - template - typename P, - template - typename AP> - friend class MPMCStack; - template - friend class MemoryProviderStateMixin; - AtomicCapPtr next = nullptr; - - public: - void init() - { - kind = Large; - } - }; - - /** - * A slab that has been decommitted. The first page remains committed and - * the only fields that are guaranteed to exist are the kind and next - * pointer from the superclass. - */ - struct Decommittedslab : public Largeslab - { - /** - * Constructor. Expected to be called via placement new into some memory - * that was formerly a superslab or large allocation and is now just some - * spare address space. - */ - Decommittedslab() - { - kind = Decommitted; - } - }; - - // This represents the state that the large allcoator needs to add to the - // global state of the allocator. This is currently stored in the memory - // provider, so we add this in. - template - class MemoryProviderStateMixin - { - /** - * Simple flag for checking if another instance of lazy-decommit is - * running - */ - std::atomic_flag lazy_decommit_guard = {}; - - /** - * Instantiate the ArenaMap here. - * - * In most cases, this will be a purely static object (a DefaultArenaMap - * using a GlobalPagemapTemplate or ExternalGlobalPagemapTemplate). For - * sandboxes, this may have per-instance state (e.g., the sandbox root); - * presently, that's handled by the MemoryProviderStateMixin constructor - * that takes a pointer to address space it owns. There is some - * non-orthogonality of concerns here. - */ - ArenaMap arena_map = {}; - - using ASM = AddressSpaceManager; - /** - * Manages address space for this memory provider. - */ - ASM address_space = {}; - - /** - * High-water mark of used memory. - */ - std::atomic peak_memory_used_bytes{0}; - - /** - * Memory current available in large_stacks - */ - std::atomic available_large_chunks_in_bytes{0}; - - /** - * Stack of large allocations that have been returned for reuse. - */ - ModArray< - NUM_LARGE_CLASSES, - MPMCStack> - large_stack; - - public: - using Pal = PAL; - - /** - * Pop an allocation from a large-allocation stack. This is safe to call - * concurrently with other acceses. If there is no large allocation on a - * particular stack then this will return `nullptr`. - */ - SNMALLOC_FAST_PATH CapPtr - pop_large_stack(size_t large_class) - { - auto p = large_stack[large_class].pop(); - if (p != nullptr) - { - const size_t rsize = bits::one_at_bit(SUPERSLAB_BITS) << large_class; - available_large_chunks_in_bytes -= rsize; - } - return p; - } - - /** - * Push `slab` onto the large-allocation stack associated with the size - * class specified by `large_class`. Always succeeds. - */ - SNMALLOC_FAST_PATH void - push_large_stack(CapPtr slab, size_t large_class) - { - const size_t rsize = bits::one_at_bit(SUPERSLAB_BITS) << large_class; - available_large_chunks_in_bytes += rsize; - large_stack[large_class].push(slab); - } - - /** - * Default constructor. This constructs a memory provider that doesn't yet - * own any memory, but which can claim memory from the PAL. - */ - MemoryProviderStateMixin() = default; - - /** - * Construct a memory provider owning some memory. The PAL provided with - * memory providers constructed in this way does not have to be able to - * allocate memory, if the initial reservation is sufficient. - */ - MemoryProviderStateMixin(CapPtr start, size_t len) - : address_space(start, len) - {} - /** - * Make a new memory provide for this PAL. - */ - static MemoryProviderStateMixin* make() noexcept - { - // Temporary stack-based storage to start the allocator in. - ASM local_asm{}; - ArenaMap local_am{}; - - // Allocate permanent storage for the allocator usung temporary allocator - MemoryProviderStateMixin* allocated = - local_asm - .template reserve_with_left_over( - sizeof(MemoryProviderStateMixin), local_am) - .template as_static() - .unsafe_capptr; - - if (allocated == nullptr) - error("Failed to initialise system!"); - - // Move address range inside itself - allocated->address_space = std::move(local_asm); - allocated->arena_map = std::move(local_am); - - // Register this allocator for low-memory call-backs - if constexpr (pal_supports) - { - auto callback = - allocated->template alloc_chunk( - allocated); - PAL::register_for_low_memory_callback(callback); - } - - return allocated; - } - - private: - SNMALLOC_SLOW_PATH void lazy_decommit() - { - // If another thread is try to do lazy decommit, let it continue. If - // we try to parallelise this, we'll most likely end up waiting on the - // same page table locks. - if (!lazy_decommit_guard.test_and_set()) - { - return; - } - // When we hit low memory, iterate over size classes and decommit all of - // the memory that we can. Start with the small size classes so that we - // hit cached superslabs first. - // FIXME: We probably shouldn't do this all at once. - // FIXME: We currently Decommit all the sizeclasses larger than 0. - for (size_t large_class = 0; large_class < NUM_LARGE_CLASSES; - large_class++) - { - if (!PAL::expensive_low_memory_check()) - { - break; - } - size_t rsize = bits::one_at_bit(SUPERSLAB_BITS) << large_class; - size_t decommit_size = rsize - OS_PAGE_SIZE; - // Grab all of the chunks of this size class. - CapPtr slab = large_stack[large_class].pop_all(); - while (slab != nullptr) - { - // Decommit all except for the first page and then put it back on - // the stack. - if (slab->get_kind() != Decommitted) - { - PAL::notify_not_using( - pointer_offset(slab.unsafe_capptr, OS_PAGE_SIZE), decommit_size); - } - // Once we've removed these from the stack, there will be no - // concurrent accesses and removal should have established a - // happens-before relationship, so it's safe to use relaxed loads - // here. - auto next = slab->next.load(std::memory_order_relaxed); - large_stack[large_class].push(CapPtr( - new (slab.unsafe_capptr) Decommittedslab())); - slab = next; - } - } - lazy_decommit_guard.clear(); - } - - class LowMemoryNotificationObject : public PalNotificationObject - { - MemoryProviderStateMixin* memory_provider; - - /*** - * Method for callback object to perform lazy decommit. - */ - static void process(PalNotificationObject* p) - { - // Unsafe downcast here. Don't want vtable and RTTI. - auto self = reinterpret_cast(p); - self->memory_provider->lazy_decommit(); - } - - public: - LowMemoryNotificationObject(MemoryProviderStateMixin* memory_provider) - : PalNotificationObject(&process), memory_provider(memory_provider) - {} - }; - - public: - /** - * Primitive allocator for structure that are required before - * the allocator can be running. - */ - template - T* alloc_chunk(Args&&... args) - { - // Cache line align - size_t size = bits::align_up(sizeof(T), 64); - size = bits::max(size, alignment); - auto p = - address_space.template reserve_with_left_over(size, arena_map); - if (p == nullptr) - return nullptr; - - peak_memory_used_bytes += size; - - return new (p.unsafe_capptr) T(std::forward(args)...); - } - - template - CapPtr reserve(size_t large_class) noexcept - { - size_t size = bits::one_at_bit(SUPERSLAB_BITS) << large_class; - peak_memory_used_bytes += size; - return address_space.template reserve(size, arena_map) - .template as_static(); - } - - /** - * Returns a pair of current memory usage and peak memory usage. - * Both statistics are very coarse-grained. - */ - std::pair memory_usage() - { - size_t avail = available_large_chunks_in_bytes; - size_t peak = peak_memory_used_bytes; - return {peak - avail, peak}; - } - - template - SNMALLOC_FAST_PATH CapPtr capptr_amplify(CapPtr r) - { - return arena_map.template capptr_amplify(r); - } - - ArenaMap& arenamap() - { - return arena_map; - } - }; - - using Stats = AllocStats; - - template - class LargeAlloc - { - public: - // This will be a zero-size structure if stats are not enabled. - Stats stats; - - MemoryProvider& memory_provider; - - LargeAlloc(MemoryProvider& mp) : memory_provider(mp) {} - - template - CapPtr - alloc(size_t large_class, size_t rsize, size_t size) - { - SNMALLOC_ASSERT( - (bits::one_at_bit(SUPERSLAB_BITS) << large_class) == rsize); - - CapPtr p = - memory_provider.pop_large_stack(large_class); - - if (p == nullptr) - { - p = memory_provider.template reserve(large_class); - if (p == nullptr) - return nullptr; - MemoryProvider::Pal::template notify_using( - p.unsafe_capptr, rsize); - } - else - { - stats.superslab_pop(); - - // Cross-reference alloc.h's large_dealloc decommitment condition. - bool decommitted = - ((decommit_strategy == DecommitSuperLazy) && - (p.template as_static().unsafe_capptr->get_kind() == - Decommitted)) || - (large_class > 0) || (decommit_strategy == DecommitSuper); - - if (decommitted) - { - // The first page is already in "use" for the stack element, - // this will need zeroing for a YesZero call. - if constexpr (zero_mem == YesZero) - pal_zero(p, OS_PAGE_SIZE); - - // Notify we are using the rest of the allocation. - // Passing zero_mem ensures the PAL provides zeroed pages if - // required. - MemoryProvider::Pal::template notify_using( - pointer_offset(p.unsafe_capptr, OS_PAGE_SIZE), - rsize - OS_PAGE_SIZE); - } - else - { - // This is a superslab that has not been decommitted. - if constexpr (zero_mem == YesZero) - pal_zero( - p, bits::align_up(size, OS_PAGE_SIZE)); - else - UNUSED(size); - } - } - - SNMALLOC_ASSERT(p.as_void() == pointer_align_up(p.as_void(), rsize)); - return p; - } - - void dealloc(CapPtr p, size_t large_class) - { - if constexpr (decommit_strategy == DecommitSuperLazy) - { - static_assert( - pal_supports, - "A lazy decommit strategy cannot be implemented on platforms " - "without low memory notifications"); - } - - size_t rsize = bits::one_at_bit(SUPERSLAB_BITS) << large_class; - - // Cross-reference largealloc's alloc() decommitted condition. - if ( - (decommit_strategy != DecommitNone) && - (large_class != 0 || decommit_strategy == DecommitSuper)) - { - MemoryProvider::Pal::notify_not_using( - pointer_offset(p, OS_PAGE_SIZE).unsafe_capptr, rsize - OS_PAGE_SIZE); - } - - stats.superslab_push(); - memory_provider.push_large_stack(p, large_class); - } - - template - SNMALLOC_FAST_PATH CapPtr capptr_amplify(CapPtr r) - { - return memory_provider.template capptr_amplify(r); - } - }; - - struct DefaultPrimAlloc; - -#ifndef SNMALLOC_DEFAULT_MEMORY_PROVIDER -# define SNMALLOC_DEFAULT_MEMORY_PROVIDER \ - MemoryProviderStateMixin> -#endif - - /** - * The type of the default memory allocator. This can be changed by defining - * `SNMALLOC_DEFAULT_MEMORY_PROVIDER` before including this file. By default - * it is `MemoryProviderStateMixin` a class that allocates directly from - * the platform abstraction layer. - */ - using GlobalVirtual = SNMALLOC_DEFAULT_MEMORY_PROVIDER; - - /** - * The memory provider that will be used if no other provider is explicitly - * passed as an argument. - */ - inline GlobalVirtual& default_memory_provider() - { - return *(Singleton::get()); - } - - struct DefaultPrimAlloc - { - template - static T* alloc_chunk(Args&&... args) - { - return default_memory_provider().alloc_chunk(args...); - } - }; -} // namespace snmalloc diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h new file mode 100644 index 000000000..9a674916f --- /dev/null +++ b/src/mem/localalloc.h @@ -0,0 +1,535 @@ +#pragma once + +#ifdef _MSC_VER +# define ALLOCATOR __declspec(allocator) +#else +# define ALLOCATOR +#endif + +#include "../ds/ptrwrap.h" +#include "corealloc.h" +#include "freelist.h" +#include "localcache.h" +#include "pool.h" +#include "remotecache.h" +#include "sizeclasstable.h" + +#ifdef SNMALLOC_TRACING +# include +#endif +#include +#include +namespace snmalloc +{ + enum Boundary + { + /** + * The location of the first byte of this allocation. + */ + Start, + /** + * The location of the last byte of the allocation. + */ + End, + /** + * The location one past the end of the allocation. This is mostly useful + * for bounds checking, where anything less than this value is safe. + */ + OnePastEnd + }; + + // This class contains the fastest path code for the allocator. + template + class LocalAllocator + { + using CoreAlloc = CoreAllocator; + + private: + /** + * Contains a way to access all the shared state for this allocator. + * This may have no dynamic state, and be purely static. + */ + SharedStateHandle handle; + + // Free list per small size class. These are used for + // allocation on the fast path. This part of the code is inspired by + // mimalloc. + // Also contains remote deallocation cache. + LocalCache local_cache; + + // Underlying allocator for most non-fast path operations. + CoreAlloc* core_alloc{nullptr}; + + // As allocation and deallocation can occur during thread teardown + // we need to record if we are already in that state as we will not + // receive another teardown call, so each operation needs to release + // the underlying data structures after the call. + bool post_teardown{false}; + + /** + * Checks if the core allocator has been initialised, and runs the + * `action` with the arguments, args. + * + * If the core allocator is not initialised, then first initialise it, + * and then perform the action using the core allocator. + * + * This is an abstraction of the common pattern of check initialisation, + * and then performing the operations. It is carefully crafted to tail + * call the continuations, and thus generate good code for the fast path. + */ + template + SNMALLOC_FAST_PATH decltype(auto) check_init(Action action, Args... args) + { + if (likely(core_alloc != nullptr)) + { + return core_alloc->handle_message_queue(action, core_alloc, args...); + } + return lazy_init(action, args...); + } + + /** + * This initialises the fast allocator by acquiring a core allocator, and + * setting up its local copy of data structures. + */ + template + SNMALLOC_SLOW_PATH decltype(auto) lazy_init(Action action, Args... args) + { + SNMALLOC_ASSERT(core_alloc == nullptr); + + // Initialise the thread local allocator + init(); + + // register_clean_up must be called after init. register clean up may be + // implemented with allocation, so need to ensure we have a valid + // allocator at this point. + if (!post_teardown) + // Must be called at least once per thread. + // A pthread implementation only calls the thread destruction handle + // if the key has been set. + handle.register_clean_up(); + + // Perform underlying operation + auto r = action(core_alloc, args...); + + // After performing underlying operation, in the case of teardown already + // having begun, we must flush any state we just acquired. + if (post_teardown) + { +#ifdef SNMALLOC_TRACING + std::cout << "post_teardown flush()" << std::endl; +#endif + // We didn't have an allocator because the thread is being torndown. + // We need to return any local state, so we don't leak it. + flush(); + } + + return r; + } + + /** + * Allocation that are larger than are handled by the fast allocator must be + * passed to the core allocator. + */ + template + SNMALLOC_SLOW_PATH void* alloc_not_small(size_t size) + { + if (size == 0) + { + // Deal with alloc zero of with a small object here. + // Alternative semantics giving nullptr is also allowed by the + // standard. + return small_alloc(1); + } + + return check_init([&](CoreAlloc* core_alloc) { + // Grab slab of correct size + // Set remote as large allocator remote. + auto [chunk, meta] = ChunkAllocator::alloc_chunk( + handle, + core_alloc->backend_state, + bits::next_pow2_bits(size), // TODO + large_size_to_chunk_sizeclass(size), + large_size_to_chunk_size(size), + handle.fake_large_remote); + // set up meta data so sizeclass is correct, and hence alloc size, and + // external pointer. +#ifdef SNMALLOC_TRACING + std::cout << "size " << size << " sizeclass " << size_to_sizeclass(size) + << std::endl; +#endif + + // Note that meta data is not currently used for large allocs. + // meta->initialise(size_to_sizeclass(size)); + UNUSED(meta); + + if (zero_mem == YesZero) + { + SharedStateHandle::Backend::Pal::template zero( + chunk.unsafe_ptr(), size); + } + + return chunk.unsafe_ptr(); + }); + } + + template + SNMALLOC_FAST_PATH void* small_alloc(size_t size) + { + // SNMALLOC_ASSUME(size <= sizeclass_to_size(NUM_SIZECLASSES)); + auto slowpath = [&]( + sizeclass_t sizeclass, + FreeListIter* fl) SNMALLOC_FAST_PATH_LAMBDA { + if (likely(core_alloc != nullptr)) + { + return core_alloc->handle_message_queue( + [](CoreAlloc* core_alloc, sizeclass_t sizeclass, FreeListIter* fl) { + return core_alloc->template small_alloc(sizeclass, *fl); + }, + core_alloc, + sizeclass, + fl); + } + return lazy_init( + [&](CoreAlloc*, sizeclass_t sizeclass) { + return small_alloc(sizeclass_to_size(sizeclass)); + }, + sizeclass); + }; + + return local_cache.template alloc( + size, slowpath); + } + + /** + * Send all remote deallocation to other threads. + */ + void post_remote_cache() + { + core_alloc->post(); + } + + /** + * Slow path for deallocation we do not have space for this remote + * deallocation. This could be because, + * - we actually don't have space for this remote deallocation, + * and need to send them on; or + * - the allocator was not already initialised. + * In the second case we need to recheck if this is a remote deallocation, + * as we might acquire the originating allocator. + */ + SNMALLOC_SLOW_PATH void dealloc_remote_slow(void* p) + { + if (core_alloc != nullptr) + { +#ifdef SNMALLOC_TRACING + std::cout << "Remote dealloc post" << p << " size " << alloc_size(p) + << std::endl; +#endif + MetaEntry entry = SharedStateHandle::Backend::get_meta_data( + handle.get_backend_state(), address_cast(p)); + local_cache.remote_dealloc_cache.template dealloc( + entry.get_remote()->trunc_id(), CapPtr(p)); + post_remote_cache(); + return; + } + + // Recheck what kind of dealloc we should do incase, the allocator we get + // from lazy_init is the originating allocator. + lazy_init( + [&](CoreAlloc*, void* p) { + dealloc(p); // TODO don't double count statistics + return nullptr; + }, + p); + } + + /** + * Abstracts access to the message queue to handle different + * layout configurations of the allocator. + */ + auto& message_queue() + { + return local_cache.remote_allocator->message_queue; + } + + public: + constexpr LocalAllocator() + : handle(SharedStateHandle::get_handle()), + local_cache(&handle.unused_remote) + {} + + LocalAllocator(SharedStateHandle handle) + : handle(handle), local_cache(&handle.unused_remote) + {} + + // This is effectively the constructor for the LocalAllocator, but due to + // not wanting initialisation checks on the fast path, it is initialised + // lazily. + void init() + { + // Initialise the global allocator structures + handle.ensure_init(); + + // Should only be called if the allocator has not been initialised. + SNMALLOC_ASSERT(core_alloc == nullptr); + + // Grab an allocator for this thread. + auto c = Pool::acquire(handle, &(this->local_cache), handle); + + // Attach to it. + c->attach(&local_cache); + core_alloc = c; +#ifdef SNMALLOC_TRACING + std::cout << "init(): core_alloc=" << core_alloc << "@" << &local_cache + << std::endl; +#endif + // local_cache.stats.sta rt(); + } + + // Return all state in the fast allocator and release the underlying + // core allocator. This is used during teardown to empty the thread + // local state. + void flush() + { + // Detached thread local state from allocator. + if (core_alloc != nullptr) + { + core_alloc->flush(); + + // core_alloc->stats().add(local_cache.stats); + // // Reset stats, required to deal with repeated flushing. + // new (&local_cache.stats) Stats(); + + // Detach underlying allocator + core_alloc->attached_cache = nullptr; + // Return underlying allocator to the system. + Pool::release(handle, core_alloc); + + // Set up thread local allocator to look like + // it is new to hit slow paths. + core_alloc = nullptr; +#ifdef SNMALLOC_TRACING + std::cout << "flush(): core_alloc=" << core_alloc << std::endl; +#endif + local_cache.remote_allocator = &handle.unused_remote; + local_cache.remote_dealloc_cache.capacity = 0; + } + } + + /** + * Allocate memory of a dynamically known size. + */ + template + SNMALLOC_FAST_PATH ALLOCATOR void* alloc(size_t size) + { +#ifdef SNMALLOC_PASS_THROUGH + // snmalloc guarantees a lot of alignment, so we can depend on this + // make pass through call aligned_alloc with the alignment snmalloc + // would guarantee. + void* result = external_alloc::aligned_alloc( + natural_alignment(size), round_size(size)); + if constexpr (zero_mem == YesZero) + memset(result, 0, size); + return result; +#else + // Perform the - 1 on size, so that zero wraps around and ends up on + // slow path. + if (likely((size - 1) <= (sizeclass_to_size(NUM_SIZECLASSES - 1) - 1))) + { + // Small allocations are more likely. Improve + // branch prediction by placing this case first. + return small_alloc(size); + } + + // TODO capptr_reveal? + return alloc_not_small(size); +#endif + } + + /** + * Allocate memory of a statically known size. + */ + template + SNMALLOC_FAST_PATH ALLOCATOR void* alloc() + { + // TODO optimise + return alloc(size); + } + + SNMALLOC_FAST_PATH void dealloc(void* p) + { + // TODO Pass through code! + // TODO: + // Care is needed so that dealloc(nullptr) works before init + // The backend allocator must ensure that a minimal page map exists + // before init, that maps null to a remote_deallocator that will never be + // in thread local state. + + const MetaEntry& entry = SharedStateHandle::Backend::get_meta_data( + handle.get_backend_state(), address_cast(p)); + if (likely(local_cache.remote_allocator == entry.get_remote())) + { + if (likely(CoreAlloc::dealloc_local_object_fast( + entry, p, local_cache.entropy))) + return; + core_alloc->dealloc_local_object_slow(entry); + return; + } + + if (likely(entry.get_remote() != handle.fake_large_remote)) + { + // Check if we have space for the remote deallocation + if (local_cache.remote_dealloc_cache.reserve_space(entry)) + { + local_cache.remote_dealloc_cache.template dealloc( + entry.get_remote()->trunc_id(), CapPtr(p)); +#ifdef SNMALLOC_TRACING + std::cout << "Remote dealloc fast" << p << " size " << alloc_size(p) + << std::endl; +#endif + return; + } + + dealloc_remote_slow(p); + return; + } + + // Large deallocation or null. + if (likely(p != nullptr)) + { + // Check this is managed by this pagemap. + check_client(entry.get_sizeclass() != 0, "Not allocated by snmalloc."); + + size_t size = bits::one_at_bit(entry.get_sizeclass()); + + // Check for start of allocation. + check_client( + pointer_align_down(p, size) == p, "Not start of an allocation."); + + size_t slab_sizeclass = large_size_to_chunk_sizeclass(size); +#ifdef SNMALLOC_TRACING + std::cout << "Large deallocation: " << size + << " chunk sizeclass: " << slab_sizeclass << std::endl; +#endif + ChunkRecord* slab_record = + reinterpret_cast(entry.get_metaslab()); + slab_record->chunk = CapPtr(p); + ChunkAllocator::dealloc(handle, slab_record, slab_sizeclass); + return; + } + +#ifdef SNMALLOC_TRACING + std::cout << "nullptr deallocation" << std::endl; +#endif + return; + } + + SNMALLOC_FAST_PATH void dealloc(void* p, size_t s) + { + UNUSED(s); + dealloc(p); + } + + template + SNMALLOC_FAST_PATH void dealloc(void* p) + { + UNUSED(size); + dealloc(p); + } + + void teardown() + { +#ifdef SNMALLOC_TRACING + std::cout << "Teardown: core_alloc=" << core_alloc << "@" << &local_cache + << std::endl; +#endif + post_teardown = true; + if (core_alloc != nullptr) + { + flush(); + } + } + + SNMALLOC_FAST_PATH size_t alloc_size(const void* p_raw) + { + // Note that this should return 0 for nullptr. + // Other than nullptr, we know the system will be initialised as it must + // be called with something we have already allocated. + // To handle this case we require the uninitialised pagemap contain an + // entry for the first chunk of memory, that states it represents a large + // object, so we can pull the check for null off the fast path. + MetaEntry entry = SharedStateHandle::Backend::get_meta_data( + handle.get_backend_state(), address_cast(p_raw)); + + if (likely(entry.get_remote() != handle.fake_large_remote)) + return sizeclass_to_size(entry.get_sizeclass()); + + // Sizeclass zero is for large is actually zero + if (likely(entry.get_sizeclass() != 0)) + return bits::one_at_bit(entry.get_sizeclass()); + + return 0; + } + + /** + * Returns the Start/End of an object allocated by this allocator + * + * It is valid to pass any pointer, if the object was not allocated + * by this allocator, then it give the start and end as the whole of + * the potential pointer space. + */ + template + void* external_pointer(void* p_raw) + { + // TODO bring back the CHERI bits. Wes to review if required. + if (likely(handle.is_initialised())) + { + MetaEntry entry = + SharedStateHandle::Backend::template get_meta_data( + handle.get_backend_state(), address_cast(p_raw)); + auto sizeclass = entry.get_sizeclass(); + if (likely(entry.get_remote() != handle.fake_large_remote)) + { + auto rsize = sizeclass_to_size(sizeclass); + auto offset = + address_cast(p_raw) & (sizeclass_to_slab_size(sizeclass) - 1); + auto start_offset = round_by_sizeclass(sizeclass, offset); + if constexpr (location == Start) + { + UNUSED(rsize); + return pointer_offset(p_raw, start_offset - offset); + } + else if constexpr (location == End) + return pointer_offset(p_raw, rsize + start_offset - offset - 1); + else + return pointer_offset(p_raw, rsize + start_offset - offset); + } + + // Sizeclass zero of a large allocation is used for not managed by us. + if (likely(sizeclass != 0)) + { + // This is a large allocation, find start by masking. + auto rsize = bits::one_at_bit(sizeclass); + auto start = pointer_align_down(p_raw, rsize); + if constexpr (location == Start) + return start; + else if constexpr (location == End) + return pointer_offset(start, rsize); + else + return pointer_offset(start, rsize - 1); + } + } + else + { + // Allocator not initialised, so definitely not our allocation + } + + if constexpr ((location == End) || (location == OnePastEnd)) + // We don't know the End, so return MAX_PTR + return pointer_offset(nullptr, UINTPTR_MAX); + else + // We don't know the Start, so return MIN_PTR + return nullptr; + } + }; +} // namespace snmalloc \ No newline at end of file diff --git a/src/mem/localcache.h b/src/mem/localcache.h new file mode 100644 index 000000000..ba364ad97 --- /dev/null +++ b/src/mem/localcache.h @@ -0,0 +1,112 @@ +#pragma once + +#include "../ds/ptrwrap.h" +#include "allocstats.h" +#include "freelist.h" +#include "remotecache.h" +#include "sizeclasstable.h" + +#include + +namespace snmalloc +{ + using Stats = AllocStats; + + inline static SNMALLOC_FAST_PATH void* finish_alloc_no_zero( + snmalloc::CapPtr p, + sizeclass_t sizeclass) + { + SNMALLOC_ASSERT(Metaslab::is_start_of_object(sizeclass, address_cast(p))); + UNUSED(sizeclass); + + auto r = capptr_reveal(capptr_export(p.as_void())); + + return r; + } + + template + inline static SNMALLOC_FAST_PATH void* finish_alloc( + snmalloc::CapPtr p, + sizeclass_t sizeclass) + { + auto r = finish_alloc_no_zero(p, sizeclass); + + if constexpr (zero_mem == YesZero) + SharedStateHandle::Backend::Pal::zero(r, sizeclass_to_size(sizeclass)); + + // TODO: Should this be zeroing the FreeObject state, in the non-zeroing + // case? + + return r; + } + + // This is defined on its own, so that it can be embedded in the + // thread local fast allocator, but also referenced from the + // thread local core allocator. + struct LocalCache + { + // Free list per small size class. These are used for + // allocation on the fast path. This part of the code is inspired by + // mimalloc. + FreeListIter small_fast_free_lists[NUM_SIZECLASSES]; + + // This is the entropy for a particular thread. + LocalEntropy entropy; + + // TODO: Minimal stats object for just the stats on this datastructure. + // This will be a zero-size structure if stats are not enabled. + Stats stats; + + // Pointer to the remote allocator message_queue, used to check + // if a deallocation is local. + RemoteAllocator* remote_allocator; + + /** + * Remote deallocations for other threads + */ + RemoteDeallocCache remote_dealloc_cache; + + constexpr LocalCache(RemoteAllocator* remote_allocator) + : remote_allocator(remote_allocator) + {} + + template< + size_t allocator_size, + typename DeallocFun, + typename SharedStateHandle> + bool flush(DeallocFun dealloc, SharedStateHandle handle) + { + // Return all the free lists to the allocator. + // Used during thread teardown + for (size_t i = 0; i < NUM_SIZECLASSES; i++) + { + // TODO could optimise this, to return the whole list in one append + // call. + while (!small_fast_free_lists[i].empty()) + { + auto p = small_fast_free_lists[i].take(entropy); + dealloc(finish_alloc_no_zero(p, i)); + } + } + + return remote_dealloc_cache.post( + handle, remote_allocator->trunc_id()); + } + + template + SNMALLOC_FAST_PATH void* alloc(size_t size, Slowpath slowpath) + { + sizeclass_t sizeclass = size_to_sizeclass(size); + stats.alloc_request(size); + stats.sizeclass_alloc(sizeclass); + auto& fl = small_fast_free_lists[sizeclass]; + if (likely(!fl.empty())) + { + auto p = fl.take(entropy); + return finish_alloc(p, sizeclass); + } + return slowpath(sizeclass, &fl); + } + }; + +} // namespace snmalloc \ No newline at end of file diff --git a/src/mem/mediumslab.h b/src/mem/mediumslab.h deleted file mode 100644 index c1ea9be33..000000000 --- a/src/mem/mediumslab.h +++ /dev/null @@ -1,156 +0,0 @@ -#pragma once - -#include "../ds/dllist.h" -#include "allocconfig.h" -#include "allocslab.h" -#include "sizeclass.h" - -namespace snmalloc -{ - class Mediumslab : public Allocslab - { - // This is the view of a 16 mb area when it is being used to allocate - // medium sized classes: 64 kb to 16 mb, non-inclusive. - private: - friend DLList; - - // Keep the allocator pointer on a separate cache line. It is read by - // other threads, and does not change, so we avoid false sharing. - alignas(CACHELINE_SIZE) CapPtr next; - CapPtr prev; - - // Store a pointer to ourselves without platform constraints applied, - // as we need this to be able to zero memory by manipulating the VM map - CapPtr self_chunk; - - uint16_t free; - uint8_t head; - uint8_t sizeclass; - uint16_t stack[SLAB_COUNT - 1]; - - public: - static constexpr size_t header_size() - { - static_assert( - sizeof(Mediumslab) < OS_PAGE_SIZE, - "Mediumslab header size must be less than the page size"); - static_assert( - sizeof(Mediumslab) < SLAB_SIZE, - "Mediumslab header size must be less than the slab size"); - - /* - * Always use a full page or SLAB, whichever is smaller, in order - * to get good alignment of individual allocations. Some platforms - * have huge minimum pages (e.g., Linux on PowerPC uses 64KiB) and - * our SLABs are occasionally small by comparison (e.g., in OE, when - * we take them to be 8KiB). - */ - return bits::align_up(sizeof(Mediumslab), min(OS_PAGE_SIZE, SLAB_SIZE)); - } - - /** - * Given a highly-privileged pointer pointing to or within an object in - * this slab, return a pointer to the slab headers. - * - * In debug builds on StrictProvenance architectures, we will enforce the - * slab bounds on this returned pointer. In non-debug builds, we will - * return a highly-privileged pointer (i.e., CBArena) instead as these - * pointers are not exposed from the allocator. - */ - template - static SNMALLOC_FAST_PATH CapPtr - get(CapPtr p) - { - return capptr_bound_chunkd( - pointer_align_down(p.as_void()), - SUPERSLAB_SIZE); - } - - static void init( - CapPtr self, - RemoteAllocator* alloc, - sizeclass_t sc, - size_t rsize) - { - SNMALLOC_ASSERT(sc >= NUM_SMALL_CLASSES); - SNMALLOC_ASSERT((sc - NUM_SMALL_CLASSES) < NUM_MEDIUM_CLASSES); - - self->allocator = alloc; - self->head = 0; - - // If this was previously a Mediumslab of the same sizeclass, don't - // initialise the allocation stack. - if ((self->kind != Medium) || (self->sizeclass != sc)) - { - self->self_chunk = self.as_void(); - self->sizeclass = static_cast(sc); - uint16_t ssize = static_cast(rsize >> 8); - self->kind = Medium; - self->free = medium_slab_free(sc); - for (uint16_t i = self->free; i > 0; i--) - self->stack[self->free - i] = - static_cast((SUPERSLAB_SIZE >> 8) - (i * ssize)); - } - else - { - SNMALLOC_ASSERT(self->free == medium_slab_free(sc)); - SNMALLOC_ASSERT(self->self_chunk == self.as_void()); - } - } - - uint8_t get_sizeclass() - { - return sizeclass; - } - - template - static CapPtr - alloc(CapPtr self, size_t size) - { - SNMALLOC_ASSERT(!full(self)); - - uint16_t index = self->stack[self->head++]; - auto p = pointer_offset(self, (static_cast(index) << 8)); - self->free--; - - if constexpr (zero_mem == YesZero) - pal_zero(Aal::capptr_rebound(self->self_chunk, p), size); - else - UNUSED(size); - - return Aal::capptr_bound(p, size); - } - - static bool - dealloc(CapPtr self, CapPtr p) - { - SNMALLOC_ASSERT(self->head > 0); - - // Returns true if the Mediumslab was full before this deallocation. - bool was_full = full(self); - self->free++; - self->stack[--(self->head)] = self->address_to_index(address_cast(p)); - - return was_full; - } - - template - static bool full(CapPtr self) - { - return self->free == 0; - } - - template - static bool empty(CapPtr self) - { - return self->head == 0; - } - - private: - uint16_t address_to_index(address_t p) - { - // Get the offset from the slab for a memory location. - return static_cast((p - address_cast(this)) >> 8); - } - }; -} // namespace snmalloc diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index c1b51f544..f341064a3 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -3,16 +3,16 @@ #include "../ds/cdllist.h" #include "../ds/dllist.h" #include "../ds/helpers.h" +#include "../mem/remoteallocator.h" #include "freelist.h" #include "ptrhelpers.h" -#include "sizeclass.h" +#include "sizeclasstable.h" namespace snmalloc { class Slab; - using SlabList = CDLLNode; - using SlabLink = CDLLNode; + using SlabLink = CDLLNode<>; static_assert( sizeof(SlabLink) <= MIN_ALLOC_SIZE, @@ -27,25 +27,32 @@ namespace snmalloc struct MetaslabEnd { /** - * How many entries are not in the free list of slab, i.e. - * how many entries are needed to fully free this slab. - * - * In the case of a fully allocated slab, where prev==0 needed - * will be 1. This enables 'return_object' to detect the slow path - * case with a single operation subtract and test. + * The number of deallocation required until we hit a slow path. This + * counts down in two different ways that are handled the same on the + * fast path. The first is + * - deallocations until the slab has sufficient entries to be considered + * useful to allocate from. This could be as low as 1, or when we have + * a requirement for entropy then it could be much higher. + * - deallocations until the slab is completely unused. This is needed + * to be detected, so that the statistics can be kept up to date, and + * potentially return memory to the a global pool of slabs/chunks. */ uint16_t needed = 0; - uint8_t sizeclass; - // Initially zero to encode the superslabs relative list of slabs. - uint8_t next = 0; + /** + * Flag that is used to indicate that the slab is currently not active. + * I.e. it is not in a CoreAllocator cache for the appropriate sizeclass. + */ + bool sleeping = false; }; // The Metaslab represent the status of a single slab. // This can be either a short or a standard slab. - class Metaslab : public SlabLink + class alignas(CACHELINE_SIZE) Metaslab : public SlabLink { public: + constexpr Metaslab() : SlabLink(true) {} + /** * Data-structure for building the free list for this slab. * @@ -62,26 +69,23 @@ namespace snmalloc return free_queue.s.needed; } - uint8_t sizeclass() - { - return free_queue.s.sizeclass; - } - - uint8_t& next() + bool& sleeping() { - return free_queue.s.next; + return free_queue.s.sleeping; } - void initialise(sizeclass_t sizeclass, CapPtr slab) + /** + * Initialise Metaslab for a slab. + */ + void initialise(sizeclass_t sizeclass) { - free_queue.s.sizeclass = static_cast(sizeclass); free_queue.init(); // Set up meta data as if the entire slab has been turned into a free // list. This means we don't have to check for special cases where we have // returned all the elements, but this is a slab that is still being bump // allocated from. Hence, the bump allocator slab will never be returned // for use in another size class. - set_full(slab); + set_sleeping(sizeclass); } /** @@ -101,155 +105,111 @@ namespace snmalloc return needed() == 0; } - bool is_full() - { - return get_prev() == nullptr; - } - - /** - * Only wake slab if we have this many free allocations - * - * This helps remove bouncing around empty to non-empty cases. - * - * It also increases entropy, when we have randomisation. - */ - uint16_t threshold_for_waking_slab(bool is_short_slab) + bool is_sleeping() { - auto capacity = get_slab_capacity(sizeclass(), is_short_slab); - uint16_t threshold = (capacity / 8) | 1; - uint16_t max = 32; - return bits::min(threshold, max); + return sleeping(); } - template - SNMALLOC_FAST_PATH void set_full(CapPtr slab) + SNMALLOC_FAST_PATH void set_sleeping(sizeclass_t sizeclass) { - static_assert(B == CBChunkD || B == CBChunk); SNMALLOC_ASSERT(free_queue.empty()); - // Prepare for the next free queue to be built. - free_queue.open(slab.as_void()); - // Set needed to at least one, possibly more so we only use // a slab when it has a reasonable amount of free elements - needed() = threshold_for_waking_slab(Metaslab::is_short(slab)); - null_prev(); + needed() = threshold_for_waking_slab(sizeclass); + sleeping() = true; } - template - static SNMALLOC_FAST_PATH CapPtr()> - get_slab(CapPtr p) + SNMALLOC_FAST_PATH void set_not_sleeping(sizeclass_t sizeclass) { - static_assert(B == CBArena || B == CBChunkD || B == CBChunk); + auto allocated = sizeclass_to_slab_object_count(sizeclass); + needed() = allocated - threshold_for_waking_slab(sizeclass); - return capptr_bound_chunkd( - pointer_align_down(p.as_void()), SLAB_SIZE); - } + // Design ensures we can't move from full to empty. + // There are always some more elements to free at this + // point. This is because the threshold is always less + // than the count for the slab + SNMALLOC_ASSERT(needed() != 0); - template - static bool is_short(CapPtr p) - { - return pointer_align_down(p.as_void()) == p; + sleeping() = false; } - SNMALLOC_FAST_PATH bool is_start_of_object(address_t p) + static SNMALLOC_FAST_PATH bool + is_start_of_object(sizeclass_t sizeclass, address_t p) { return is_multiple_of_sizeclass( - sizeclass(), SLAB_SIZE - (p - address_align_down(p))); + sizeclass, + p - (bits::align_down(p, sizeclass_to_slab_size(sizeclass)))); } /** - * Takes a free list out of a slabs meta data. - * Returns the link as the allocation, and places the free list into the - * `fast_free_list` for further allocations. + * TODO */ - template - static SNMALLOC_FAST_PATH CapPtr alloc( - CapPtr self, + static SNMALLOC_FAST_PATH CapPtr alloc( + Metaslab* meta, FreeListIter& fast_free_list, - size_t rsize, - LocalEntropy& entropy) + LocalEntropy& entropy, + sizeclass_t sizeclass) { - SNMALLOC_ASSERT(rsize == sizeclass_to_size(self->sizeclass())); - SNMALLOC_ASSERT(!self->is_full()); - - self->free_queue.close(fast_free_list, entropy); - auto p = fast_free_list.take(entropy); - auto slab = Aal::capptr_rebound(self.as_void(), p); - auto meta = Metaslab::get_slab(slab); + FreeListIter tmp_fl; + meta->free_queue.close(tmp_fl, entropy); + auto p = tmp_fl.take(entropy); + fast_free_list = tmp_fl; +#ifdef CHECK_CLIENT entropy.refresh_bits(); +#endif // Treat stealing the free list as allocating it all. - self->remove(); - self->set_full(meta); - - SNMALLOC_ASSERT(self->is_start_of_object(address_cast(p))); - - self->debug_slab_invariant(meta, entropy); - - if constexpr (zero_mem == YesZero) - { - if (rsize < PAGE_ALIGNED_SIZE) - pal_zero(p, rsize); - else - pal_zero(Aal::capptr_rebound(self.as_void(), p), rsize); - } - else - { - UNUSED(rsize); - } - - // TODO: Should this be zeroing the FreeObject state? - return capptr_export(p.as_void()); - } + // This marks the slab as sleeping, and sets a wakeup + // when sufficient deallocations have occurred to this slab. + meta->set_sleeping(sizeclass); - template - void debug_slab_invariant(CapPtr slab, LocalEntropy& entropy) - { - static_assert(B == CBChunkD || B == CBChunk); - -#if !defined(NDEBUG) && !defined(SNMALLOC_CHEAP_CHECKS) - bool is_short = Metaslab::is_short(slab); - - if (is_full()) - { - size_t count = free_queue.debug_length(entropy); - SNMALLOC_ASSERT(count < threshold_for_waking_slab(is_short)); - return; - } - - if (is_unused()) - return; - - size_t size = sizeclass_to_size(sizeclass()); - size_t offset = get_initial_offset(sizeclass(), is_short); - size_t accounted_for = needed() * size + offset; + return p; + } + }; - // Block is not full - SNMALLOC_ASSERT(SLAB_SIZE > accounted_for); + struct RemoteAllocator; - // Account for list size - size_t count = free_queue.debug_length(entropy); - accounted_for += count * size; + /** + * Entry stored in the pagemap. + */ + class MetaEntry + { + Metaslab* meta{nullptr}; + uintptr_t remote_and_sizeclass{0}; - SNMALLOC_ASSERT(count <= get_slab_capacity(sizeclass(), is_short)); + public: + constexpr MetaEntry() = default; - auto bumpptr = (get_slab_capacity(sizeclass(), is_short) * size) + offset; - // Check we haven't allocated more than fits in a slab - SNMALLOC_ASSERT(bumpptr <= SLAB_SIZE); + MetaEntry(Metaslab* meta, RemoteAllocator* remote, sizeclass_t sizeclass) + : meta(meta) + { + remote_and_sizeclass = + address_cast(pointer_offset(remote, sizeclass)); + } - // Account for to be bump allocated space - accounted_for += SLAB_SIZE - bumpptr; + [[nodiscard]] Metaslab* get_metaslab() const + { + return meta; + } - SNMALLOC_ASSERT(!is_full()); + [[nodiscard]] RemoteAllocator* get_remote() const + { + return reinterpret_cast( + bits::align_down(remote_and_sizeclass, alignof(RemoteAllocator))); + } - // All space accounted for - SNMALLOC_ASSERT(SLAB_SIZE == accounted_for); -#else - UNUSED(slab); - UNUSED(entropy); -#endif + [[nodiscard]] sizeclass_t get_sizeclass() const + { + return remote_and_sizeclass & (alignof(RemoteAllocator) - 1); } }; + + struct MetaslabCache : public CDLLNode<> + { + uint16_t unused; + uint16_t length; + }; + } // namespace snmalloc diff --git a/src/mem/pagemap.h b/src/mem/pagemap.h deleted file mode 100644 index 5f0972cf4..000000000 --- a/src/mem/pagemap.h +++ /dev/null @@ -1,521 +0,0 @@ -#pragma once - -#include "../ds/bits.h" -#include "../ds/helpers.h" -#include "../ds/invalidptr.h" - -#include -#include - -namespace snmalloc -{ - static constexpr size_t PAGEMAP_NODE_BITS = 16; - static constexpr size_t PAGEMAP_NODE_SIZE = 1ULL << PAGEMAP_NODE_BITS; - - /** - * Structure describing the configuration of a pagemap. When querying a - * pagemap from a different instantiation of snmalloc, the pagemap is exposed - * as a `void*`. This structure allows the caller to check whether the - * pagemap is of the format that they expect. - */ - struct PagemapConfig - { - /** - * The version of the pagemap structure. This is always 1 in existing - * versions of snmalloc. This will be incremented every time the format - * changes in an incompatible way. Changes to the format may add fields to - * the end of this structure. - */ - uint32_t version; - /** - * Is this a flat pagemap? If this field is false, the pagemap is the - * hierarchical structure. - */ - bool is_flat_pagemap; - /** - * Number of bytes in a pointer. - */ - uint8_t sizeof_pointer; - /** - * The number of bits of the address used to index into the pagemap. - */ - uint64_t pagemap_bits; - /** - * The size (in bytes) of a pagemap entry. - */ - size_t size_of_entry; - }; - - /** - * The Pagemap is the shared data structure ultimately used by multiple - * snmalloc threads / allocators to determine who owns memory and, - * therefore, to whom deallocated memory should be returned. The - * allocators do not interact with this directly but rather via the - * static ChunkMap object, which encapsulates knowledge about the - * pagemap's parametric type T. - * - * The other template paramters are... - * - * GRANULARITY_BITS: the log2 of the size in bytes of the address space - * granule associated with each entry. - * - * default_content: An initial value of T (typically "0" or something akin) - * - * PrimAlloc: A class used to source PageMap-internal memory; it must have a - * method callable as if it had the following type: - * - * template static T* alloc_chunk(void); - */ - template< - size_t GRANULARITY_BITS, - typename T, - T default_content, - typename PrimAlloc> - class Pagemap - { - private: - static constexpr size_t COVERED_BITS = - bits::ADDRESS_BITS - GRANULARITY_BITS; - static constexpr size_t CONTENT_BITS = - bits::next_pow2_bits_const(sizeof(T)); - - static_assert( - PAGEMAP_NODE_BITS - CONTENT_BITS < COVERED_BITS, - "Should use the FlatPageMap as it does not require a tree"); - - static constexpr size_t BITS_FOR_LEAF = PAGEMAP_NODE_BITS - CONTENT_BITS; - static constexpr size_t ENTRIES_PER_LEAF = 1 << BITS_FOR_LEAF; - static constexpr size_t LEAF_MASK = ENTRIES_PER_LEAF - 1; - - static constexpr size_t BITS_PER_INDEX_LEVEL = - PAGEMAP_NODE_BITS - POINTER_BITS; - static constexpr size_t ENTRIES_PER_INDEX_LEVEL = 1 << BITS_PER_INDEX_LEVEL; - static constexpr size_t ENTRIES_MASK = ENTRIES_PER_INDEX_LEVEL - 1; - - static constexpr size_t INDEX_BITS = - BITS_FOR_LEAF > COVERED_BITS ? 0 : COVERED_BITS - BITS_FOR_LEAF; - - static constexpr size_t INDEX_LEVELS = INDEX_BITS / BITS_PER_INDEX_LEVEL; - static constexpr size_t TOPLEVEL_BITS = - INDEX_BITS - (INDEX_LEVELS * BITS_PER_INDEX_LEVEL); - static constexpr size_t TOPLEVEL_ENTRIES = 1 << TOPLEVEL_BITS; - static constexpr size_t TOPLEVEL_SHIFT = - (INDEX_LEVELS * BITS_PER_INDEX_LEVEL) + BITS_FOR_LEAF + GRANULARITY_BITS; - - // Value used to represent when a node is being added too - static constexpr InvalidPointer<1> LOCKED_ENTRY{}; - - struct Leaf - { - TrivialInitAtomic values[ENTRIES_PER_LEAF]; - - static_assert(sizeof(TrivialInitAtomic) == sizeof(T)); - static_assert(alignof(TrivialInitAtomic) == alignof(T)); - }; - - struct PagemapEntry - { - TrivialInitAtomic entries[ENTRIES_PER_INDEX_LEVEL]; - - static_assert( - sizeof(TrivialInitAtomic) == sizeof(PagemapEntry*)); - static_assert( - alignof(TrivialInitAtomic) == alignof(PagemapEntry*)); - }; - - static_assert( - sizeof(PagemapEntry) == sizeof(Leaf), "Should be the same size"); - - static_assert( - sizeof(PagemapEntry) == PAGEMAP_NODE_SIZE, "Should be the same size"); - - // Init removed as not required as this is only ever a global - // cl is generating a memset of zero, which will be a problem - // in libc/ucrt bring up. On ucrt this will run after the first - // allocation. - // TODO: This is fragile that it is not being memset, and we should review - // to ensure we don't get bitten by this in the future. - TrivialInitAtomic top[TOPLEVEL_ENTRIES]; - - template - SNMALLOC_FAST_PATH PagemapEntry* - get_node(TrivialInitAtomic* e, bool& result) - { - // The page map nodes are all allocated directly from the OS zero - // initialised with a system call. We don't need any ordered to guarantee - // to see that correctly. The only transistions are monotone and handled - // by the slow path. - PagemapEntry* value = e->load(std::memory_order_relaxed); - - if (likely(value > LOCKED_ENTRY)) - { - result = true; - return value; - } - if constexpr (create_addr) - { - return get_node_slow(e, result); - } - else - { - result = false; - return nullptr; - } - } - - SNMALLOC_SLOW_PATH PagemapEntry* - get_node_slow(TrivialInitAtomic* e, bool& result) - { - // The page map nodes are all allocated directly from the OS zero - // initialised with a system call. We don't need any ordered to guarantee - // to see that correctly. - PagemapEntry* value = e->load(std::memory_order_relaxed); - - if ((value == nullptr) || (value == LOCKED_ENTRY)) - { - value = nullptr; - - if (e->compare_exchange_strong( - value, LOCKED_ENTRY, std::memory_order_relaxed)) - { - value = PrimAlloc::template alloc_chunk(); - e->store(value, std::memory_order_release); - } - else - { - while (address_cast(e->load(std::memory_order_relaxed)) == - LOCKED_ENTRY) - { - Aal::pause(); - } - value = e->load(std::memory_order_acquire); - } - } - result = true; - return value; - } - - template - SNMALLOC_FAST_PATH std::pair - get_leaf_index(uintptr_t addr, bool& result) - { -#ifdef FreeBSD_KERNEL - // Zero the top 16 bits - kernel addresses all have them set, but the - // data structure assumes that they're zero. - addr &= 0xffffffffffffULL; -#endif - size_t ix = addr >> TOPLEVEL_SHIFT; - size_t shift = TOPLEVEL_SHIFT; - TrivialInitAtomic* e = &top[ix]; - - // This is effectively a - // for (size_t i = 0; i < INDEX_LEVELS; i++) - // loop, but uses constexpr to guarantee optimised version - // where the INDEX_LEVELS in {0,1}. - if constexpr (INDEX_LEVELS != 0) - { - size_t i = 0; - while (true) - { - PagemapEntry* value = get_node(e, result); - if (unlikely(!result)) - return {nullptr, 0}; - - shift -= BITS_PER_INDEX_LEVEL; - ix = (static_cast(addr) >> shift) & ENTRIES_MASK; - e = &value->entries[ix]; - - if constexpr (INDEX_LEVELS == 1) - { - UNUSED(i); - break; - } - else - { - i++; - if (i == INDEX_LEVELS) - break; - } - } - } - - Leaf* leaf = reinterpret_cast(get_node(e, result)); - - if (unlikely(!result)) - return {nullptr, 0}; - - shift -= BITS_FOR_LEAF; - ix = (static_cast(addr) >> shift) & LEAF_MASK; - return {leaf, ix}; - } - - template - SNMALLOC_FAST_PATH TrivialInitAtomic* - get_addr(uintptr_t p, bool& success) - { - auto leaf_ix = get_leaf_index(p, success); - return &(leaf_ix.first->values[leaf_ix.second]); - } - - TrivialInitAtomic* get_ptr(uintptr_t p) - { - bool success; - return get_addr(p, success); - } - - public: - /** - * The pagemap configuration describing this instantiation of the template. - */ - static constexpr PagemapConfig config = { - 1, false, sizeof(uintptr_t), GRANULARITY_BITS, sizeof(T)}; - - /** - * Cast a `void*` to a pointer to this template instantiation, given a - * config describing the configuration. Return null if the configuration - * passed does not correspond to this template instantiation. - * - * This intended to allow code that depends on the pagemap having a - * specific representation to fail gracefully. - */ - static Pagemap* cast_to_pagemap(void* pm, const PagemapConfig* c) - { - if ( - (c->version != 1) || (c->is_flat_pagemap) || - (c->sizeof_pointer != sizeof(uintptr_t)) || - (c->pagemap_bits != GRANULARITY_BITS) || - (c->size_of_entry != sizeof(T)) || (!std::is_integral_v)) - { - return nullptr; - } - return static_cast(pm); - } - - /** - * Returns the index of a pagemap entry within a given page. This is used - * in code that propagates changes to the pagemap elsewhere. - */ - size_t index_for_address(uintptr_t p) - { - bool success; - return (OS_PAGE_SIZE - 1) & - reinterpret_cast(get_addr(p, success)); - } - - /** - * Returns the address of the page containing - */ - void* page_for_address(uintptr_t p) - { - bool success; - return pointer_align_down(get_addr(p, success)); - } - - T get(uintptr_t p) - { - bool success; - auto addr = get_addr(p, success); - if (!success) - return default_content; - return addr->load(std::memory_order_relaxed); - } - - void set(uintptr_t p, T x) - { - bool success; - auto addr = get_addr(p, success); - addr->store(x, std::memory_order_relaxed); - } - - void set_range(uintptr_t p, T x, size_t length) - { - bool success; - do - { - auto leaf_ix = get_leaf_index(p, success); - size_t ix = leaf_ix.second; - - auto last = bits::min(LEAF_MASK + 1, ix + length); - - auto diff = last - ix; - - for (; ix < last; ix++) - { - SNMALLOC_ASSUME(leaf_ix.first != nullptr); - leaf_ix.first->values[ix].store(x); - } - - length = length - diff; - p = p + (diff << GRANULARITY_BITS); - } while (length > 0); - } - }; - - /** - * Simple pagemap that for each GRANULARITY_BITS of the address range - * stores a T. - */ - template - class alignas(OS_PAGE_SIZE) FlatPagemap - { - private: - static constexpr size_t COVERED_BITS = - bits::ADDRESS_BITS - GRANULARITY_BITS; - static constexpr size_t ENTRIES = 1ULL << COVERED_BITS; - static constexpr size_t SHIFT = GRANULARITY_BITS; - - TrivialInitAtomic top[ENTRIES]; - - static_assert(sizeof(TrivialInitAtomic) == sizeof(T)); - static_assert(alignof(TrivialInitAtomic) == alignof(T)); - - public: - /** - * The pagemap configuration describing this instantiation of the template. - */ - static constexpr PagemapConfig config = { - 1, true, sizeof(uintptr_t), GRANULARITY_BITS, sizeof(T)}; - - /** - * Cast a `void*` to a pointer to this template instantiation, given a - * config describing the configuration. Return null if the configuration - * passed does not correspond to this template instantiation. - * - * This intended to allow code that depends on the pagemap having a - * specific representation to fail gracefully. - */ - static FlatPagemap* cast_to_pagemap(void* pm, const PagemapConfig* c) - { - if ( - (c->version != 1) || (!c->is_flat_pagemap) || - (c->sizeof_pointer != sizeof(uintptr_t)) || - (c->pagemap_bits != GRANULARITY_BITS) || - (c->size_of_entry != sizeof(T)) || (!std::is_integral_v)) - { - return nullptr; - } - return static_cast(pm); - } - - T get(uintptr_t p) - { - return top[p >> SHIFT].load(std::memory_order_relaxed); - } - - void set(uintptr_t p, T x) - { - top[p >> SHIFT].store(x, std::memory_order_relaxed); - } - - void set_range(uintptr_t p, T x, size_t length) - { - size_t index = p >> SHIFT; - do - { - top[index].store(x, std::memory_order_relaxed); - index++; - length--; - } while (length > 0); - } - - /** - * Returns the index within a page for the specified address. - */ - size_t index_for_address(uintptr_t p) - { - return (static_cast(p) >> SHIFT) % OS_PAGE_SIZE; - } - - /** - * Returns the address of the page containing the pagemap address p. - */ - void* page_for_address(uintptr_t p) - { - SNMALLOC_ASSERT( - (reinterpret_cast(&top) & (OS_PAGE_SIZE - 1)) == 0); - return reinterpret_cast( - reinterpret_cast(&top[p >> SHIFT]) & ~(OS_PAGE_SIZE - 1)); - } - }; - - /** - * Mixin used by `ChunkMap` and other `PageMap` consumers to directly access - * the pagemap via a global variable. This should be used from within the - * library or program that owns the pagemap. - * - * This class makes the global pagemap a static field so that its name - * includes the type mangling. If two compilation units try to instantiate - * two different types of pagemap then they will see two distinct pagemaps. - * This will prevent allocating with one and freeing with the other (because - * the memory will show up as not owned by any allocator in the other - * compilation unit) but will prevent the same memory being interpreted as - * having two different types. - * - * Simiarly, perhaps two modules wish to instantiate *different* pagemaps - * of the *same* type. Therefore, we add a `Purpose` parameter that can be - * used to pry symbols apart. By default, the `Purpose` is just the type of - * the pagemap; that is, pagemaps default to discrimination solely by their - * type. - */ - template - class GlobalPagemapTemplate - { - /** - * The global pagemap variable. The name of this symbol will include the - * type of `T` and `U`. - */ - inline static T global_pagemap; - - public: - /** - * Returns the pagemap. - */ - static T& pagemap() - { - return global_pagemap; - } - }; - - /** - * Mixin used by `ChunkMap` and other `PageMap` consumers to access the global - * pagemap via a type-checked C interface. This should be used when another - * library (e.g. your C standard library) uses snmalloc and you wish to use a - * different configuration in your program or library, but wish to share a - * pagemap so that either version can deallocate memory. - * - * The `Purpose` parameter is as with `GlobalPgemapTemplate`. - */ - template< - typename T, - void* (*raw_get)(const PagemapConfig**), - typename Purpose = T> - class ExternalGlobalPagemapTemplate - { - /** - * A pointer to the pagemap. - */ - inline static T* external_pagemap; - - public: - /** - * Returns the exported pagemap. - * Accesses the pagemap via the C ABI accessor and casts it to - * the expected type, failing in cases of ABI mismatch. - */ - static T& pagemap() - { - if (external_pagemap == nullptr) - { - const snmalloc::PagemapConfig* c = nullptr; - void* raw_pagemap = raw_get(&c); - external_pagemap = T::cast_to_pagemap(raw_pagemap, c); - if (!external_pagemap) - { - Pal::error("Incorrect ABI of global pagemap."); - } - } - return *external_pagemap; - } - }; - -} // namespace snmalloc diff --git a/src/mem/pool.h b/src/mem/pool.h index e9e4c9993..e57dbc5ab 100644 --- a/src/mem/pool.h +++ b/src/mem/pool.h @@ -17,39 +17,30 @@ namespace snmalloc * * This is used to bootstrap the allocation of allocators. */ - template - class Pool + template + class PoolState { - private: - friend Pooled; - template - friend class MemoryProviderStateMixin; - friend SNMALLOC_DEFAULT_MEMORY_PROVIDER; + template + friend class Pool; + private: std::atomic_flag lock = ATOMIC_FLAG_INIT; MPMCStack stack; - T* list = nullptr; - - Pool(MemoryProvider& m) : memory_provider(m) {} + T* list{nullptr}; public: - MemoryProvider& memory_provider; - - static Pool* make(MemoryProvider& memory_provider) noexcept - { - return memory_provider.template alloc_chunk( - memory_provider); - } - - static Pool* make() noexcept - { - return Pool::make(default_memory_provider()); - } + constexpr PoolState() = default; + }; - template - T* acquire(Args&&... args) + template + class Pool + { + public: + template + static T* acquire(SharedStateHandle h, Args&&... args) { - T* p = stack.pop(); + PoolState& pool = h.pool(); + T* p = pool.stack.pop(); if (p != nullptr) { @@ -57,13 +48,12 @@ namespace snmalloc return p; } - p = memory_provider - .template alloc_chunk( - std::forward(args)...); + p = ChunkAllocator::alloc_meta_data( + h, nullptr, std::forward(args)...); - FlagLock f(lock); - p->list_next = list; - list = p; + FlagLock f(pool.lock); + p->list_next = pool.list; + pool.list = p; p->set_in_use(); return p; @@ -74,20 +64,22 @@ namespace snmalloc * * Do not return objects from `extract`. */ - void release(T* p) + template + static void release(SharedStateHandle h, T* p) { // The object's destructor is not run. If the object is "reallocated", it // is returned without the constructor being run, so the object is reused // without re-initialisation. p->reset_in_use(); - stack.push(p); + h.pool().stack.push(p); } - T* extract(T* p = nullptr) + template + static T* extract(SharedStateHandle h, T* p = nullptr) { // Returns a linked list of all objects in the stack, emptying the stack. if (p == nullptr) - return stack.pop_all(); + return h.pool().stack.pop_all(); return p->next; } @@ -97,17 +89,19 @@ namespace snmalloc * * Do not return objects from `acquire`. */ - void restore(T* first, T* last) + template + static void restore(SharedStateHandle h, T* first, T* last) { // Pushes a linked list of objects onto the stack. Use to put a linked // list returned by extract back onto the stack. - stack.push(first, last); + h.pool().stack.push(first, last); } - T* iterate(T* p = nullptr) + template + static T* iterate(SharedStateHandle h, T* p = nullptr) { if (p == nullptr) - return list; + return h.pool().list; return p->list_next; } diff --git a/src/mem/pooled.h b/src/mem/pooled.h index a4ffa1e3b..93daad9f3 100644 --- a/src/mem/pooled.h +++ b/src/mem/pooled.h @@ -8,15 +8,9 @@ namespace snmalloc class Pooled { private: - template + template friend class Pool; - template< - class a, - Construction c, - template - typename P, - template - typename AP> + template friend class MPMCStack; /// Used by the pool for chaining together entries when not in use. diff --git a/src/mem/ptrhelpers.h b/src/mem/ptrhelpers.h index e43119d13..8b0cee925 100644 --- a/src/mem/ptrhelpers.h +++ b/src/mem/ptrhelpers.h @@ -57,7 +57,7 @@ namespace snmalloc #endif { UNUSED(sz); - return CapPtr()>(p.unsafe_capptr); + return CapPtr()>(p.unsafe_ptr()); } } @@ -78,7 +78,7 @@ namespace snmalloc #ifndef NDEBUG // On debug builds, CBChunkD are already bounded as if CBChunk. UNUSED(sz); - return CapPtr(p.unsafe_capptr); + return CapPtr(p.unsafe_ptr()); #else // On non-debug builds, apply bounds now, as they haven't been already. return Aal::capptr_bound(p, sz); @@ -93,6 +93,6 @@ namespace snmalloc SNMALLOC_FAST_PATH CapPtr capptr_debug_chunkd_from_chunk(CapPtr p) { - return CapPtr(p.unsafe_capptr); + return CapPtr(p.unsafe_ptr()); } } // namespace snmalloc diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index 2c04f1f59..c639e5c47 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -3,9 +3,10 @@ #include "../ds/mpscq.h" #include "../mem/allocconfig.h" #include "../mem/freelist.h" -#include "../mem/sizeclass.h" -#include "../mem/superslab.h" +#include "../mem/metaslab.h" +#include "../mem/sizeclasstable.h" +#include #include #ifdef CHECK_CLIENT @@ -28,75 +29,7 @@ namespace snmalloc AtomicCapPtr next{nullptr}; }; -#ifdef SNMALLOC_DONT_CACHE_ALLOCATOR_PTR - /** - * Cache the size class of the object to improve performance. - * - * This implementation does not cache the allocator id due to security - * concerns. Alternative implementations may store the allocator - * id, so that amplification costs can be mitigated on CHERI with MTE. - */ - sizeclass_t sizeclasscache; -#else - /* This implementation assumes that storing the allocator ID in a freed - * object is not a security concern. Either we trust the code running on - * top of the allocator, or additional security measure are in place such - * as MTE + CHERI. - * - * We embed the size class in the bottom 8 bits of an allocator ID (i.e., - * the address of an Alloc's remote_alloc's message_queue; in practice we - * only need 7 bits, but using 8 is conjectured to be faster). The hashing - * algorithm of the Alloc's RemoteCache already ignores the bottom - * "initial_shift" bits, which is, in practice, well above 8. There's a - * static_assert() over there that helps ensure this stays true. - * - * This does mean that we might have message_queues that always collide in - * the hash algorithm, if they're within "initial_shift" of each other. Such - * pairings will substantially decrease performance and so we prohibit them - * and use SNMALLOC_ASSERT to verify that they do not exist in debug builds. - */ - alloc_id_t alloc_id_and_sizeclass; -#endif - - /** - * Set up a remote object. Potentially cache sizeclass and allocator id. - */ - void set_info(alloc_id_t id, sizeclass_t sc) - { -#ifdef SNMALLOC_DONT_CACHE_ALLOCATOR_PTR - UNUSED(id); - sizeclasscache = sc; -#else - alloc_id_and_sizeclass = (id & ~SIZECLASS_MASK) | sc; -#endif - } - - /** - * Return allocator for this object. This may perform amplification. - */ - template - static alloc_id_t - trunc_target_id(CapPtr r, LargeAlloc* large_allocator) - { -#ifdef SNMALLOC_DONT_CACHE_ALLOCATOR_PTR - // Rederive allocator id. - auto r_auth = large_allocator->template capptr_amplify(r); - auto super = Superslab::get(r_auth); - return super->get_allocator()->trunc_id(); -#else - UNUSED(large_allocator); - return r->alloc_id_and_sizeclass & ~SIZECLASS_MASK; -#endif - } - - sizeclass_t sizeclass() - { -#ifdef SNMALLOC_DONT_CACHE_ALLOCATOR_PTR - return sizeclasscache; -#else - return alloc_id_and_sizeclass & SIZECLASS_MASK; -#endif - } + constexpr Remote() : next(nullptr) {} /** Zero out a Remote tracking structure, return pointer to object base */ template @@ -111,13 +44,18 @@ namespace snmalloc sizeof(Remote) <= MIN_ALLOC_SIZE, "Needs to be able to fit in smallest allocation."); - struct RemoteAllocator + // Remotes need to be aligned enough that all the + // small size classes can fit in the bottom bits. + static constexpr size_t REMOTE_MIN_ALIGN = bits::min( + CACHELINE_SIZE, bits::next_pow2_const(NUM_SIZECLASSES + 1)); + + struct alignas(REMOTE_MIN_ALIGN) RemoteAllocator { using alloc_id_t = Remote::alloc_id_t; // Store the message queue on a separate cacheline. It is mutable data that // is read by other threads. - alignas(CACHELINE_SIZE) - MPSCQ message_queue; + + MPSCQ message_queue; alloc_id_t trunc_id() { @@ -126,138 +64,4 @@ namespace snmalloc ~SIZECLASS_MASK; } }; - - /* - * A singly-linked list of Remote objects, supporting append and - * take-all operations. Intended only for the private use of this - * allocator; the Remote objects here will later be taken and pushed - * to the inter-thread message queues. - */ - struct RemoteList - { - /* - * A stub Remote object that will always be the head of this list; - * never taken for further processing. - */ - Remote head{}; - - CapPtr last{&head}; - - void clear() - { - last = CapPtr(&head); - } - - bool empty() - { - return address_cast(last) == address_cast(&head); - } - }; - - struct RemoteCache - { - /** - * The total amount of memory we are waiting for before we will dispatch - * to other allocators. Zero or negative mean we should dispatch on the - * next remote deallocation. This is initialised to the 0 so that we - * always hit a slow path to start with, when we hit the slow path and - * need to dispatch everything, we can check if we are a real allocator - * and lazily provide a real allocator. - */ - int64_t capacity{0}; - std::array list{}; - - /// Used to find the index into the array of queues for remote - /// deallocation - /// r is used for which round of sending this is. - template - inline size_t get_slot(size_t id, size_t r) - { - constexpr size_t allocator_size = sizeof(Alloc); - constexpr size_t initial_shift = - bits::next_pow2_bits_const(allocator_size); - static_assert( - initial_shift >= 8, - "Can't embed sizeclass_t into allocator ID low bits"); - SNMALLOC_ASSERT((initial_shift + (r * REMOTE_SLOT_BITS)) < 64); - return (id >> (initial_shift + (r * REMOTE_SLOT_BITS))) & REMOTE_MASK; - } - - template - SNMALLOC_FAST_PATH void dealloc( - Remote::alloc_id_t target_id, - CapPtr p, - sizeclass_t sizeclass) - { - this->capacity -= sizeclass_to_size(sizeclass); - auto r = p.template as_reinterpret(); - - r->set_info(target_id, sizeclass); - - RemoteList* l = &list[get_slot(target_id, 0)]; - l->last->non_atomic_next = r; - l->last = r; - } - - template - void post(Alloc* allocator, Remote::alloc_id_t id) - { - // When the cache gets big, post lists to their target allocators. - capacity = REMOTE_CACHE; - - size_t post_round = 0; - - while (true) - { - auto my_slot = get_slot(id, post_round); - - for (size_t i = 0; i < REMOTE_SLOTS; i++) - { - if (i == my_slot) - continue; - - RemoteList* l = &list[i]; - CapPtr first = l->head.non_atomic_next; - - if (!l->empty()) - { - // Send all slots to the target at the head of the list. - auto first_auth = - allocator->large_allocator.template capptr_amplify(first); - auto super = Superslab::get(first_auth); - super->get_allocator()->message_queue.enqueue(first, l->last); - l->clear(); - } - } - - RemoteList* resend = &list[my_slot]; - if (resend->empty()) - break; - - // Entries could map back onto the "resend" list, - // so take copy of the head, mark the last element, - // and clear the original list. - CapPtr r = resend->head.non_atomic_next; - resend->last->non_atomic_next = nullptr; - resend->clear(); - - post_round++; - - while (r != nullptr) - { - // Use the next N bits to spread out remote deallocs in our own - // slot. - size_t slot = get_slot( - Remote::trunc_target_id(r, &allocator->large_allocator), - post_round); - RemoteList* l = &list[slot]; - l->last->non_atomic_next = r; - l->last = r; - - r = r->non_atomic_next; - } - } - } - }; - } // namespace snmalloc diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h new file mode 100644 index 000000000..a017b674a --- /dev/null +++ b/src/mem/remotecache.h @@ -0,0 +1,199 @@ +#pragma once + +#include "../ds/mpscq.h" +#include "../mem/allocconfig.h" +#include "../mem/freelist.h" +#include "../mem/metaslab.h" +#include "../mem/remoteallocator.h" +#include "../mem/sizeclasstable.h" + +#include +#include + +namespace snmalloc +{ + /** + * Stores the remote deallocation to batch them before sending + */ + struct RemoteDeallocCache + { + /* + * A singly-linked list of Remote objects, supporting append and + * take-all operations. Intended only for the private use of this + * allocator; the Remote objects here will later be taken and pushed + * to the inter-thread message queues. + */ + struct RemoteList + { + /* + * A stub Remote object that will always be the head of this list; + * never taken for further processing. + */ + Remote head{}; + + /** + * Initially is null ptr, and needs to be non-null before anything runs on + * this. + */ + CapPtr last{nullptr}; + + void clear() + { + last = CapPtr(&head); + } + + bool empty() + { + return address_cast(last) == address_cast(&head); + } + + constexpr RemoteList() = default; + }; + + std::array list{}; + + /** + * The total amount of memory we are waiting for before we will dispatch + * to other allocators. Zero can mean we have not initialised the allocator + * yet. This is initialised to the 0 so that we always hit a slow path to + * start with, when we hit the slow path and need to dispatch everything, we + * can check if we are a real allocator and lazily provide a real allocator. + */ + int64_t capacity{0}; + +#ifndef NDEBUG + bool initialised = false; +#endif + + /// Used to find the index into the array of queues for remote + /// deallocation + /// r is used for which round of sending this is. + template + inline size_t get_slot(size_t i, size_t r) + { + constexpr size_t initial_shift = + bits::next_pow2_bits_const(allocator_size); + // static_assert( + // initial_shift >= 8, + // "Can't embed sizeclass_t into allocator ID low bits"); + SNMALLOC_ASSERT((initial_shift + (r * REMOTE_SLOT_BITS)) < 64); + return (i >> (initial_shift + (r * REMOTE_SLOT_BITS))) & REMOTE_MASK; + } + + /** + * Checks if the capacity has enough to cache an entry from this + * slab. Returns true, if this does not overflow the budget. + * + * This does not require initialisation to be safely called. + */ + SNMALLOC_FAST_PATH bool reserve_space(const MetaEntry& entry) + { + auto size = + static_cast(sizeclass_to_size(entry.get_sizeclass())); + + bool result = capacity > size; + if (result) + capacity -= size; + return result; + } + + template + SNMALLOC_FAST_PATH void + dealloc(Remote::alloc_id_t target_id, CapPtr p) + { + SNMALLOC_ASSERT(initialised); + auto r = p.template as_reinterpret(); + + RemoteList* l = &list[get_slot(target_id, 0)]; + l->last->non_atomic_next = r; + l->last = r; + } + + template + bool post(SharedStateHandle handle, Remote::alloc_id_t id) + { + SNMALLOC_ASSERT(initialised); + size_t post_round = 0; + bool sent_something = false; + + while (true) + { + auto my_slot = get_slot(id, post_round); + + for (size_t i = 0; i < REMOTE_SLOTS; i++) + { + if (i == my_slot) + continue; + + RemoteList* l = &list[i]; + CapPtr first = l->head.non_atomic_next; + + if (!l->empty()) + { + MetaEntry entry = SharedStateHandle::Backend::get_meta_data( + handle.get_backend_state(), address_cast(first)); + entry.get_remote()->message_queue.enqueue(first, l->last); + l->clear(); + sent_something = true; + } + } + + RemoteList* resend = &list[my_slot]; + if (resend->empty()) + break; + + // Entries could map back onto the "resend" list, + // so take copy of the head, mark the last element, + // and clear the original list. + CapPtr r = resend->head.non_atomic_next; + resend->last->non_atomic_next = nullptr; + resend->clear(); + + post_round++; + + while (r != nullptr) + { + // Use the next N bits to spread out remote deallocs in our own + // slot. + MetaEntry entry = SharedStateHandle::Backend::get_meta_data( + handle.get_backend_state(), address_cast(r)); + auto i = entry.get_remote()->trunc_id(); + // TODO correct size for slot offset + size_t slot = get_slot(i, post_round); + RemoteList* l = &list[slot]; + l->last->non_atomic_next = r; + l->last = r; + + r = r->non_atomic_next; + } + } + + // Reset capacity as we have empty everything + capacity = REMOTE_CACHE; + + return sent_something; + } + + /** + * Constructor design to allow constant init + */ + constexpr RemoteDeallocCache() = default; + + /** + * Must be called before anything else to ensure actually initialised + * not just zero init. + */ + void init() + { +#ifndef NDEBUG + initialised = true; +#endif + for (auto& l : list) + { + SNMALLOC_ASSERT(l.last == nullptr || l.empty()); + l.clear(); + } + capacity = REMOTE_CACHE; + } + }; +} // namespace snmalloc diff --git a/src/mem/slowalloc.h b/src/mem/scopedalloc.h similarity index 67% rename from src/mem/slowalloc.h rename to src/mem/scopedalloc.h index 87be4d10a..d0e179e20 100644 --- a/src/mem/slowalloc.h +++ b/src/mem/scopedalloc.h @@ -1,6 +1,8 @@ #pragma once -#include "globalalloc.h" +/** + * This header requires that Alloc has been defined. + */ namespace snmalloc { @@ -13,57 +15,65 @@ namespace snmalloc * This does not depend on thread-local storage working, so can be used for * bootstrapping. */ - struct SlowAllocator + struct ScopedAllocator { /** * The allocator that this wrapper will use. */ - Alloc* alloc; + Alloc alloc; + /** * Constructor. Claims an allocator from the global pool */ - SlowAllocator() : alloc(current_alloc_pool()->acquire()) {} + ScopedAllocator() = default; + /** * Copying is not supported, it could easily lead to accidental sharing of * allocators. */ - SlowAllocator(const SlowAllocator&) = delete; + ScopedAllocator(const ScopedAllocator&) = delete; + /** * Moving is not supported, though it would be easy to add if there's a use * case for it. */ - SlowAllocator(SlowAllocator&&) = delete; + ScopedAllocator(ScopedAllocator&&) = delete; + /** * Copying is not supported, it could easily lead to accidental sharing of * allocators. */ - SlowAllocator& operator=(const SlowAllocator&) = delete; + ScopedAllocator& operator=(const ScopedAllocator&) = delete; + /** * Moving is not supported, though it would be easy to add if there's a use * case for it. */ - SlowAllocator& operator=(SlowAllocator&&) = delete; + ScopedAllocator& operator=(ScopedAllocator&&) = delete; + /** * Destructor. Returns the allocator to the pool. */ - ~SlowAllocator() + ~ScopedAllocator() { - current_alloc_pool()->release(alloc); + alloc.flush(); } + /** * Arrow operator, allows methods exposed by `Alloc` to be called on the * wrapper. */ Alloc* operator->() { - return alloc; + return &alloc; } }; + /** - * Returns a new slow allocator. When the `SlowAllocator` goes out of scope, - * the underlying `Alloc` will be returned to the pool. + * Returns a new scoped allocator. When the `ScopedAllocator` goes out of + * scope, the underlying `Alloc` will be returned to the pool. */ - inline SlowAllocator get_slow_allocator() + inline ScopedAllocator get_scoped_allocator() { return {}; } diff --git a/src/mem/sizeclass.h b/src/mem/sizeclass.h deleted file mode 100644 index 1a94cb573..000000000 --- a/src/mem/sizeclass.h +++ /dev/null @@ -1,89 +0,0 @@ -#pragma once - -#include "../pal/pal.h" -#include "allocconfig.h" - -namespace snmalloc -{ - // Both usings should compile - // We use size_t as it generates better code. - using sizeclass_t = size_t; - // using sizeclass_t = uint8_t; - using sizeclass_compress_t = uint8_t; - - constexpr static uintptr_t SIZECLASS_MASK = 0xFF; - - constexpr static uint16_t get_initial_offset(sizeclass_t sc, bool is_short); - constexpr static uint16_t get_slab_capacity(sizeclass_t sc, bool is_short); - - constexpr static size_t sizeclass_to_size(sizeclass_t sizeclass); - constexpr static uint16_t medium_slab_free(sizeclass_t sizeclass); - static sizeclass_t size_to_sizeclass(size_t size); - - constexpr static inline sizeclass_t size_to_sizeclass_const(size_t size) - { - // Don't use sizeclasses that are not a multiple of the alignment. - // For example, 24 byte allocations can be - // problematic for some data due to alignment issues. - auto sc = static_cast( - bits::to_exp_mant_const(size)); - - SNMALLOC_ASSERT(sc == static_cast(sc)); - - return sc; - } - - constexpr static inline size_t large_sizeclass_to_size(uint8_t large_class) - { - return bits::one_at_bit(large_class + SUPERSLAB_BITS); - } - - // Small classes range from [MIN, SLAB], i.e. inclusive. - static constexpr size_t NUM_SMALL_CLASSES = - size_to_sizeclass_const(bits::one_at_bit(SLAB_BITS)) + 1; - - static constexpr size_t NUM_SIZECLASSES = - size_to_sizeclass_const(SUPERSLAB_SIZE); - - // Medium classes range from (SLAB, SUPERSLAB), i.e. non-inclusive. - static constexpr size_t NUM_MEDIUM_CLASSES = - NUM_SIZECLASSES - NUM_SMALL_CLASSES; - - // Large classes range from [SUPERSLAB, ADDRESS_SPACE). - static constexpr size_t NUM_LARGE_CLASSES = - bits::ADDRESS_BITS - SUPERSLAB_BITS; - - SNMALLOC_FAST_PATH static size_t aligned_size(size_t alignment, size_t size) - { - // Client responsible for checking alignment is not zero - SNMALLOC_ASSERT(alignment != 0); - // Client responsible for checking alignment is a power of two - SNMALLOC_ASSERT(bits::is_pow2(alignment)); - - return ((alignment - 1) | (size - 1)) + 1; - } - - SNMALLOC_FAST_PATH static size_t round_size(size_t size) - { - if (size > sizeclass_to_size(NUM_SIZECLASSES - 1)) - { - return bits::next_pow2(size); - } - if (size == 0) - { - size = 1; - } - return sizeclass_to_size(size_to_sizeclass(size)); - } - - // Uses table for reciprocal division, so provide forward reference. - static bool is_multiple_of_sizeclass(sizeclass_t sc, size_t offset); - - /// Returns the alignment that this size naturally has, that is - /// all allocations of size `size` will be aligned to the returned value. - SNMALLOC_FAST_PATH static size_t natural_alignment(size_t size) - { - auto rsize = round_size(size); - return bits::one_at_bit(bits::ctz(rsize)); - } -} // namespace snmalloc diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index 322fa2f3c..a8607d5c5 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -1,58 +1,121 @@ #pragma once +#include "../ds/bits.h" +#include "../ds/defines.h" #include "../ds/helpers.h" -#include "superslab.h" +#include "allocconfig.h" namespace snmalloc { - constexpr size_t PTR_BITS = bits::next_pow2_bits_const(sizeof(void*)); + // Both usings should compile + // We use size_t as it generates better code. + using sizeclass_t = size_t; + // using sizeclass_t = uint8_t; + using sizeclass_compress_t = uint8_t; + + constexpr static uintptr_t SIZECLASS_MASK = 0xFF; + + constexpr static inline sizeclass_t size_to_sizeclass_const(size_t size) + { + // Don't use sizeclasses that are not a multiple of the alignment. + // For example, 24 byte allocations can be + // problematic for some data due to alignment issues. + auto sc = static_cast( + bits::to_exp_mant_const(size)); + + SNMALLOC_ASSERT(sc == static_cast(sc)); + + return sc; + } + + static inline size_t large_sizeclass_to_size(uint8_t large_class) + { + UNUSED(large_class); + abort(); + // return bits::one_at_bit(large_class + SUPERSLAB_BITS); + } + + static constexpr size_t NUM_SIZECLASSES = + size_to_sizeclass_const(MAX_SIZECLASS_SIZE); + + // Large classes range from [SUPERSLAB, ADDRESS_SPACE).// TODO + static constexpr size_t NUM_LARGE_CLASSES = + bits::ADDRESS_BITS - MAX_SIZECLASS_BITS; + + inline SNMALLOC_FAST_PATH static size_t + aligned_size(size_t alignment, size_t size) + { + // Client responsible for checking alignment is not zero + SNMALLOC_ASSERT(alignment != 0); + // Client responsible for checking alignment is a power of two + SNMALLOC_ASSERT(bits::is_pow2(alignment)); + + return ((alignment - 1) | (size - 1)) + 1; + } constexpr static SNMALLOC_PURE size_t sizeclass_lookup_index(const size_t s) { - // We subtract and shirt to reduce the size of the table, i.e. we don't have - // to store a value for every size class. - // We could shift by MIN_ALLOC_BITS, as this would give us the most - // compressed table, but by shifting by PTR_BITS the code-gen is better - // as the most important path using this subsequently shifts left by - // PTR_BITS, hence they can be fused into a single mask. - return (s - 1) >> PTR_BITS; + // We subtract and shift to reduce the size of the table, i.e. we don't have + // to store a value for every size. + return (s - 1) >> MIN_ALLOC_BITS; } + constexpr static size_t NUM_SIZECLASSES_EXTENDED = + size_to_sizeclass_const(bits::one_at_bit(bits::ADDRESS_BITS - 1)); + constexpr static size_t sizeclass_lookup_size = - sizeclass_lookup_index(SLAB_SIZE + 1); + sizeclass_lookup_index(MAX_SIZECLASS_SIZE); struct SizeClassTable { - sizeclass_t sizeclass_lookup[sizeclass_lookup_size] = {{}}; - ModArray size; - ModArray initial_offset_ptr; - ModArray short_initial_offset_ptr; - ModArray capacity; - ModArray short_capacity; - ModArray medium_slab_slots; + sizeclass_compress_t sizeclass_lookup[sizeclass_lookup_size] = {{}}; + ModArray size; + + ModArray capacity; + ModArray waking; + // We store the mask as it is used more on the fast path, and the size of + // the slab. + ModArray slab_mask; + // Table of constants for reciprocal division for each sizeclass. ModArray div_mult; // Table of constants for reciprocal modulus for each sizeclass. ModArray mod_mult; constexpr SizeClassTable() - : size(), - initial_offset_ptr(), - short_initial_offset_ptr(), - capacity(), - short_capacity(), - medium_slab_slots(), - div_mult(), - mod_mult() + : size(), capacity(), waking(), slab_mask(), div_mult(), mod_mult() { - size_t curr = 1; - for (sizeclass_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++) + for (sizeclass_compress_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; + sizeclass++) { - size[sizeclass] = + size_t rsize = bits::from_exp_mant(sizeclass); + size[sizeclass] = rsize; + size_t slab_bits = bits::max( + bits::next_pow2_bits_const(MIN_OBJECT_COUNT * rsize), MIN_CHUNK_BITS); + + slab_mask[sizeclass] = bits::one_at_bit(slab_bits) - 1; + + capacity[sizeclass] = + static_cast((slab_mask[sizeclass] + 1) / rsize); + + waking[sizeclass] = + static_cast(bits::min((capacity[sizeclass] / 4), 32)); + } + + for (sizeclass_compress_t sizeclass = NUM_SIZECLASSES; + sizeclass < NUM_SIZECLASSES_EXTENDED; + sizeclass++) + { + size[sizeclass] = bits::prev_pow2_const( + bits::from_exp_mant(sizeclass)); + } - div_mult[sizeclass] = - (bits::one_at_bit(bits::BITS - SUPERSLAB_BITS) / + for (sizeclass_compress_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; + sizeclass++) + { + div_mult[sizeclass] = // TODO is MAX_SIZECLASS_BITS right? + (bits::one_at_bit(bits::BITS - 24) / (size[sizeclass] / MIN_ALLOC_SIZE)); if (!bits::is_pow2(size[sizeclass])) div_mult[sizeclass]++; @@ -65,97 +128,94 @@ namespace snmalloc // overflows, and thus the top SUPERSLAB_BITS will be zero if the mod is // zero. mod_mult[sizeclass] *= 2; - - if (sizeclass < NUM_SMALL_CLASSES) - { - for (; curr <= size[sizeclass]; curr += 1 << PTR_BITS) - { - sizeclass_lookup[sizeclass_lookup_index(curr)] = sizeclass; - } - } - } - - size_t header_size = sizeof(Superslab); - size_t short_slab_size = SLAB_SIZE - header_size; - - for (sizeclass_t i = 0; i < NUM_SMALL_CLASSES; i++) - { - // We align to the end of the block to remove special cases for the - // short block. Calculate remainders - size_t short_correction = short_slab_size % size[i]; - size_t correction = SLAB_SIZE % size[i]; - - // First element in the block is the link - initial_offset_ptr[i] = static_cast(correction); - short_initial_offset_ptr[i] = - static_cast(header_size + short_correction); - - capacity[i] = static_cast( - (SLAB_SIZE - initial_offset_ptr[i]) / (size[i])); - short_capacity[i] = static_cast( - (SLAB_SIZE - short_initial_offset_ptr[i]) / (size[i])); } - for (sizeclass_t i = NUM_SMALL_CLASSES; i < NUM_SIZECLASSES; i++) + size_t curr = 1; + for (sizeclass_compress_t sizeclass = 0; sizeclass <= NUM_SIZECLASSES; + sizeclass++) { - medium_slab_slots[i - NUM_SMALL_CLASSES] = static_cast( - (SUPERSLAB_SIZE - Mediumslab::header_size()) / size[i]); + for (; curr <= size[sizeclass]; curr += 1 << MIN_ALLOC_BITS) + { + auto i = sizeclass_lookup_index(curr); + if (i == sizeclass_lookup_size) + break; + sizeclass_lookup[i] = sizeclass; + } } } }; - static constexpr SizeClassTable sizeclass_metadata = SizeClassTable(); + static inline constexpr SizeClassTable sizeclass_metadata = SizeClassTable(); + + constexpr static inline size_t sizeclass_to_size(sizeclass_t sizeclass) + { + return sizeclass_metadata.size[sizeclass]; + } - static inline constexpr uint16_t - get_initial_offset(sizeclass_t sc, bool is_short) + inline static size_t sizeclass_to_slab_size(sizeclass_t sizeclass) { - if (is_short) - return sizeclass_metadata.short_initial_offset_ptr[sc]; + return sizeclass_metadata.slab_mask[sizeclass] + 1; + } - return sizeclass_metadata.initial_offset_ptr[sc]; + /** + * Only wake slab if we have this many free allocations + * + * This helps remove bouncing around empty to non-empty cases. + * + * It also increases entropy, when we have randomisation. + */ + inline uint16_t threshold_for_waking_slab(sizeclass_t sizeclass) + { + // #ifdef CHECK_CLIENT + return sizeclass_metadata.waking[sizeclass]; + // #else + // UNUSED(sizeclass); + // return 1; + // #endif } - static inline constexpr uint16_t - get_slab_capacity(sizeclass_t sc, bool is_short) + inline static size_t sizeclass_to_slab_sizeclass(sizeclass_t sizeclass) { - if (is_short) - return sizeclass_metadata.short_capacity[sc]; + size_t ssize = sizeclass_to_slab_size(sizeclass); - return sizeclass_metadata.capacity[sc]; + return bits::next_pow2_bits(ssize) - MIN_CHUNK_BITS; } - constexpr static inline size_t sizeclass_to_size(sizeclass_t sizeclass) + inline static size_t slab_sizeclass_to_size(sizeclass_t sizeclass) { - return sizeclass_metadata.size[sizeclass]; + return bits::one_at_bit(MIN_CHUNK_BITS + sizeclass); + } + + inline constexpr static uint16_t + sizeclass_to_slab_object_count(sizeclass_t sizeclass) + { + return sizeclass_metadata.capacity[sizeclass]; } static inline sizeclass_t size_to_sizeclass(size_t size) { - if ((size - 1) <= (SLAB_SIZE - 1)) + auto index = sizeclass_lookup_index(size); + if (index < sizeclass_lookup_size) { - auto index = sizeclass_lookup_index(size); - SNMALLOC_ASSUME(index <= sizeclass_lookup_index(SLAB_SIZE)); return sizeclass_metadata.sizeclass_lookup[index]; } // Don't use sizeclasses that are not a multiple of the alignment. // For example, 24 byte allocations can be // problematic for some data due to alignment issues. + + // TODO hack to power of 2 for large sizes + size = bits::next_pow2(size); + return static_cast( bits::to_exp_mant(size)); } - constexpr static inline uint16_t medium_slab_free(sizeclass_t sizeclass) - { - return sizeclass_metadata - .medium_slab_slots[(sizeclass - NUM_SMALL_CLASSES)]; - } - inline static size_t round_by_sizeclass(sizeclass_t sc, size_t offset) { // Only works up to certain offsets, exhaustively tested upto // SUPERSLAB_SIZE. - SNMALLOC_ASSERT(offset <= SUPERSLAB_SIZE); + // SNMALLOC_ASSERT(offset <= SUPERSLAB_SIZE); auto rsize = sizeclass_to_size(sc); @@ -168,11 +228,12 @@ namespace snmalloc // sufficient bits to do this completely efficiently as 24 * 3 is larger // than 64 bits. But we can pre-round by MIN_ALLOC_SIZE which gets us an // extra 4 * 3 bits, and thus achievable in 64bit multiplication. - static_assert( - SUPERSLAB_BITS <= 24, "The following code assumes max of 24 bits"); + // static_assert( + // SUPERSLAB_BITS <= 24, "The following code assumes max of 24 bits"); + // TODO 24 hack return (((offset >> MIN_ALLOC_BITS) * sizeclass_metadata.div_mult[sc]) >> - (bits::BITS - SUPERSLAB_BITS)) * + (bits::BITS - 24)) * rsize; } else @@ -185,7 +246,7 @@ namespace snmalloc { // Only works up to certain offsets, exhaustively tested upto // SUPERSLAB_SIZE. - SNMALLOC_ASSERT(offset <= SUPERSLAB_SIZE); + // SNMALLOC_ASSERT(offset <= SUPERSLAB_SIZE); if constexpr (bits::is64()) { @@ -195,10 +256,10 @@ namespace snmalloc // get larger then we should review this code. The modulus code // has fewer restrictions than division, as it only requires the // square of the offset to be representable. - static_assert( - SUPERSLAB_BITS <= 24, "The following code assumes max of 24 bits"); + // TODO 24 hack. Redo the maths given the multiple + // slab sizes static constexpr size_t MASK = - ~(bits::one_at_bit(bits::BITS - 1 - SUPERSLAB_BITS) - 1); + ~(bits::one_at_bit(bits::BITS - 1 - 24) - 1); return ((offset * sizeclass_metadata.mod_mult[sc]) & MASK) == 0; } @@ -208,4 +269,36 @@ namespace snmalloc return static_cast(offset % sizeclass_to_size(sc)) == 0; } + inline SNMALLOC_FAST_PATH static size_t round_size(size_t size) + { + if (size > sizeclass_to_size(NUM_SIZECLASSES - 1)) + { + return bits::next_pow2(size); + } + if (size == 0) + { + return 0; + } + return sizeclass_to_size(size_to_sizeclass(size)); + } + + /// Returns the alignment that this size naturally has, that is + /// all allocations of size `size` will be aligned to the returned value. + inline SNMALLOC_FAST_PATH static size_t natural_alignment(size_t size) + { + auto rsize = round_size(size); + if (size == 0) + return 1; + return bits::one_at_bit(bits::ctz(rsize)); + } + + inline static size_t large_size_to_chunk_size(size_t size) + { + return bits::next_pow2(size); + } + + inline static size_t large_size_to_chunk_sizeclass(size_t size) + { + return bits::next_pow2_bits(size) - MIN_CHUNK_BITS; + } } // namespace snmalloc diff --git a/src/mem/slab.h b/src/mem/slab.h deleted file mode 100644 index d8f8b1f81..000000000 --- a/src/mem/slab.h +++ /dev/null @@ -1,197 +0,0 @@ -#pragma once - -#include "freelist.h" -#include "ptrhelpers.h" -#include "superslab.h" - -#include - -namespace snmalloc -{ - class Slab - { - private: - uint16_t address_to_index(address_t p) - { - // Get the offset from the slab for a memory location. - return static_cast(p - address_cast(this)); - } - - public: - template - static CapPtr get_meta(CapPtr self) - { - static_assert(B == CBArena || B == CBChunkD || B == CBChunk); - - auto super = Superslab::get(self); - return super->get_meta(self); - } - - /** - * Given a bumpptr and a fast_free_list head reference, builds a new free - * list, and stores it in the fast_free_list. It will only create a page - * worth of allocations, or one if the allocation size is larger than a - * page. - */ - static SNMALLOC_FAST_PATH void alloc_new_list( - CapPtr& bumpptr, - FreeListIter& fast_free_list, - size_t rsize, - LocalEntropy& entropy) - { - auto slab_end = pointer_align_up(pointer_offset(bumpptr, 1)); - - FreeListBuilder b; - SNMALLOC_ASSERT(b.empty()); - - b.open(bumpptr); - -#ifdef CHECK_CLIENT - // Structure to represent the temporary list elements - struct PreAllocObject - { - CapPtr next; - }; - // The following code implements Sattolo's algorithm for generating - // random cyclic permutations. This implementation is in the opposite - // direction, so that the original space does not need initialising. This - // is described as outside-in without citation on Wikipedia, appears to be - // Folklore algorithm. - - // Note the wide bounds on curr relative to each of the ->next fields; - // curr is not persisted once the list is built. - CapPtr curr = - pointer_offset(bumpptr, 0).template as_static(); - curr->next = Aal::capptr_bound(curr, rsize); - - uint16_t count = 1; - for (curr = - pointer_offset(curr, rsize).template as_static(); - curr.as_void() < slab_end; - curr = - pointer_offset(curr, rsize).template as_static()) - { - size_t insert_index = entropy.sample(count); - curr->next = std::exchange( - pointer_offset(bumpptr, insert_index * rsize) - .template as_static() - ->next, - Aal::capptr_bound(curr, rsize)); - count++; - } - - // Pick entry into space, and then build linked list by traversing cycle - // to the start. Use ->next to jump from CBArena to CBAlloc. - auto start_index = entropy.sample(count); - auto start_ptr = pointer_offset(bumpptr, start_index * rsize) - .template as_static() - ->next; - auto curr_ptr = start_ptr; - do - { - b.add(FreeObject::make(curr_ptr.as_void()), entropy); - curr_ptr = curr_ptr->next; - } while (curr_ptr != start_ptr); -#else - for (auto p = bumpptr; p < slab_end; p = pointer_offset(p, rsize)) - { - b.add(Aal::capptr_bound(p, rsize), entropy); - } -#endif - // This code consumes everything up to slab_end. - bumpptr = slab_end; - - SNMALLOC_ASSERT(!b.empty()); - b.close(fast_free_list, entropy); - } - - // Returns true, if it deallocation can proceed without changing any status - // bits. Note that this does remove the use from the meta slab, so it - // doesn't need doing on the slow path. - static SNMALLOC_FAST_PATH bool dealloc_fast( - CapPtr self, - CapPtr super, - CapPtr p, - LocalEntropy& entropy) - { - auto meta = super->get_meta(self); - SNMALLOC_ASSERT(!meta->is_unused()); - - if (unlikely(meta->return_object())) - return false; - - // Update the head and the next pointer in the free list. - meta->free_queue.add(p, entropy); - - return true; - } - - // If dealloc fast returns false, then call this. - // This does not need to remove the "use" as done by the fast path. - // Returns a complex return code for managing the superslab meta data. - // i.e. This deallocation could make an entire superslab free. - static SNMALLOC_SLOW_PATH typename Superslab::Action dealloc_slow( - CapPtr self, - SlabList* sl, - CapPtr super, - CapPtr p, - LocalEntropy& entropy) - { - auto meta = super->get_meta(self); - meta->debug_slab_invariant(self, entropy); - - if (meta->is_full()) - { - auto allocated = get_slab_capacity( - meta->sizeclass(), - Metaslab::is_short( - Metaslab::get_slab(Aal::capptr_rebound(super.as_void(), p)))); - // We are not on the sizeclass list. - if (allocated == 1) - { - // Dealloc on the superslab. - if (Metaslab::is_short(self)) - return super->dealloc_short_slab(); - - return super->dealloc_slab(self); - } - - meta->free_queue.add(p, entropy); - // Remove trigger threshold from how many we need before we have fully - // freed the slab. - meta->needed() = - allocated - meta->threshold_for_waking_slab(Metaslab::is_short(self)); - - // Push on the list of slabs for this sizeclass. - // ChunkD-to-Chunk conversion might apply bounds, so we need to do so to - // the aligned object and then shift over to these bounds. - auto super_chunk = capptr_chunk_from_chunkd(super, SUPERSLAB_SIZE); - auto metalink = Aal::capptr_rebound( - super_chunk.as_void(), meta.template as_static()); - sl->insert_prev(metalink); - meta->debug_slab_invariant(self, entropy); - return Superslab::NoSlabReturn; - } - -#ifdef CHECK_CLIENT - size_t count = 1; - // Check free list is well-formed on platforms with - // integers as pointers. - FreeListIter fl; - meta->free_queue.close(fl, entropy); - - while (!fl.empty()) - { - fl.take(entropy); - count++; - } -#endif - - meta->remove(); - - if (Metaslab::is_short(self)) - return super->dealloc_short_slab(); - return super->dealloc_slab(self); - } - }; -} // namespace snmalloc diff --git a/src/mem/slaballocator.h b/src/mem/slaballocator.h new file mode 100644 index 000000000..30b3dafce --- /dev/null +++ b/src/mem/slaballocator.h @@ -0,0 +1,163 @@ +#pragma once + +#include "../ds/mpmcstack.h" +#include "../mem/metaslab.h" +#include "../mem/sizeclasstable.h" + +#ifdef SNMALLOC_TRACING +# include +#endif + +namespace snmalloc +{ + /** + * Used to store slabs in the unused sizes. + */ + struct ChunkRecord + { + std::atomic next; + CapPtr chunk; + }; + + /** + * How many slab sizes that can be provided. + */ + constexpr size_t NUM_SLAB_SIZES = bits::ADDRESS_BITS - MIN_CHUNK_BITS; + + /** + * Used to ensure the per slab meta data is large enough for both use cases. + */ + static_assert( + sizeof(Metaslab) >= sizeof(ChunkRecord), "We conflat these two types."); + + /** + * This is the global state required for the chunk allocator. + * It must be provided as a part of the shared state handle + * to the chunk allocator. + */ + class ChunkAllocatorState + { + friend class ChunkAllocator; + /** + * Stack of slabs that have been returned for reuse. + */ + ModArray> chunk_stack; + + /** + * All memory issued by this address space manager + */ + std::atomic peak_memory_usage_{0}; + + std::atomic memory_in_stacks{0}; + + public: + size_t unused_memory() + { + return memory_in_stacks; + } + + size_t peak_memory_usage() + { + return peak_memory_usage_; + } + + void add_peak_memory_usage(size_t size) + { + peak_memory_usage_ += size; +#ifdef SNMALLOC_TRACING + std::cout << "peak_memory_usage_: " << peak_memory_usage_ << std::endl; +#endif + } + }; + + class ChunkAllocator + { + public: + template + static std::pair, Metaslab*> alloc_chunk( + SharedStateHandle h, + typename SharedStateHandle::Backend::LocalState& backend_state, + sizeclass_t sizeclass, + sizeclass_t slab_sizeclass, // TODO sizeclass_t + size_t slab_size, + RemoteAllocator* remote) + { + ChunkAllocatorState& state = h.get_slab_allocator_state(); + // Pop a slab + auto chunk_record = state.chunk_stack[slab_sizeclass].pop(); + + if (chunk_record != nullptr) + { + auto slab = chunk_record->chunk; + state.memory_in_stacks -= slab_size; + auto meta = reinterpret_cast(chunk_record); +#ifdef SNMALLOC_TRACING + std::cout << "Reuse slab:" << slab.unsafe_ptr() << " slab_sizeclass " + << slab_sizeclass << " size " << slab_size + << " memory in stacks " << state.memory_in_stacks + << std::endl; +#endif + MetaEntry entry{meta, remote, sizeclass}; + SharedStateHandle::Backend::set_meta_data( + h.get_backend_state(), address_cast(slab), slab_size, entry); + return {slab, meta}; + } + + // Allocate a fresh slab as there are no available ones. + // First create meta-data + auto [slab, meta] = SharedStateHandle::Backend::alloc_chunk( + h.get_backend_state(), &backend_state, slab_size, remote, sizeclass); +#ifdef SNMALLOC_TRACING + std::cout << "Create slab:" << slab.unsafe_ptr() << " slab_sizeclass " + << slab_sizeclass << " size " << slab_size << std::endl; +#endif + + state.add_peak_memory_usage(slab_size); + state.add_peak_memory_usage(sizeof(Metaslab)); + // TODO handle bounded versus lazy pagemaps in stats + state.add_peak_memory_usage( + (slab_size / MIN_CHUNK_SIZE) * sizeof(MetaEntry)); + + return {slab, meta}; + } + + template + SNMALLOC_SLOW_PATH static void + dealloc(SharedStateHandle h, ChunkRecord* p, size_t slab_sizeclass) + { + auto& state = h.get_slab_allocator_state(); +#ifdef SNMALLOC_TRACING + std::cout << "Return slab:" << p->chunk.unsafe_ptr() << " slab_sizeclass " + << slab_sizeclass << " size " + << slab_sizeclass_to_size(slab_sizeclass) + << " memory in stacks " << state.memory_in_stacks << std::endl; +#endif + state.chunk_stack[slab_sizeclass].push(p); + state.memory_in_stacks += slab_sizeclass_to_size(slab_sizeclass); + } + + /** + * Provide a block of meta-data with size and align. + * + * Backend allocator may use guard pages and separate area of + * address space to protect this from corruption. + */ + template + static U* alloc_meta_data( + SharedStateHandle h, + typename SharedStateHandle::Backend::LocalState* local_state, + Args&&... args) + { + // Cache line align + size_t size = bits::align_up(sizeof(U), 64); + + CapPtr p = SharedStateHandle::Backend::alloc_meta_data( + h.get_backend_state(), local_state, size); + + if (p == nullptr) + return nullptr; + + return new (p.unsafe_ptr()) U(std::forward(args)...); + } + }; +} // namespace snmalloc diff --git a/src/mem/superslab.h b/src/mem/superslab.h deleted file mode 100644 index 27775034c..000000000 --- a/src/mem/superslab.h +++ /dev/null @@ -1,272 +0,0 @@ -#pragma once - -#include "../ds/helpers.h" -#include "allocslab.h" -#include "metaslab.h" - -#include - -namespace snmalloc -{ - /** - * Superslabs are, to first approximation, a `CHUNK_SIZE`-sized and -aligned - * region of address space, internally composed of a header (a `Superslab` - * structure) followed by an array of `Slab`s, each `SLAB_SIZE`-sized and - * -aligned. Each active `Slab` holds an array of identically sized - * allocations strung on an invasive free list, which is lazily constructed - * from a bump-pointer allocator (see `Metaslab::alloc_new_list`). - * - * In order to minimize overheads, Slab metadata is held externally, in - * `Metaslab` structures; all `Metaslab`s for the Slabs within a Superslab are - * densely packed within the `Superslab` structure itself. Moreover, as the - * `Superslab` structure is typically much smaller than `SLAB_SIZE`, a "short - * Slab" is overlaid with the `Superslab`. This short Slab can hold only - * allocations that are smaller than the `SLAB_SIZE - sizeof(Superslab)` - * bytes; see `Superslab::is_short_sizeclass`. The Metaslab state for a short - * slabs is constructed in a way that avoids branches on fast paths; - * effectively, the object slots that overlay the `Superslab` at the start are - * omitted from consideration. - */ - class Superslab : public Allocslab - { - private: - friend DLList; - - // Keep the allocator pointer on a separate cache line. It is read by - // other threads, and does not change, so we avoid false sharing. - alignas(CACHELINE_SIZE) - // The superslab is kept on a doubly linked list of superslabs which - // have some space. - CapPtr next; - CapPtr prev; - - // This is a reference to the first unused slab in the free slab list - // It is does not contain the short slab, which is handled using a bit - // in the "used" field below. The list is terminated by pointing to - // the short slab. - // The head linked list has an absolute pointer for head, but the next - // pointers stores in the metaslabs are relative pointers, that is they - // are the relative offset to the next entry minus 1. This means that - // all zeros is a list that chains through all the blocks, so the zero - // initialised memory requires no more work. - Mod head; - - // Represents twice the number of full size slabs used - // plus 1 for the short slab. i.e. using 3 slabs and the - // short slab would be 6 + 1 = 7 - uint16_t used; - - ModArray meta; - - // Used size_t as results in better code in MSVC - template - size_t slab_to_index(CapPtr slab) - { - auto res = (pointer_diff(this, slab.unsafe_capptr) >> SLAB_BITS); - SNMALLOC_ASSERT(res == static_cast(res)); - return static_cast(res); - } - - public: - enum Status - { - Full, - Available, - OnlyShortSlabAvailable, - Empty - }; - - enum Action - { - NoSlabReturn = 0, - NoStatusChange = 1, - StatusChange = 2 - }; - - /** - * Given a highly-privileged pointer pointing to or within an object in - * this slab, return a pointer to the slab headers. - * - * In debug builds on StrictProvenance architectures, we will enforce the - * slab bounds on this returned pointer. In non-debug builds, we will - * return a highly-privileged pointer (i.e., CBArena) instead as these - * pointers are not exposed from the allocator. - */ - template - static SNMALLOC_FAST_PATH CapPtr()> - get(CapPtr p) - { - static_assert(B == CBArena || B == CBChunkD || B == CBChunk); - - return capptr_bound_chunkd( - pointer_align_down(p.as_void()), - SUPERSLAB_SIZE); - } - - static bool is_short_sizeclass(sizeclass_t sizeclass) - { - static_assert(SLAB_SIZE > sizeof(Superslab), "Meta data requires this."); - /* - * size_to_sizeclass_const rounds *up* and returns the smallest class that - * could contain (and so may be larger than) the free space available for - * the short slab. While we could detect the exact fit case and compare - * `<= h` therein, it's simpler to just treat this class as a strict upper - * bound and only permit strictly smaller classes in short slabs. - */ - constexpr sizeclass_t h = - size_to_sizeclass_const(SLAB_SIZE - sizeof(Superslab)); - return sizeclass < h; - } - - void init(RemoteAllocator* alloc) - { - allocator = alloc; - - // If Superslab is larger than a page, then we cannot guarantee it still - // has a valid layout as the subsequent pages could have been freed and - // zeroed, hence only skip initialisation if smaller. - if (kind != Super || (sizeof(Superslab) >= OS_PAGE_SIZE)) - { - if (kind != Fresh) - { - // If this wasn't previously Fresh, we need to zero some things. - used = 0; - for (size_t i = 0; i < SLAB_COUNT; i++) - { - new (&(meta[i])) Metaslab(); - } - } - - // If this wasn't previously a Superslab, we need to set up the - // header. - kind = Super; - // Point head at the first non-short slab. - head = 1; - } - -#ifndef NDEBUG - auto curr = head; - for (size_t i = 0; i < SLAB_COUNT - used - 1; i++) - { - curr = (curr + meta[curr].next() + 1) & (SLAB_COUNT - 1); - } - if (curr != 0) - abort(); - - for (size_t i = 0; i < SLAB_COUNT; i++) - { - SNMALLOC_ASSERT(meta[i].is_unused()); - } -#endif - } - - bool is_empty() - { - return used == 0; - } - - bool is_full() - { - return (used == (((SLAB_COUNT - 1) << 1) + 1)); - } - - bool is_almost_full() - { - return (used >= ((SLAB_COUNT - 1) << 1)); - } - - Status get_status() - { - if (!is_almost_full()) - { - if (!is_empty()) - { - return Available; - } - - return Empty; - } - - if (!is_full()) - { - return OnlyShortSlabAvailable; - } - - return Full; - } - - template - CapPtr get_meta(CapPtr slab) - { - return CapPtr(&meta[slab_to_index(slab)]); - } - - static CapPtr - alloc_short_slab(CapPtr self, sizeclass_t sizeclass) - { - if ((self->used & 1) == 1) - return alloc_slab(self, sizeclass); - - auto slab = self.template as_reinterpret(); - auto& metaz = self->meta[0]; - - metaz.initialise(sizeclass, slab); - - self->used++; - return slab; - } - - static CapPtr - alloc_slab(CapPtr self, sizeclass_t sizeclass) - { - uint8_t h = self->head; - auto slab = pointer_offset(self, (static_cast(h) << SLAB_BITS)) - .template as_static(); - - auto& metah = self->meta[h]; - uint8_t n = metah.next(); - - metah.initialise(sizeclass, slab); - - self->head = h + n + 1; - self->used += 2; - - return slab; - } - - // Returns true, if this alters the value of get_status - template - Action dealloc_slab(CapPtr slab) - { - static_assert(B == CBArena || B == CBChunkD || B == CBChunk); - - // This is not the short slab. - uint8_t index = static_cast(slab_to_index(slab)); - uint8_t n = head - index - 1; - - meta[index].next() = n; - head = index; - bool was_almost_full = is_almost_full(); - used -= 2; - - SNMALLOC_ASSERT(meta[index].is_unused()); - if (was_almost_full || is_empty()) - return StatusChange; - - return NoStatusChange; - } - - // Returns true, if this alters the value of get_status - Action dealloc_short_slab() - { - bool was_full = is_full(); - used--; - - SNMALLOC_ASSERT(meta[0].is_unused()); - if (was_full || is_empty()) - return StatusChange; - - return NoStatusChange; - } - }; -} // namespace snmalloc diff --git a/src/mem/threadalloc.h b/src/mem/threadalloc.h index 306c78215..f2b6f4bd6 100644 --- a/src/mem/threadalloc.h +++ b/src/mem/threadalloc.h @@ -1,12 +1,32 @@ #pragma once #include "../ds/helpers.h" -#include "globalalloc.h" -#if defined(SNMALLOC_USE_THREAD_DESTRUCTOR) && \ - defined(SNMALLOC_USE_THREAD_CLEANUP) -#error At most one out of SNMALLOC_USE_THREAD_CLEANUP and SNMALLOC_USE_THREAD_DESTRUCTOR may be defined. +#include "globalconfig.h" +#include "localalloc.h" + +#if defined(SNMALLOC_EXTERNAL_THREAD_ALLOC) +# define SNMALLOC_THREAD_TEARDOWN_DEFINED +#endif + +#if defined(SNMALLOC_USE_THREAD_CLEANUP) +# if defined(SNMALLOC_THREAD_TEARDOWN_DEFINED) +# error At most one out of method of thread teardown can be specified. +# else +# define SNMALLOC_THREAD_TEARDOWN_DEFINED +# endif +#endif + +#if defined(SNMALLOC_USE_PTHREAD_DESTRUCTORS) +# if defined(SNMALLOC_THREAD_TEARDOWN_DEFINED) +# error At most one out of method of thread teardown can be specified. +# else +# define SNMALLOC_THREAD_TEARDOWN_DEFINED +# endif #endif +#if !defined(SNMALLOC_THREAD_TEARDOWN_DEFINED) +# define SNMALLOC_USE_CXX_THREAD_DESTRUCTORS +#endif extern "C" void _malloc_thread_cleanup(); namespace snmalloc @@ -14,42 +34,23 @@ namespace snmalloc #ifdef SNMALLOC_EXTERNAL_THREAD_ALLOC /** * Version of the `ThreadAlloc` interface that does no management of thread - * local state, and just assumes that "ThreadAllocUntyped::get" has been - * declared before including snmalloc.h. As it is included before, it cannot - * know the allocator type, hence the casting. + * local state. * - * This class is used only when snmalloc is compiled as part of a runtime, - * which has its own management of the thread local allocator pointer. + * It assumes that Alloc has been defined, and `ThreadAllocExternal` class + * has access to snmalloc_core.h. */ - class ThreadAllocUntypedWrapper + class ThreadAlloc { protected: static void register_cleanup() {} public: - static SNMALLOC_FAST_PATH Alloc* get_noncachable() - { - return (Alloc*)ThreadAllocUntyped::get(); - } - - static SNMALLOC_FAST_PATH Alloc* get() + static SNMALLOC_FAST_PATH Alloc& get() { - return (Alloc*)ThreadAllocUntyped::get(); + return ThreadAllocExternal::get(); } }; - /** - * Function passed as a template parameter to `Allocator` to allow lazy - * replacement. This function returns true, if the allocator passed in - * requires initialisation. As the TLS state is managed externally, - * this will always return false. - */ - SNMALLOC_FAST_PATH bool needs_initialisation(void* existing) - { - UNUSED(existing); - return false; - } - /** * Function passed as a template parameter to `Allocator` to allow lazy * replacement. There is nothing to initialise in this case, so we expect @@ -62,247 +63,100 @@ namespace snmalloc # pragma warning(push) # pragma warning(disable : 4702) # endif - SNMALLOC_FAST_PATH void* init_thread_allocator(function_ref f) + inline void register_clean_up() { error("Critical Error: This should never be called."); - return f(nullptr); } # ifdef _MSC_VER # pragma warning(pop) # endif - - using ThreadAlloc = ThreadAllocUntypedWrapper; #else - /** - * A global fake allocator object. This never allocates memory and, as a - * result, never owns any slabs. On the slow paths, where it would fetch - * slabs to allocate from, it will discover that it is the placeholder and - * replace itself with the thread-local allocator, allocating one if - * required. This avoids a branch on the fast path. - * - * The fake allocator is a zero initialised area of memory of the correct - * size. All data structures used potentially before initialisation must be - * okay with zero init to move to the slow path, that is, zero must signify - * empty. - */ - inline const char GlobalPlaceHolder[sizeof(Alloc)] = {0}; - inline Alloc* get_GlobalPlaceHolder() - { - // This cast is not legal. Effectively, we want a minimal constructor - // for the global allocator as zero, and then a second constructor for - // the rest. This is UB. - auto a = reinterpret_cast(&GlobalPlaceHolder); - return const_cast(a); - } /** - * Common aspects of thread local allocator. Subclasses handle how releasing - * the allocator is triggered. + * Holds the thread local state for the allocator. The state is constant + * initialised, and has no direct dectructor. Instead snmalloc will call + * `register_clean_up` on the slow path for bringing up thread local state. + * This is responsible for calling `teardown`, which effectively destructs the + * data structure, but in a way that allow it to still be used. */ - class ThreadAllocCommon + class ThreadAlloc { - friend void* init_thread_allocator(function_ref); - - protected: - /** - * Thread local variable that is set to true, once `inner_release` - * has been run. If we try to reinitialise the allocator once - * `inner_release` has run, then we can stay on the slow path so we don't - * leak allocators. - * - * This is required to allow for the allocator to be called during - * destructors of other thread_local state. - */ - inline static thread_local bool destructor_has_run = false; - - static inline void inner_release() - { - auto& per_thread = get_reference(); - if (per_thread != get_GlobalPlaceHolder()) - { - current_alloc_pool()->release(per_thread); - destructor_has_run = true; - per_thread = get_GlobalPlaceHolder(); - } - } - - /** - * Default clean up does nothing except print statistics if enabled. - */ - static bool register_cleanup() - { -# ifdef USE_SNMALLOC_STATS - Singleton::get(); -# endif - return false; - } - -# ifdef USE_SNMALLOC_STATS - static void print_stats() - { - Stats s; - current_alloc_pool()->aggregate_stats(s); - s.print(std::cout); - } - - static int atexit_print_stats() noexcept - { - return atexit(print_stats); - } -# endif - public: /** - * Returns a reference to the allocator for the current thread. This allows - * the caller to replace the current thread's allocator. - */ - static inline Alloc*& get_reference() - { - // Inline casting as codegen doesn't create a lazy init like this. - static thread_local Alloc* alloc = - const_cast(reinterpret_cast(&GlobalPlaceHolder)); - return alloc; - } - - /** - * Public interface, returns the allocator for this thread, constructing - * one if necessary. - * - * If no operations have been performed on an allocator returned by either - * `get()` nor `get_noncachable()`, then the value contained in the return - * will be an Alloc* that will always use the slow path. + * Handle on thread local allocator * - * Only use this API if you intend to use the returned allocator just once - * per call, or if you know other calls have already been made to the - * allocator. + * This structure will self initialise if it has not been called yet. + * It can be used during thread teardown, but its performance will be + * less good. */ - static inline Alloc* get_noncachable() + static SNMALLOC_FAST_PATH Alloc& get() { - return get_reference(); - } - - /** - * Public interface, returns the allocator for this thread, constructing - * one if necessary. - * This incurs a cost, so use `get_noncachable` if you can meet its - * criteria. - */ - static SNMALLOC_FAST_PATH Alloc* get() - { -# ifdef SNMALLOC_PASS_THROUGH - return get_reference(); -# else - auto*& alloc = get_reference(); - if (unlikely(needs_initialisation(alloc)) && !destructor_has_run) - { - // Call `init_thread_allocator` to perform down call in case - // register_clean_up does more. - // During teardown for the destructor based ThreadAlloc this will set - // alloc to GlobalPlaceHolder; - init_thread_allocator([](void*) { return nullptr; }); - } + SNMALLOC_REQUIRE_CONSTINIT static thread_local Alloc alloc; return alloc; -# endif } }; +# ifdef SNMALLOC_USE_PTHREAD_DESTRUCTOR /** - * Version of the `ThreadAlloc` interface that uses a hook provided by libc - * to destroy thread-local state. This is the ideal option, because it - * enforces ordering of destruction such that the malloc state is destroyed - * after anything that can allocate memory. - * - * This class is used only when snmalloc is compiled as part of a compatible - * libc (for example, FreeBSD libc). - */ - class ThreadAllocLibcCleanup : public ThreadAllocCommon - { - /** - * Libc will call `_malloc_thread_cleanup` just before a thread terminates. - * This function must be allowed to call back into this class to destroy - * the state. - */ - friend void ::_malloc_thread_cleanup(); - }; - - /** - * Version of the `ThreadAlloc` interface that uses C++ `thread_local` - * destructors for cleanup. If a per-thread allocator is used during the - * destruction of other per-thread data, this class will create a new - * instance and register its destructor, so should eventually result in - * cleanup, but may result in allocators being returned to the global pool - * and then reacquired multiple times. - * - * This implementation depends on nothing outside of a working C++ - * environment and so should be the simplest for initial bringup on an - * unsupported platform. It is currently used in the FreeBSD kernel version. + * Used to give correct signature to teardown required by pthread_key. */ - class ThreadAllocThreadDestructor : public ThreadAllocCommon + inline void pthread_cleanup(void*) { - template - friend class OnDestruct; - - public: - static bool register_cleanup() - { - static thread_local OnDestruct tidier; - - ThreadAllocCommon::register_cleanup(); - - return destructor_has_run; - } - }; - -# ifdef SNMALLOC_USE_THREAD_CLEANUP + ThreadAlloc::get().teardown(); + } /** - * Entry point that allows libc to call into the allocator for per-thread - * cleanup. + * Used to give correct signature to the pthread call for the Singleton class. */ - extern "C" void _malloc_thread_cleanup() + inline pthread_key_t pthread_create() noexcept { - ThreadAllocLibcCleanup::inner_release(); + pthread_key_t key; + pthread_key_create(&key, &pthread_cleanup); + return key; } - using ThreadAlloc = ThreadAllocLibcCleanup; -# else - using ThreadAlloc = ThreadAllocThreadDestructor; -# endif - /** - * Slow path for the placeholder replacement. - * Function passed as a tempalte parameter to `Allocator` to allow lazy - * replacement. This function initialises the thread local state if requried. - * The simple check that this is the global placeholder is inlined, the rest - * of it is only hit in a very unusual case and so should go off the fast - * path. - * The second component of the return indicates if this TLS is being torndown. + * Performs thread local teardown for the allocator using the pthread library. + * + * This removes the dependence on the C++ runtime. */ - SNMALLOC_FAST_PATH void* init_thread_allocator(function_ref f) + inline void register_clean_up() { - auto*& local_alloc = ThreadAlloc::get_reference(); - // If someone reuses a noncachable call, then we can end up here - // with an already initialised allocator. Could either error - // to say stop doing this, or just give them the initialised version. - if (local_alloc == get_GlobalPlaceHolder()) - { - local_alloc = current_alloc_pool()->acquire(); - } - auto result = f(local_alloc); - // Check if we have already run the destructor for the TLS. If so, - // we need to deallocate the allocator. - if (ThreadAlloc::register_cleanup()) - ThreadAlloc::inner_release(); - return result; + Singleton p_key; + // We need to set a non-null value, so that the destructor is called, + // we never look at the value. + pthread_setspecific(p_key.get(), reinterpret_cast(1)); +# ifdef SNMALLOC_TRACING + std::cout << "Using pthread clean up" << std::endl; +# endif } - +# elif defined(SNMALLOC_USE_CXX_THREAD_DESTRUCTORS) /** - * Function passed as a template parameter to `Allocator` to allow lazy - * replacement. This function returns true, if the allocated passed in, - * is the placeholder allocator. If it returns true, then - * `init_thread_allocator` should be called. + * This function is called by each thread once it starts using the + * thread local allocator. + * + * This implementation depends on nothing outside of a working C++ + * environment and so should be the simplest for initial bringup on an + * unsupported platform. */ - SNMALLOC_FAST_PATH bool needs_initialisation(void* existing) + inline void register_clean_up() { - return existing == get_GlobalPlaceHolder(); + static thread_local OnDestruct dummy( + []() { ThreadAlloc::get().teardown(); }); + UNUSED(dummy); +# ifdef SNMALLOC_TRACING + std::cout << "Using C++ destructor clean up" << std::endl; +# endif } +# endif #endif } // namespace snmalloc + +#ifdef SNMALLOC_USE_THREAD_CLEANUP +/** + * Entry point that allows libc to call into the allocator for per-thread + * cleanup. + */ +void _malloc_thread_cleanup() +{ + ThreadAlloc::get().teardown(); +} +#endif diff --git a/src/override/malloc-extensions.cc b/src/override/malloc-extensions.cc index ff621a98e..9245cc923 100644 --- a/src/override/malloc-extensions.cc +++ b/src/override/malloc-extensions.cc @@ -6,7 +6,10 @@ using namespace snmalloc; void get_malloc_info_v1(malloc_info_v1* stats) { - auto next_memory_usage = default_memory_provider().memory_usage(); - stats->current_memory_usage = next_memory_usage.first; - stats->peak_memory_usage = next_memory_usage.second; + auto unused_chunks = + Globals::get_handle().get_slab_allocator_state().unused_memory(); + auto peak = + Globals::get_handle().get_slab_allocator_state().peak_memory_usage(); + stats->current_memory_usage = peak - unused_chunks; + stats->peak_memory_usage = peak; } \ No newline at end of file diff --git a/src/override/malloc.cc b/src/override/malloc.cc index adc272b2e..a04004268 100644 --- a/src/override/malloc.cc +++ b/src/override/malloc.cc @@ -1,5 +1,16 @@ -#include "../mem/slowalloc.h" -#include "../snmalloc.h" +// Core implementation of snmalloc independent of the configuration mode +#include "../snmalloc_core.h" + +#ifndef SNMALLOC_PROVIDE_OWN_CONFIG +// The default configuration for snmalloc is used if alternative not defined +namespace snmalloc +{ + using Alloc = snmalloc::LocalAllocator; +} // namespace snmalloc +#endif + +// User facing API surface, needs to know what `Alloc` is. +#include "../snmalloc_front.h" #include #include @@ -24,78 +35,60 @@ using namespace snmalloc; extern "C" { - void SNMALLOC_NAME_MANGLE(check_start)(void* ptr) - { -#if !defined(NDEBUG) && !defined(SNMALLOC_PASS_THROUGH) - if (ThreadAlloc::get_noncachable()->external_pointer(ptr) != ptr) - { - error("Using pointer that is not to the start of an allocation"); - } -#else - UNUSED(ptr); -#endif - } - SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(__malloc_end_pointer)(void* ptr) { - return ThreadAlloc::get_noncachable()->external_pointer(ptr); + return ThreadAlloc::get().external_pointer(ptr); } SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(malloc)(size_t size) { - return ThreadAlloc::get_noncachable()->alloc(size); + return ThreadAlloc::get().alloc(size); } SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(free)(void* ptr) { - SNMALLOC_NAME_MANGLE(check_start)(ptr); - ThreadAlloc::get_noncachable()->dealloc(ptr); + ThreadAlloc::get().dealloc(ptr); } SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(cfree)(void* ptr) { - SNMALLOC_NAME_MANGLE(free)(ptr); + ThreadAlloc::get().dealloc(ptr); + } + + /** + * Clang was helpfully inlining the constant return value, and + * thus converting from a tail call to an ordinary call. + */ + SNMALLOC_EXPORT inline void* snmalloc_not_allocated = nullptr; + + static SNMALLOC_SLOW_PATH void* SNMALLOC_NAME_MANGLE(snmalloc_set_error)() + { + errno = ENOMEM; + return snmalloc_not_allocated; } SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(calloc)(size_t nmemb, size_t size) { bool overflow = false; size_t sz = bits::umul(size, nmemb, overflow); - if (overflow) + if (unlikely(overflow)) { - errno = ENOMEM; - return nullptr; + return SNMALLOC_NAME_MANGLE(snmalloc_set_error)(); } - return ThreadAlloc::get_noncachable()->alloc(sz); + return ThreadAlloc::get().alloc(sz); } SNMALLOC_EXPORT size_t SNMALLOC_NAME_MANGLE(malloc_usable_size)( MALLOC_USABLE_SIZE_QUALIFIER void* ptr) { - return ThreadAlloc::get_noncachable()->alloc_size(ptr); + return ThreadAlloc::get().alloc_size(ptr); } SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(realloc)(void* ptr, size_t size) { - if (size == (size_t)-1) - { - errno = ENOMEM; - return nullptr; - } - if (ptr == nullptr) - { - return SNMALLOC_NAME_MANGLE(malloc)(size); - } - if (size == 0) - { - SNMALLOC_NAME_MANGLE(free)(ptr); - return nullptr; - } - - SNMALLOC_NAME_MANGLE(check_start)(ptr); - - size_t sz = ThreadAlloc::get_noncachable()->alloc_size(ptr); + auto& a = ThreadAlloc::get(); + size_t sz = a.alloc_size(ptr); // Keep the current allocation if the given size is in the same sizeclass. if (sz == round_size(size)) { @@ -108,13 +101,26 @@ extern "C" return ptr; #endif } - void* p = SNMALLOC_NAME_MANGLE(malloc)(size); - if (p != nullptr) + + if (size == (size_t)-1) + { + errno = ENOMEM; + return nullptr; + } + + void* p = a.alloc(size); + if (likely(p != nullptr)) { - SNMALLOC_NAME_MANGLE(check_start)(p); sz = bits::min(size, sz); - memcpy(p, ptr, sz); - SNMALLOC_NAME_MANGLE(free)(ptr); + // Guard memcpy as GCC is assuming not nullptr for ptr after the memcpy + // otherwise. + if (sz != 0) + memcpy(p, ptr, sz); + a.dealloc(ptr); + } + else if (likely(size == 0)) + { + a.dealloc(ptr); } return p; } @@ -149,8 +155,7 @@ extern "C" return nullptr; } - return SNMALLOC_NAME_MANGLE(malloc)( - size ? aligned_size(alignment, size) : alignment); + return SNMALLOC_NAME_MANGLE(malloc)(aligned_size(alignment, size)); } SNMALLOC_EXPORT void* @@ -163,17 +168,16 @@ extern "C" SNMALLOC_EXPORT int SNMALLOC_NAME_MANGLE(posix_memalign)( void** memptr, size_t alignment, size_t size) { - if ( - ((alignment % sizeof(uintptr_t)) != 0) || - ((alignment & (alignment - 1)) != 0) || (alignment == 0)) + if ((alignment < sizeof(uintptr_t) || ((alignment & (alignment - 1)) != 0))) { return EINVAL; } void* p = SNMALLOC_NAME_MANGLE(memalign)(alignment, size); - if (p == nullptr) + if (unlikely(p == nullptr)) { - return ENOMEM; + if (size != 0) + return ENOMEM; } *memptr = p; return 0; @@ -212,44 +216,14 @@ extern "C" return ENOENT; } -#ifdef SNMALLOC_EXPOSE_PAGEMAP - /** - * Export the pagemap. The return value is a pointer to the pagemap - * structure. The argument is used to return a pointer to a `PagemapConfig` - * structure describing the type of the pagemap. Static methods on the - * concrete pagemap templates can then be used to safely cast the return from - * this function to the correct type. This allows us to preserve some - * semblance of ABI safety via a pure C API. - */ - SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(snmalloc_chunkmap_global_get)( - PagemapConfig const** config) - { - auto& pm = GlobalChunkmap::pagemap(); - if (config) - { - *config = &ChunkmapPagemap::config; - SNMALLOC_ASSERT(ChunkmapPagemap::cast_to_pagemap(&pm, *config) == &pm); - } - return ± - } -#endif - -#ifdef SNMALLOC_EXPOSE_RESERVE - SNMALLOC_EXPORT void* - SNMALLOC_NAME_MANGLE(snmalloc_reserve_shared)(size_t* size, size_t align) - { - return snmalloc::default_memory_provider().reserve(size, align); - } -#endif - -#if !defined(__PIC__) && !defined(NO_BOOTSTRAP_ALLOCATOR) +#if !defined(__PIC__) && defined(SNMALLOC_BOOTSTRAP_ALLOCATOR) // The following functions are required to work before TLS is set up, in // statically-linked programs. These temporarily grab an allocator from the // pool and return it. void* __je_bootstrap_malloc(size_t size) { - return get_slow_allocator()->alloc(size); + return get_scoped_allocator().alloc(size); } void* __je_bootstrap_calloc(size_t nmemb, size_t size) @@ -263,12 +237,12 @@ extern "C" } // Include size 0 in the first sizeclass. sz = ((sz - 1) >> (bits::BITS - 1)) + sz; - return get_slow_allocator()->alloc(sz); + return get_scoped_allocator().alloc(sz); } void __je_bootstrap_free(void* ptr) { - get_slow_allocator()->dealloc(ptr); + get_scoped_allocator().dealloc(ptr); } #endif } diff --git a/src/override/new.cc b/src/override/new.cc index 70083da37..96b1c94d2 100644 --- a/src/override/new.cc +++ b/src/override/new.cc @@ -1,6 +1,4 @@ -#include "../mem/alloc.h" -#include "../mem/threadalloc.h" -#include "../snmalloc.h" +#include "malloc.cc" #ifdef _WIN32 # define EXCEPTSPEC @@ -18,54 +16,54 @@ using namespace snmalloc; void* operator new(size_t size) { - return ThreadAlloc::get_noncachable()->alloc(size); + return ThreadAlloc::get().alloc(size); } void* operator new[](size_t size) { - return ThreadAlloc::get_noncachable()->alloc(size); + return ThreadAlloc::get().alloc(size); } void* operator new(size_t size, std::nothrow_t&) { - return ThreadAlloc::get_noncachable()->alloc(size); + return ThreadAlloc::get().alloc(size); } void* operator new[](size_t size, std::nothrow_t&) { - return ThreadAlloc::get_noncachable()->alloc(size); + return ThreadAlloc::get().alloc(size); } void operator delete(void* p)EXCEPTSPEC { - ThreadAlloc::get_noncachable()->dealloc(p); + ThreadAlloc::get().dealloc(p); } void operator delete(void* p, size_t size)EXCEPTSPEC { if (p == nullptr) return; - ThreadAlloc::get_noncachable()->dealloc(p, size); + ThreadAlloc::get().dealloc(p, size); } void operator delete(void* p, std::nothrow_t&) { - ThreadAlloc::get_noncachable()->dealloc(p); + ThreadAlloc::get().dealloc(p); } void operator delete[](void* p) EXCEPTSPEC { - ThreadAlloc::get_noncachable()->dealloc(p); + ThreadAlloc::get().dealloc(p); } void operator delete[](void* p, size_t size) EXCEPTSPEC { if (p == nullptr) return; - ThreadAlloc::get_noncachable()->dealloc(p, size); + ThreadAlloc::get().dealloc(p, size); } void operator delete[](void* p, std::nothrow_t&) { - ThreadAlloc::get_noncachable()->dealloc(p); + ThreadAlloc::get().dealloc(p); } diff --git a/src/override/rust.cc b/src/override/rust.cc index 0ef25644d..b54be6bc9 100644 --- a/src/override/rust.cc +++ b/src/override/rust.cc @@ -11,20 +11,19 @@ using namespace snmalloc; extern "C" SNMALLOC_EXPORT void* rust_alloc(size_t alignment, size_t size) { - return ThreadAlloc::get_noncachable()->alloc(aligned_size(alignment, size)); + return ThreadAlloc::get().alloc(aligned_size(alignment, size)); } extern "C" SNMALLOC_EXPORT void* rust_alloc_zeroed(size_t alignment, size_t size) { - return ThreadAlloc::get_noncachable()->alloc( - aligned_size(alignment, size)); + return ThreadAlloc::get().alloc(aligned_size(alignment, size)); } extern "C" SNMALLOC_EXPORT void rust_dealloc(void* ptr, size_t alignment, size_t size) { - ThreadAlloc::get_noncachable()->dealloc(ptr, aligned_size(alignment, size)); + ThreadAlloc::get().dealloc(ptr, aligned_size(alignment, size)); } extern "C" SNMALLOC_EXPORT void* @@ -35,11 +34,11 @@ rust_realloc(void* ptr, size_t alignment, size_t old_size, size_t new_size) if ( size_to_sizeclass(aligned_old_size) == size_to_sizeclass(aligned_new_size)) return ptr; - void* p = ThreadAlloc::get_noncachable()->alloc(aligned_new_size); + void* p = ThreadAlloc::get().alloc(aligned_new_size); if (p) { std::memcpy(p, ptr, old_size < new_size ? old_size : new_size); - ThreadAlloc::get_noncachable()->dealloc(ptr, aligned_old_size); + ThreadAlloc::get().dealloc(ptr, aligned_old_size); } return p; } diff --git a/src/pal/pal.h b/src/pal/pal.h index e5343bd0b..00111a953 100644 --- a/src/pal/pal.h +++ b/src/pal/pal.h @@ -61,8 +61,7 @@ namespace snmalloc DefaultPal; #endif - [[noreturn]] SNMALLOC_SLOW_PATH inline SNMALLOC_COLD void - error(const char* const str) + [[noreturn]] SNMALLOC_SLOW_PATH inline void error(const char* const str) { Pal::error(str); } @@ -77,16 +76,16 @@ namespace snmalloc * disruption to PALs for platforms that do not support StrictProvenance AALs. */ template - static SNMALLOC_FAST_PATH typename std::enable_if_t< + static inline typename std::enable_if_t< !aal_supports, CapPtr()>> capptr_export(CapPtr p) { - return CapPtr()>(p.unsafe_capptr); + return CapPtr()>(p.unsafe_ptr()); } template - static SNMALLOC_FAST_PATH typename std::enable_if_t< + static inline typename std::enable_if_t< aal_supports, CapPtr()>> capptr_export(CapPtr p) @@ -107,7 +106,7 @@ namespace snmalloc { static_assert( !page_aligned || B == CBArena || B == CBChunkD || B == CBChunk); - PAL::template zero(p.unsafe_capptr, sz); + PAL::template zero(p.unsafe_ptr(), sz); } static_assert( diff --git a/src/pal/pal_noalloc.h b/src/pal/pal_noalloc.h index c35530cbe..b5c286ee1 100644 --- a/src/pal/pal_noalloc.h +++ b/src/pal/pal_noalloc.h @@ -3,6 +3,9 @@ #pragma once +#include "../aal/aal.h" +#include "pal_consts.h" + #include namespace snmalloc diff --git a/src/pal/pal_posix.h b/src/pal/pal_posix.h index 9622ec8c5..dde665ae5 100644 --- a/src/pal/pal_posix.h +++ b/src/pal/pal_posix.h @@ -1,5 +1,8 @@ #pragma once +#ifdef SNMALLOC_TRACING +# include +#endif #include "../ds/address.h" #if defined(BACKTRACE_HEADER) # include BACKTRACE_HEADER @@ -255,7 +258,7 @@ namespace snmalloc // Magic number for over-allocating chosen by the Pal // These should be further refined based on experiments. constexpr size_t min_size = - bits::is64() ? bits::one_at_bit(32) : bits::one_at_bit(28); + bits::is64() ? bits::one_at_bit(31) : bits::one_at_bit(27); for (size_t size_request = bits::max(size, min_size); size_request >= size; @@ -270,7 +273,13 @@ namespace snmalloc 0); if (p != MAP_FAILED) + { +#ifdef SNMALLOC_TRACING + std::cout << "Pal_posix reserved: " << p << " (" << size_request + << ")" << std::endl; +#endif return {p, size_request}; + } } OS::error("Out of memory"); diff --git a/src/snmalloc.h b/src/snmalloc.h index 8c2264726..87f3b5ff8 100644 --- a/src/snmalloc.h +++ b/src/snmalloc.h @@ -1,3 +1,13 @@ #pragma once -#include "mem/threadalloc.h" +// Core implementation of snmalloc independent of the configuration mode +#include "snmalloc_core.h" + +// The default configuration for snmalloc +namespace snmalloc +{ + using Alloc = snmalloc::LocalAllocator; +} + +// User facing API surface, needs to know what `Alloc` is. +#include "snmalloc_front.h" \ No newline at end of file diff --git a/src/snmalloc_core.h b/src/snmalloc_core.h new file mode 100644 index 000000000..231d79be9 --- /dev/null +++ b/src/snmalloc_core.h @@ -0,0 +1,4 @@ +#pragma once + +#include "mem/globalalloc.h" +#include "mem/globalconfig.h" \ No newline at end of file diff --git a/src/snmalloc_front.h b/src/snmalloc_front.h new file mode 100644 index 000000000..00167ddb2 --- /dev/null +++ b/src/snmalloc_front.h @@ -0,0 +1,2 @@ +#include "mem/scopedalloc.h" +#include "mem/threadalloc.h" \ No newline at end of file diff --git a/src/test/func/external_pagemap/external_pagemap.cc b/src/test/func/external_pagemap/external_pagemap.cc index 75dc1976a..6fee03f24 100644 --- a/src/test/func/external_pagemap/external_pagemap.cc +++ b/src/test/func/external_pagemap/external_pagemap.cc @@ -1,4 +1,5 @@ -#if defined(SNMALLOC_PASS_THROUGH) || defined(_WIN32) +#if defined(SNMALLOC_PASS_THROUGH) || defined(_WIN32) || \ + !defined(TODO_REINSTATE_POSSIBLY) // This test does not make sense with malloc pass-through, skip it. // The malloc definitions are also currently incompatible with Windows headers // so skip this test on Windows as well. diff --git a/src/test/func/first_operation/first_operation.cc b/src/test/func/first_operation/first_operation.cc index 05092f12a..0337b2252 100644 --- a/src/test/func/first_operation/first_operation.cc +++ b/src/test/func/first_operation/first_operation.cc @@ -7,67 +7,35 @@ #include "test/setup.h" +#include #include #include -/** - * This test is checking lazy init is correctly done with `get`. - * - * The test is written so platforms that do not do lazy init can satify the - * test. - */ -void get_test() -{ - // This should get the GlobalPlaceHolder if using lazy init - auto a1 = snmalloc::ThreadAlloc::get_noncachable(); - - // This should get a real allocator - auto a2 = snmalloc::ThreadAlloc::get(); - - // Trigger potential lazy_init if `get` didn't (shouldn't happen). - a2->dealloc(a2->alloc(5)); - - // Get an allocated allocator. - auto a3 = snmalloc::ThreadAlloc::get_noncachable(); - - if (a1 != a3) - { - printf("Lazy test!\n"); - // If the allocators are different then lazy_init has occurred. - // This should have been caused by the call to `get` rather than - // the allocations. - if (a2 != a3) - { - abort(); - } - } -} - void alloc1(size_t size) { - void* r = snmalloc::ThreadAlloc::get_noncachable()->alloc(size); - snmalloc::ThreadAlloc::get_noncachable()->dealloc(r); + void* r = snmalloc::ThreadAlloc::get().alloc(size); + snmalloc::ThreadAlloc::get().dealloc(r); } void alloc2(size_t size) { - auto a = snmalloc::ThreadAlloc::get_noncachable(); - void* r = a->alloc(size); - a->dealloc(r); + auto& a = snmalloc::ThreadAlloc::get(); + void* r = a.alloc(size); + a.dealloc(r); } void alloc3(size_t size) { - auto a = snmalloc::ThreadAlloc::get_noncachable(); - void* r = a->alloc(size); - a->dealloc(r, size); + auto& a = snmalloc::ThreadAlloc::get(); + void* r = a.alloc(size); + a.dealloc(r, size); } void alloc4(size_t size) { - auto a = snmalloc::ThreadAlloc::get(); - void* r = a->alloc(size); - a->dealloc(r); + auto& a = snmalloc::ThreadAlloc::get(); + void* r = a.alloc(size); + a.dealloc(r); } void check_calloc(void* p, size_t size) @@ -77,7 +45,16 @@ void check_calloc(void* p, size_t size) for (size_t i = 0; i < size; i++) { if (((uint8_t*)p)[i] != 0) + { + std::cout << "Calloc contents:" << std::endl; + for (size_t j = 0; j < size; j++) + { + std::cout << std::hex << (size_t)((uint8_t*)p)[j] << " "; + if (j % 32 == 0) + std::cout << std::endl; + } abort(); + } // ((uint8_t*)p)[i] = 0x5a; } } @@ -86,54 +63,53 @@ void check_calloc(void* p, size_t size) void calloc1(size_t size) { void* r = - snmalloc::ThreadAlloc::get_noncachable()->alloc( - size); + snmalloc::ThreadAlloc::get().alloc(size); check_calloc(r, size); - snmalloc::ThreadAlloc::get_noncachable()->dealloc(r); + snmalloc::ThreadAlloc::get().dealloc(r); } void calloc2(size_t size) { - auto a = snmalloc::ThreadAlloc::get_noncachable(); - void* r = a->alloc(size); + auto& a = snmalloc::ThreadAlloc::get(); + void* r = a.alloc(size); check_calloc(r, size); - a->dealloc(r); + a.dealloc(r); } void calloc3(size_t size) { - auto a = snmalloc::ThreadAlloc::get_noncachable(); - void* r = a->alloc(size); + auto& a = snmalloc::ThreadAlloc::get(); + void* r = a.alloc(size); check_calloc(r, size); - a->dealloc(r, size); + a.dealloc(r, size); } void calloc4(size_t size) { - auto a = snmalloc::ThreadAlloc::get(); - void* r = a->alloc(size); + auto& a = snmalloc::ThreadAlloc::get(); + void* r = a.alloc(size); check_calloc(r, size); - a->dealloc(r); + a.dealloc(r); } void dealloc1(void* p, size_t) { - snmalloc::ThreadAlloc::get_noncachable()->dealloc(p); + snmalloc::ThreadAlloc::get().dealloc(p); } void dealloc2(void* p, size_t size) { - snmalloc::ThreadAlloc::get_noncachable()->dealloc(p, size); + snmalloc::ThreadAlloc::get().dealloc(p, size); } void dealloc3(void* p, size_t) { - snmalloc::ThreadAlloc::get()->dealloc(p); + snmalloc::ThreadAlloc::get().dealloc(p); } void dealloc4(void* p, size_t size) { - snmalloc::ThreadAlloc::get()->dealloc(p, size); + snmalloc::ThreadAlloc::get().dealloc(p, size); } void f(size_t size) @@ -148,31 +124,32 @@ void f(size_t size) auto t7 = std::thread(calloc3, size); auto t8 = std::thread(calloc4, size); - auto a = snmalloc::current_alloc_pool()->acquire(); - auto p1 = a->alloc(size); - auto p2 = a->alloc(size); - auto p3 = a->alloc(size); - auto p4 = a->alloc(size); - - auto t9 = std::thread(dealloc1, p1, size); - auto t10 = std::thread(dealloc2, p2, size); - auto t11 = std::thread(dealloc3, p3, size); - auto t12 = std::thread(dealloc4, p4, size); - - t1.join(); - t2.join(); - t3.join(); - t4.join(); - t5.join(); - t6.join(); - t7.join(); - t8.join(); - t9.join(); - t10.join(); - t11.join(); - t12.join(); - snmalloc::current_alloc_pool()->release(a); - snmalloc::current_alloc_pool()->debug_in_use(0); + { + auto a = snmalloc::get_scoped_allocator(); + auto p1 = a->alloc(size); + auto p2 = a->alloc(size); + auto p3 = a->alloc(size); + auto p4 = a->alloc(size); + + auto t9 = std::thread(dealloc1, p1, size); + auto t10 = std::thread(dealloc2, p2, size); + auto t11 = std::thread(dealloc3, p3, size); + auto t12 = std::thread(dealloc4, p4, size); + + t1.join(); + t2.join(); + t3.join(); + t4.join(); + t5.join(); + t6.join(); + t7.join(); + t8.join(); + t9.join(); + t10.join(); + t11.join(); + t12.join(); + } // Drops a. + // snmalloc::current_alloc_pool()->debug_in_use(0); printf("."); fflush(stdout); } @@ -183,16 +160,13 @@ int main(int, char**) printf("."); fflush(stdout); - std::thread t(get_test); - t.join(); - f(0); f(1); f(3); f(5); f(7); printf("\n"); - for (size_t exp = 1; exp < snmalloc::SUPERSLAB_BITS; exp++) + for (size_t exp = 1; exp < snmalloc::MAX_SIZECLASS_BITS; exp++) { auto shifted = [exp](size_t v) { return v << exp; }; diff --git a/src/test/func/fixed_region/fixed_region.cc b/src/test/func/fixed_region/fixed_region.cc index 2059fc958..99cf2aa6a 100644 --- a/src/test/func/fixed_region/fixed_region.cc +++ b/src/test/func/fixed_region/fixed_region.cc @@ -1,7 +1,6 @@ -#define SNMALLOC_SGX -#define OPEN_ENCLAVE -#define OE_OK 0 -#define OPEN_ENCLAVE_SIMULATION +#include "mem/fixedglobalconfig.h" +#include "mem/globalconfig.h" + #include #include @@ -10,58 +9,59 @@ #endif #define assert please_use_SNMALLOC_ASSERT -extern "C" void* oe_memset_s(void* p, size_t p_size, int c, size_t size) -{ - UNUSED(p_size); - return memset(p, c, size); -} +using namespace snmalloc; -extern "C" int oe_random(void* p, size_t p_size) -{ - UNUSED(p_size); - UNUSED(p); - // Stub for random data. - return 0; -} +using FixedAlloc = LocalAllocator; -extern "C" void oe_abort() -{ - abort(); -} - -using namespace snmalloc; int main() { #ifndef SNMALLOC_PASS_THROUGH // Depends on snmalloc specific features - auto& mp = *MemoryProviderStateMixin< - DefaultPal, - DefaultArenaMap>::make(); + + // Create a standard address space to get initial allocation + // this just bypasses having to understand the test platform. + AddressSpaceManager address_space; // 28 is large enough to produce a nested allocator. // It is also large enough for the example to run in. // For 1MiB superslabs, SUPERSLAB_BITS + 4 is not big enough for the example. - size_t large_class = 28 - SUPERSLAB_BITS; - size_t size = bits::one_at_bit(SUPERSLAB_BITS + large_class); - void* oe_base = mp.reserve(large_class).unsafe_capptr; - void* oe_end = (uint8_t*)oe_base + size; - PALOpenEnclave::setup_initial_range(oe_base, oe_end); - std::cout << "Allocated region " << oe_base << " - " << oe_end << std::endl; + size_t size = bits::one_at_bit(28); + auto oe_base = address_space.reserve(size); + auto oe_end = pointer_offset(oe_base, size).unsafe_ptr(); + std::cout << "Allocated region " << oe_base.unsafe_ptr() << " - " + << pointer_offset(oe_base, size).unsafe_ptr() << std::endl; - auto a = ThreadAlloc::get(); + FixedGlobals fixed_handle; + FixedGlobals::init(oe_base, size); + FixedAlloc a(fixed_handle); + size_t object_size = 128; + size_t count = 0; while (true) { - auto r1 = a->alloc(100); + auto r1 = a.alloc(object_size); + count += object_size; // Run until we exhaust the fixed region. // This should return null. if (r1 == nullptr) - return 0; + break; - if (oe_base > r1) + if (oe_base.unsafe_ptr() > r1) + { + std::cout << "Allocated: " << r1 << std::endl; abort(); + } if (oe_end < r1) + { + std::cout << "Allocated: " << r1 << std::endl; abort(); + } } + + std::cout << "Total allocated: " << count << " out of " << size << std::endl; + std::cout << "Overhead: 1/" << (double)size / (double)(size - count) + << std::endl; + + a.teardown(); #endif } diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index cba2beb06..e87e0eeed 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -8,18 +8,29 @@ using namespace snmalloc; void check_result(size_t size, size_t align, void* p, int err, bool null) { + bool failed = false; if (errno != err) - abort(); + { + printf("Expected error: %d but got %d\n", err, errno); + failed = true; + } if (null) { if (p != nullptr) - abort(); - + { + printf("Expected null, and got non-null return!\n"); + failed = true; + } our_free(p); return; } + if ((p == nullptr) && (size != 0)) + { + printf("Unexpected null returned.\n"); + failed = true; + } const auto alloc_size = our_malloc_usable_size(p); const auto expected_size = round_size(size); #ifdef SNMALLOC_PASS_THROUGH @@ -36,7 +47,7 @@ void check_result(size_t size, size_t align, void* p, int err, bool null) "Usable size is %zu, but required to be %zu.\n", alloc_size, expected_size); - abort(); + failed = true; } if ((!exact_size) && (alloc_size < expected_size)) { @@ -44,15 +55,17 @@ void check_result(size_t size, size_t align, void* p, int err, bool null) "Usable size is %zu, but required to be at least %zu.\n", alloc_size, expected_size); - abort(); + failed = true; } - if (static_cast(reinterpret_cast(p) % align) != 0) + if ( + (static_cast(reinterpret_cast(p) % align) != 0) && + (size != 0)) { printf( "Address is 0x%zx, but required to be aligned to 0x%zx.\n", reinterpret_cast(p), align); - abort(); + failed = true; } if ( static_cast( @@ -62,15 +75,17 @@ void check_result(size_t size, size_t align, void* p, int err, bool null) "Address is 0x%zx, but should have natural alignment to 0x%zx.\n", reinterpret_cast(p), natural_alignment(size)); - abort(); + failed = true; } + if (failed) + printf("check_result failed! %p", p); our_free(p); } void test_calloc(size_t nmemb, size_t size, int err, bool null) { - fprintf(stderr, "calloc(%zu, %zu)\n", nmemb, size); + printf("calloc(%zu, %zu) combined size %zu\n", nmemb, size, nmemb * size); errno = 0; void* p = our_calloc(nmemb, size); @@ -79,7 +94,10 @@ void test_calloc(size_t nmemb, size_t size, int err, bool null) for (size_t i = 0; i < (size * nmemb); i++) { if (((uint8_t*)p)[i] != 0) + { + printf("non-zero at @%zu\n", i); abort(); + } } } check_result(nmemb * size, 1, p, err, null); @@ -91,7 +109,7 @@ void test_realloc(void* p, size_t size, int err, bool null) if (p != nullptr) old_size = our_malloc_usable_size(p); - fprintf(stderr, "realloc(%p(%zu), %zu)\n", p, old_size, size); + printf("realloc(%p(%zu), %zu)\n", p, old_size, size); errno = 0; auto new_p = our_realloc(p, size); // Realloc failure case, deallocate original block @@ -102,7 +120,7 @@ void test_realloc(void* p, size_t size, int err, bool null) void test_posix_memalign(size_t size, size_t align, int err, bool null) { - fprintf(stderr, "posix_memalign(&p, %zu, %zu)\n", align, size); + printf("posix_memalign(&p, %zu, %zu)\n", align, size); void* p = nullptr; errno = our_posix_memalign(&p, align, size); check_result(size, align, p, err, null); @@ -110,7 +128,7 @@ void test_posix_memalign(size_t size, size_t align, int err, bool null) void test_memalign(size_t size, size_t align, int err, bool null) { - fprintf(stderr, "memalign(%zu, %zu)\n", align, size); + printf("memalign(%zu, %zu)\n", align, size); errno = 0; void* p = our_memalign(align, size); check_result(size, align, p, err, null); @@ -123,11 +141,11 @@ int main(int argc, char** argv) setup(); - constexpr int SUCCESS = 0; + our_free(nullptr); - test_realloc(our_malloc(64), 4194304, SUCCESS, false); + constexpr int SUCCESS = 0; - for (sizeclass_t sc = 0; sc < (SUPERSLAB_BITS + 4); sc++) + for (sizeclass_t sc = 0; sc < (MAX_SIZECLASS_BITS + 4); sc++) { const size_t size = bits::one_at_bit(sc); printf("malloc: %zu\n", size); @@ -137,12 +155,15 @@ int main(int argc, char** argv) test_calloc(0, 0, SUCCESS, false); + our_free(nullptr); + for (sizeclass_t sc = 0; sc < NUM_SIZECLASSES; sc++) { const size_t size = sizeclass_to_size(sc); bool overflow = false; - for (size_t n = 1; bits::umul(size, n, overflow) <= SUPERSLAB_SIZE; n *= 5) + for (size_t n = 1; bits::umul(size, n, overflow) <= MAX_SIZECLASS_SIZE; + n *= 5) { if (overflow) break; @@ -168,14 +189,14 @@ int main(int argc, char** argv) } } - for (sizeclass_t sc = 0; sc < (SUPERSLAB_BITS + 4); sc++) + for (sizeclass_t sc = 0; sc < (MAX_SIZECLASS_BITS + 4); sc++) { const size_t size = bits::one_at_bit(sc); test_realloc(our_malloc(size), size, SUCCESS, false); test_realloc(our_malloc(size), 0, SUCCESS, true); test_realloc(nullptr, size, SUCCESS, false); test_realloc(our_malloc(size), (size_t)-1, ENOMEM, true); - for (sizeclass_t sc2 = 0; sc2 < (SUPERSLAB_BITS + 4); sc2++) + for (sizeclass_t sc2 = 0; sc2 < (MAX_SIZECLASS_BITS + 4); sc2++) { const size_t size2 = bits::one_at_bit(sc2); printf("size1: %zu, size2:%zu\n", size, size2); @@ -184,14 +205,16 @@ int main(int argc, char** argv) } } + test_realloc(our_malloc(64), 4194304, SUCCESS, false); + test_posix_memalign(0, 0, EINVAL, true); test_posix_memalign((size_t)-1, 0, EINVAL, true); test_posix_memalign(OS_PAGE_SIZE, sizeof(uintptr_t) / 2, EINVAL, true); - for (size_t align = sizeof(uintptr_t); align <= SUPERSLAB_SIZE * 8; + for (size_t align = sizeof(uintptr_t); align < MAX_SIZECLASS_SIZE * 8; align <<= 1) { - for (sizeclass_t sc = 0; sc < NUM_SIZECLASSES; sc++) + for (sizeclass_t sc = 0; sc < NUM_SIZECLASSES - 6; sc++) { const size_t size = sizeclass_to_size(sc); test_posix_memalign(size, align, SUCCESS, false); @@ -203,6 +226,12 @@ int main(int argc, char** argv) test_posix_memalign(0, align + 1, EINVAL, true); } - current_alloc_pool()->debug_check_empty(); + if (our_malloc_usable_size(nullptr) != 0) + { + printf("malloc_usable_size(nullptr) should be zero"); + abort(); + } + + snmalloc::debug_check_empty(Globals::get_handle()); return 0; } diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index cb8e0e7ea..9c6cd359a 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -19,13 +19,17 @@ # define KiB (1024ull) # define MiB (KiB * KiB) # define GiB (KiB * MiB) +#else +using rlim64_t = size_t; #endif using namespace snmalloc; -#ifdef TEST_LIMITED void test_limited(rlim64_t as_limit, size_t& count) { + UNUSED(as_limit); + UNUSED(count); +#if false && defined(TEST_LIMITED) auto pid = fork(); if (!pid) { @@ -54,10 +58,10 @@ void test_limited(rlim64_t as_limit, size_t& count) upper_bound = std::min( upper_bound, static_cast(info.freeram >> 3u)); std::cout << "trying to alloc " << upper_bound / KiB << " KiB" << std::endl; - auto alloc = ThreadAlloc::get(); + auto& alloc = ThreadAlloc::get(); std::cout << "allocator initialised" << std::endl; - auto chunk = alloc->alloc(upper_bound); - alloc->dealloc(chunk); + auto chunk = alloc.alloc(upper_bound); + alloc.dealloc(chunk); std::cout << "success" << std::endl; std::exit(0); } @@ -71,12 +75,12 @@ void test_limited(rlim64_t as_limit, size_t& count) count++; } } -} #endif +} void test_alloc_dealloc_64k() { - auto alloc = ThreadAlloc::get(); + auto& alloc = ThreadAlloc::get(); constexpr size_t count = 1 << 12; constexpr size_t outer_count = 12; @@ -89,26 +93,26 @@ void test_alloc_dealloc_64k() // This will fill the short slab, and then start a new slab. for (size_t i = 0; i < count; i++) { - garbage[i] = alloc->alloc(16); + garbage[i] = alloc.alloc(16); } // Allocate one object on the second slab - keep_alive[j] = alloc->alloc(16); + keep_alive[j] = alloc.alloc(16); for (size_t i = 0; i < count; i++) { - alloc->dealloc(garbage[i]); + alloc.dealloc(garbage[i]); } } for (size_t j = 0; j < outer_count; j++) { - alloc->dealloc(keep_alive[j]); + alloc.dealloc(keep_alive[j]); } } void test_random_allocation() { - auto alloc = ThreadAlloc::get(); + auto& alloc = ThreadAlloc::get(); std::unordered_set allocated; constexpr size_t count = 10000; @@ -130,14 +134,14 @@ void test_random_allocation() auto& cell = objects[index % count]; if (cell != nullptr) { - alloc->dealloc(cell); + alloc.dealloc(cell); allocated.erase(cell); cell = nullptr; alloc_count--; } if (!just_dealloc) { - cell = alloc->alloc(16); + cell = alloc.alloc(16); auto pair = allocated.insert(cell); // Check not already allocated SNMALLOC_CHECK(pair.second); @@ -155,20 +159,20 @@ void test_random_allocation() // Deallocate all the remaining objects for (size_t i = 0; i < count; i++) if (objects[i] != nullptr) - alloc->dealloc(objects[i]); + alloc.dealloc(objects[i]); } void test_calloc() { - auto alloc = ThreadAlloc::get(); + auto& alloc = ThreadAlloc::get(); for (size_t size = 16; size <= (1 << 24); size <<= 1) { - void* p = alloc->alloc(size); + void* p = alloc.alloc(size); memset(p, 0xFF, size); - alloc->dealloc(p, size); + alloc.dealloc(p, size); - p = alloc->alloc(size); + p = alloc.alloc(size); for (size_t i = 0; i < size; i++) { @@ -176,92 +180,97 @@ void test_calloc() abort(); } - alloc->dealloc(p, size); + alloc.dealloc(p, size); } - current_alloc_pool()->debug_check_empty(); + snmalloc::debug_check_empty(Globals::get_handle()); } void test_double_alloc() { - auto* a1 = current_alloc_pool()->acquire(); - auto* a2 = current_alloc_pool()->acquire(); - - const size_t n = (1 << 16) / 32; - - for (size_t k = 0; k < 4; k++) { - std::unordered_set set1; - std::unordered_set set2; + auto a1 = snmalloc::get_scoped_allocator(); + auto a2 = snmalloc::get_scoped_allocator(); - for (size_t i = 0; i < (n * 2); i++) - { - void* p = a1->alloc(20); - SNMALLOC_CHECK(set1.find(p) == set1.end()); - set1.insert(p); - } + const size_t n = (1 << 16) / 32; - for (size_t i = 0; i < (n * 2); i++) + for (size_t k = 0; k < 4; k++) { - void* p = a2->alloc(20); - SNMALLOC_CHECK(set2.find(p) == set2.end()); - set2.insert(p); - } + std::unordered_set set1; + std::unordered_set set2; - while (!set1.empty()) - { - auto it = set1.begin(); - a2->dealloc(*it, 20); - set1.erase(it); - } + for (size_t i = 0; i < (n * 2); i++) + { + void* p = a1->alloc(20); + SNMALLOC_CHECK(set1.find(p) == set1.end()); + set1.insert(p); + } - while (!set2.empty()) - { - auto it = set2.begin(); - a1->dealloc(*it, 20); - set2.erase(it); + for (size_t i = 0; i < (n * 2); i++) + { + void* p = a2->alloc(20); + SNMALLOC_CHECK(set2.find(p) == set2.end()); + set2.insert(p); + } + + while (!set1.empty()) + { + auto it = set1.begin(); + a2->dealloc(*it, 20); + set1.erase(it); + } + + while (!set2.empty()) + { + auto it = set2.begin(); + a1->dealloc(*it, 20); + set2.erase(it); + } } } - - current_alloc_pool()->release(a1); - current_alloc_pool()->release(a2); - current_alloc_pool()->debug_check_empty(); + snmalloc::debug_check_empty(Globals::get_handle()); } void test_external_pointer() { // Malloc does not have an external pointer querying mechanism. - auto alloc = ThreadAlloc::get(); + auto& alloc = ThreadAlloc::get(); for (uint8_t sc = 0; sc < NUM_SIZECLASSES; sc++) { size_t size = sizeclass_to_size(sc); - void* p1 = alloc->alloc(size); + void* p1 = alloc.alloc(size); for (size_t offset = 0; offset < size; offset += 17) { void* p2 = pointer_offset(p1, offset); - void* p3 = alloc->external_pointer(p2); - void* p4 = alloc->external_pointer(p2); + void* p3 = alloc.external_pointer(p2); + void* p4 = alloc.external_pointer(p2); if (p1 != p3) { std::cout << "size: " << size << " offset: " << offset << " p1: " << p1 << " p3: " << p3 << std::endl; } SNMALLOC_CHECK(p1 == p3); + if ((size_t)p4 != (size_t)p1 + size - 1) + { + std::cout << "size: " << size << " end(p4): " << p4 << " p1: " << p1 + << " p1+size-1: " << (void*)((size_t)p1 + size - 1) + << std::endl; + } SNMALLOC_CHECK((size_t)p4 == (size_t)p1 + size - 1); } - alloc->dealloc(p1, size); + alloc.dealloc(p1, size); } - current_alloc_pool()->debug_check_empty(); + snmalloc::debug_check_empty(Globals::get_handle()); }; void check_offset(void* base, void* interior) { - auto alloc = ThreadAlloc::get(); - void* calced_base = alloc->external_pointer((void*)interior); + auto& alloc = ThreadAlloc::get(); + void* calced_base = alloc.external_pointer((void*)interior); if (calced_base != (void*)base) abort(); } @@ -281,7 +290,7 @@ void test_external_pointer_large() { xoroshiro::p128r64 r; - auto alloc = ThreadAlloc::get(); + auto& alloc = ThreadAlloc::get(); constexpr size_t count_log = snmalloc::bits::is64() ? 5 : 3; constexpr size_t count = 1 << count_log; @@ -292,14 +301,14 @@ void test_external_pointer_large() for (size_t i = 0; i < count; i++) { - size_t b = SUPERSLAB_BITS + 3; + size_t b = MAX_SIZECLASS_BITS + 3; size_t rand = r.next() & ((1 << b) - 1); size_t size = (1 << 24) + rand; total_size += size; // store object - objects[i] = (size_t*)alloc->alloc(size); + objects[i] = (size_t*)alloc.alloc(size); // Store allocators size for this object - *objects[i] = alloc->alloc_size(objects[i]); + *objects[i] = alloc.alloc_size(objects[i]); check_external_pointer_large(objects[i]); if (i > 0) @@ -317,87 +326,87 @@ void test_external_pointer_large() // Deallocate everything for (size_t i = 0; i < count; i++) { - alloc->dealloc(objects[i]); + alloc.dealloc(objects[i]); } } void test_external_pointer_dealloc_bug() { - auto alloc = ThreadAlloc::get(); - constexpr size_t count = (SUPERSLAB_SIZE / SLAB_SIZE) * 2; + auto& alloc = ThreadAlloc::get(); + constexpr size_t count = MIN_CHUNK_SIZE; void* allocs[count]; for (size_t i = 0; i < count; i++) { - allocs[i] = alloc->alloc(SLAB_SIZE / 2); + allocs[i] = alloc.alloc(MIN_CHUNK_BITS / 2); } for (size_t i = 1; i < count; i++) { - alloc->dealloc(allocs[i]); + alloc.dealloc(allocs[i]); } for (size_t i = 0; i < count; i++) { - alloc->external_pointer(allocs[i]); + alloc.external_pointer(allocs[i]); } - alloc->dealloc(allocs[0]); + alloc.dealloc(allocs[0]); } void test_alloc_16M() { - auto alloc = ThreadAlloc::get(); + auto& alloc = ThreadAlloc::get(); // sizes >= 16M use large_alloc const size_t size = 16'000'000; - void* p1 = alloc->alloc(size); - SNMALLOC_CHECK(alloc->alloc_size(alloc->external_pointer(p1)) >= size); - alloc->dealloc(p1); + void* p1 = alloc.alloc(size); + SNMALLOC_CHECK(alloc.alloc_size(alloc.external_pointer(p1)) >= size); + alloc.dealloc(p1); } void test_calloc_16M() { - auto alloc = ThreadAlloc::get(); + auto& alloc = ThreadAlloc::get(); // sizes >= 16M use large_alloc const size_t size = 16'000'000; - void* p1 = alloc->alloc(size); - SNMALLOC_CHECK(alloc->alloc_size(alloc->external_pointer(p1)) >= size); - alloc->dealloc(p1); + void* p1 = alloc.alloc(size); + SNMALLOC_CHECK(alloc.alloc_size(alloc.external_pointer(p1)) >= size); + alloc.dealloc(p1); } void test_calloc_large_bug() { - auto alloc = ThreadAlloc::get(); + auto& alloc = ThreadAlloc::get(); // Perform large calloc, to check for correct zeroing from PAL. // Some PALS have special paths for PAGE aligned zeroing of large // allocations. This is a large allocation that is intentionally // not a multiple of page size. - const size_t size = (SUPERSLAB_SIZE << 3) - 7; + const size_t size = (MAX_SIZECLASS_SIZE << 3) - 7; - void* p1 = alloc->alloc(size); - SNMALLOC_CHECK(alloc->alloc_size(alloc->external_pointer(p1)) >= size); - alloc->dealloc(p1); + void* p1 = alloc.alloc(size); + SNMALLOC_CHECK(alloc.alloc_size(alloc.external_pointer(p1)) >= size); + alloc.dealloc(p1); } template void test_static_sized_alloc() { - auto alloc = ThreadAlloc::get(); - auto p = alloc->alloc(); + auto& alloc = ThreadAlloc::get(); + auto p = alloc.alloc(); static_assert((dealloc >= 0) && (dealloc <= 2), "bad dealloc flavor"); switch (dealloc) { case 0: - alloc->dealloc(p); + alloc.dealloc(p); break; case 1: - alloc->dealloc(p, asz); + alloc.dealloc(p, asz); break; case 2: - alloc->dealloc(p); + alloc.dealloc(p); break; } } @@ -406,18 +415,19 @@ void test_static_sized_allocs() { // For each small, medium, and large class, do each kind dealloc. This is // mostly to ensure that all of these forms compile. + for (size_t sc = 0; sc < NUM_SIZECLASSES_EXTENDED; sc++) + { + // test_static_sized_alloc(); + // test_static_sized_alloc(); + // test_static_sized_alloc(); + } + // test_static_sized_alloc(); + // test_static_sized_alloc(); + // test_static_sized_alloc(); - test_static_sized_alloc(); - test_static_sized_alloc(); - test_static_sized_alloc(); - - test_static_sized_alloc(); - test_static_sized_alloc(); - test_static_sized_alloc(); - - test_static_sized_alloc(); - test_static_sized_alloc(); - test_static_sized_alloc(); + // test_static_sized_alloc(); + // test_static_sized_alloc(); + // test_static_sized_alloc(); } int main(int argc, char** argv) diff --git a/src/test/func/memory_usage/memory_usage.cc b/src/test/func/memory_usage/memory_usage.cc index 47e4ce9c8..98e35484a 100644 --- a/src/test/func/memory_usage/memory_usage.cc +++ b/src/test/func/memory_usage/memory_usage.cc @@ -2,7 +2,6 @@ * Memory usage test * Query memory usage repeatedly */ - #include #include #include @@ -35,7 +34,7 @@ bool print_memory_usage() return false; } -std::vector allocs; +std::vector allocs{}; /** * Add allocs until the statistics have changed n times. @@ -44,7 +43,8 @@ void add_n_allocs(size_t n) { while (true) { - allocs.push_back(our_malloc(1024)); + auto p = our_malloc(1024); + allocs.push_back(p); if (print_memory_usage()) { n--; @@ -61,7 +61,10 @@ void remove_n_allocs(size_t n) { while (true) { - our_free(allocs.back()); + if (allocs.empty()) + return; + auto p = allocs.back(); + our_free(p); allocs.pop_back(); if (print_memory_usage()) { diff --git a/src/test/func/pagemap/pagemap.cc b/src/test/func/pagemap/pagemap.cc index ad961703f..5f02c0e70 100644 --- a/src/test/func/pagemap/pagemap.cc +++ b/src/test/func/pagemap/pagemap.cc @@ -6,50 +6,129 @@ * but no examples were using multiple levels of pagemap. */ +#include #include #include #include #include using namespace snmalloc; -using T = size_t; -static constexpr size_t GRANULARITY_BITS = 9; -static constexpr T PRIME = 251; -Pagemap pagemap_test; - -int main(int argc, char** argv) +static constexpr size_t GRANULARITY_BITS = 20; +struct T { - UNUSED(argc); - UNUSED(argv); + size_t v = 99; + T(size_t v) : v(v) {} + T() {} +}; - setup(); +AddressSpaceManager address_space; + +FlatPagemap pagemap_test_unbound; +FlatPagemap pagemap_test_bound; + +size_t failure_count = 0; + +void check_get( + bool bounded, address_t address, T expected, const char* file, size_t lineno) +{ T value = 0; - for (uintptr_t ptr = 0; ptr < bits::one_at_bit(36); + if (bounded) + value = pagemap_test_bound.get(address); + else + value = pagemap_test_unbound.get(address); + + if (value.v != expected.v) + { + std::cout << "Location: " << (void*)address << " Read: " << value.v + << " Expected: " << expected.v << " on " << file << ":" << lineno + << std::endl; + failure_count++; + } +} + +void add(bool bounded, address_t address, T new_value) +{ + if (bounded) + pagemap_test_bound.add(address, new_value); + else + pagemap_test_unbound.add(address, new_value); +} + +void set(bool bounded, address_t address, T new_value) +{ + if (bounded) + pagemap_test_bound.set(address, new_value); + else + pagemap_test_unbound.set(address, new_value); +} + +#define CHECK_GET(b, a, e) check_get(b, a, e, __FILE__, __LINE__) + +void test_pagemap(bool bounded) +{ + address_t low = bits::one_at_bit(23); + address_t high = bits::one_at_bit(30); + + // Nullptr needs to work before initialisation + CHECK_GET(true, 0, T()); + + // Initialise the pagemap + if (bounded) + { + pagemap_test_bound.init(&address_space, low, high); + } + else + { + pagemap_test_unbound.init(&address_space); + } + + // Nullptr should still work after init. + CHECK_GET(true, 0, T()); + + // Store a pattern into page map + T value = 1; + for (uintptr_t ptr = low; ptr < high; ptr += bits::one_at_bit(GRANULARITY_BITS + 3)) { - pagemap_test.set(ptr, value); - value++; - if (value == PRIME) + add(false, ptr, value); + value.v++; + if (value.v == T().v) value = 0; - if ((ptr % (1ULL << 32)) == 0) + if ((ptr % (1ULL << 26)) == 0) std::cout << "." << std::flush; } + // Check pattern is correctly stored std::cout << std::endl; - value = 0; - for (uintptr_t ptr = 0; ptr < bits::one_at_bit(36); + value = 1; + for (uintptr_t ptr = low; ptr < high; ptr += bits::one_at_bit(GRANULARITY_BITS + 3)) { - T result = pagemap_test.get(ptr); - if (value != result) - Pal::error("Pagemap corrupt!"); - value++; - if (value == PRIME) + CHECK_GET(false, ptr, value); + value.v++; + if (value.v == T().v) value = 0; - if ((ptr % (1ULL << 32)) == 0) + if ((ptr % (1ULL << 26)) == 0) std::cout << "." << std::flush; } std::cout << std::endl; } + +int main(int argc, char** argv) +{ + UNUSED(argc); + UNUSED(argv); + + setup(); + + test_pagemap(false); + test_pagemap(true); + + if (failure_count != 0) + { + std::cout << "Failure count: " << failure_count << std::endl; + abort(); + } +} diff --git a/src/test/func/release-rounding/rounding.cc b/src/test/func/release-rounding/rounding.cc index b0c391cb7..0d9d15988 100644 --- a/src/test/func/release-rounding/rounding.cc +++ b/src/test/func/release-rounding/rounding.cc @@ -1,6 +1,6 @@ +#include #include #include - using namespace snmalloc; // Check for all sizeclass that we correctly round every offset within @@ -18,8 +18,7 @@ int main(int argc, char** argv) for (size_t size_class = 0; size_class < NUM_SIZECLASSES; size_class++) { size_t rsize = sizeclass_to_size((uint8_t)size_class); - size_t max_offset = - size_class < NUM_SMALL_CLASSES ? SLAB_SIZE : SUPERSLAB_SIZE; + size_t max_offset = sizeclass_to_slab_size(size_class); for (size_t offset = 0; offset < max_offset; offset++) { size_t rounded = (offset / rsize) * rsize; diff --git a/src/test/func/sandbox/sandbox.cc b/src/test/func/sandbox/sandbox.cc index 1b4ffa352..519488070 100644 --- a/src/test/func/sandbox/sandbox.cc +++ b/src/test/func/sandbox/sandbox.cc @@ -1,4 +1,4 @@ -#ifdef SNMALLOC_PASS_THROUGH +#if defined(SNMALLOC_PASS_THROUGH) || true /* * This test does not make sense with malloc pass-through, skip it. */ @@ -16,25 +16,14 @@ using namespace snmalloc; namespace { - /** - * Helper for Alloc that is never used as a thread-local allocator and so is - * always initialised. - * - * CapPtr-vs-MSVC triggering; xref CapPtr's constructor - */ - bool never_init(void*) - { - return false; - } /** * Helper for Alloc that never needs lazy initialisation. * * CapPtr-vs-MSVC triggering; xref CapPtr's constructor */ - void* no_op_init(function_ref) + void no_op_register_clean_up() { SNMALLOC_CHECK(0 && "Should never be called!"); - return nullptr; } /** * Sandbox class. Allocates a memory region and an allocator that can @@ -71,7 +60,7 @@ namespace * outside the sandbox proper: no memory allocation operations and * amplification confined to sandbox memory. */ - using NoOpMemoryProvider = MemoryProviderStateMixin; + using NoOpMemoryProvider = ChunkAllocator; /** * Type for the allocator that lives outside of the sandbox and allocates @@ -81,12 +70,12 @@ namespace * memory. It (insecurely) routes messages to in-sandbox snmallocs, * though, so it can free any sandbox-backed snmalloc allocation. */ - using ExternalAlloc = Allocator< - never_init, - no_op_init, - NoOpMemoryProvider, - SNMALLOC_DEFAULT_CHUNKMAP, - false>; + using ExternalCoreAlloc = + Allocator; + + using ExternalAlloc = + LocalAllocator; + /** * Proxy class that forwards requests for large allocations to the real * memory provider. @@ -158,8 +147,9 @@ namespace * Note that a real version of this would not have access to the shared * pagemap and would not be used outside of the sandbox. */ + using InternalCoreAlloc = Allocator; using InternalAlloc = - Allocator; + LocalAllocator; /** * The start of the sandbox memory region. @@ -253,7 +243,7 @@ namespace // Use the outside-sandbox snmalloc to allocate memory, rather than using // the PAL directly, so that our out-of-sandbox can amplify sandbox // pointers - return ThreadAlloc::get_noncachable()->alloc(sb_size); + return ThreadAlloc::get().alloc(sb_size); } }; } @@ -269,7 +259,7 @@ int main() auto check = [](Sandbox& sb, auto& alloc, size_t sz) { void* ptr = alloc.alloc(sz); SNMALLOC_CHECK(sb.is_in_sandbox_heap(ptr, sz)); - ThreadAlloc::get_noncachable()->dealloc(ptr); + ThreadAlloc::get().dealloc(ptr); }; auto check_with_sb = [&](Sandbox& sb) { // Check with a range of sizes diff --git a/src/test/func/sizeclass/sizeclass.cc b/src/test/func/sizeclass/sizeclass.cc index 696d0140b..3d4709e45 100644 --- a/src/test/func/sizeclass/sizeclass.cc +++ b/src/test/func/sizeclass/sizeclass.cc @@ -38,7 +38,8 @@ void test_align_size() failed |= true; } - for (size_t alignment_bits = 0; alignment_bits < snmalloc::SUPERSLAB_BITS; + for (size_t alignment_bits = 0; + alignment_bits < snmalloc::MAX_SIZECLASS_BITS; alignment_bits++) { auto alignment = (size_t)1 << alignment_bits; @@ -76,11 +77,17 @@ int main(int, char**) std::cout << "sizeclass |-> [size_low, size_high] " << std::endl; - for (snmalloc::sizeclass_t sz = 0; sz < snmalloc::NUM_SIZECLASSES; sz++) + size_t slab_size = 0; + for (snmalloc::sizeclass_t sz = 0; sz < snmalloc::NUM_SIZECLASSES + 20; sz++) { // Separate printing for small and medium sizeclasses - if (sz == snmalloc::NUM_SMALL_CLASSES) + if ( + sz < snmalloc::NUM_SIZECLASSES && + slab_size != snmalloc::sizeclass_to_slab_size(sz)) + { + slab_size = snmalloc::sizeclass_to_slab_size(sz); std::cout << std::endl; + } size_t size = snmalloc::sizeclass_to_size(sz); std::cout << (size_t)sz << " |-> " diff --git a/src/test/func/statistics/stats.cc b/src/test/func/statistics/stats.cc index bd9dfec86..36b47889d 100644 --- a/src/test/func/statistics/stats.cc +++ b/src/test/func/statistics/stats.cc @@ -3,36 +3,36 @@ int main() { #ifndef SNMALLOC_PASS_THROUGH // This test depends on snmalloc internals - snmalloc::Alloc* a = snmalloc::ThreadAlloc::get(); + snmalloc::Alloc& a = snmalloc::ThreadAlloc::get(); bool result; - auto r = a->alloc(16); + auto r = a.alloc(16); - snmalloc::current_alloc_pool()->debug_check_empty(&result); + snmalloc::debug_check_empty(snmalloc::Globals::get_handle(), &result); if (result != false) { abort(); } - a->dealloc(r); + a.dealloc(r); - snmalloc::current_alloc_pool()->debug_check_empty(&result); + snmalloc::debug_check_empty(snmalloc::Globals::get_handle(), &result); if (result != true) { abort(); } - r = a->alloc(16); + r = a.alloc(16); - snmalloc::current_alloc_pool()->debug_check_empty(&result); + snmalloc::debug_check_empty(snmalloc::Globals::get_handle(), &result); if (result != false) { abort(); } - a->dealloc(r); + a.dealloc(r); - snmalloc::current_alloc_pool()->debug_check_empty(&result); + snmalloc::debug_check_empty(snmalloc::Globals::get_handle(), &result); if (result != true) { abort(); diff --git a/src/test/func/teardown/teardown.cc b/src/test/func/teardown/teardown.cc new file mode 100644 index 000000000..7bfdbff20 --- /dev/null +++ b/src/test/func/teardown/teardown.cc @@ -0,0 +1,209 @@ +/** + * After a thread has started teardown a different path is taken for + * allocation and deallocation. This tests causes the state to be torn + * down early, and then use the teardown path for multiple allocations + * and deallocation. + */ + +#include "test/setup.h" + +#include +#include +#include + +void trigger_teardown() +{ + auto& a = snmalloc::ThreadAlloc::get(); + // Trigger init + void* r = a.alloc(16); + a.dealloc(r); + // Force teardown + a.teardown(); +} + +void alloc1(size_t size) +{ + trigger_teardown(); + void* r = snmalloc::ThreadAlloc::get().alloc(size); + snmalloc::ThreadAlloc::get().dealloc(r); +} + +void alloc2(size_t size) +{ + trigger_teardown(); + auto& a = snmalloc::ThreadAlloc::get(); + void* r = a.alloc(size); + a.dealloc(r); +} + +void alloc3(size_t size) +{ + trigger_teardown(); + auto& a = snmalloc::ThreadAlloc::get(); + void* r = a.alloc(size); + a.dealloc(r, size); +} + +void alloc4(size_t size) +{ + trigger_teardown(); + auto& a = snmalloc::ThreadAlloc::get(); + void* r = a.alloc(size); + a.dealloc(r); +} + +void check_calloc(void* p, size_t size) +{ + if (p != nullptr) + { + for (size_t i = 0; i < size; i++) + { + if (((uint8_t*)p)[i] != 0) + { + std::cout << "Calloc contents:" << std::endl; + for (size_t j = 0; j < size; j++) + { + std::cout << std::hex << (size_t)((uint8_t*)p)[j] << " "; + if (j % 32 == 0) + std::cout << std::endl; + } + abort(); + } + // ((uint8_t*)p)[i] = 0x5a; + } + } +} + +void calloc1(size_t size) +{ + trigger_teardown(); + void* r = + snmalloc::ThreadAlloc::get().alloc(size); + check_calloc(r, size); + snmalloc::ThreadAlloc::get().dealloc(r); +} + +void calloc2(size_t size) +{ + trigger_teardown(); + auto& a = snmalloc::ThreadAlloc::get(); + void* r = a.alloc(size); + check_calloc(r, size); + a.dealloc(r); +} + +void calloc3(size_t size) +{ + trigger_teardown(); + auto& a = snmalloc::ThreadAlloc::get(); + void* r = a.alloc(size); + check_calloc(r, size); + a.dealloc(r, size); +} + +void calloc4(size_t size) +{ + trigger_teardown(); + auto& a = snmalloc::ThreadAlloc::get(); + void* r = a.alloc(size); + check_calloc(r, size); + a.dealloc(r); +} + +void dealloc1(void* p, size_t) +{ + trigger_teardown(); + snmalloc::ThreadAlloc::get().dealloc(p); +} + +void dealloc2(void* p, size_t size) +{ + trigger_teardown(); + snmalloc::ThreadAlloc::get().dealloc(p, size); +} + +void dealloc3(void* p, size_t) +{ + trigger_teardown(); + snmalloc::ThreadAlloc::get().dealloc(p); +} + +void dealloc4(void* p, size_t size) +{ + trigger_teardown(); + snmalloc::ThreadAlloc::get().dealloc(p, size); +} + +void f(size_t size) +{ + auto t1 = std::thread(alloc1, size); + auto t2 = std::thread(alloc2, size); + auto t3 = std::thread(alloc3, size); + auto t4 = std::thread(alloc4, size); + + auto t5 = std::thread(calloc1, size); + auto t6 = std::thread(calloc2, size); + auto t7 = std::thread(calloc3, size); + auto t8 = std::thread(calloc4, size); + + { + auto a = snmalloc::get_scoped_allocator(); + auto p1 = a->alloc(size); + auto p2 = a->alloc(size); + auto p3 = a->alloc(size); + auto p4 = a->alloc(size); + + auto t9 = std::thread(dealloc1, p1, size); + auto t10 = std::thread(dealloc2, p2, size); + auto t11 = std::thread(dealloc3, p3, size); + auto t12 = std::thread(dealloc4, p4, size); + + t1.join(); + t2.join(); + t3.join(); + t4.join(); + t5.join(); + t6.join(); + t7.join(); + t8.join(); + t9.join(); + t10.join(); + t11.join(); + t12.join(); + } // Drops a. + // snmalloc::current_alloc_pool()->debug_in_use(0); + printf("."); + fflush(stdout); +} + +int main(int, char**) +{ + setup(); + printf("."); + fflush(stdout); + + f(0); + f(1); + f(3); + f(5); + f(7); + printf("\n"); + for (size_t exp = 1; exp < snmalloc::MAX_SIZECLASS_BITS; exp++) + { + auto shifted = [exp](size_t v) { return v << exp; }; + + f(shifted(1)); + f(shifted(3)); + f(shifted(5)); + f(shifted(7)); + f(shifted(1) + 1); + f(shifted(3) + 1); + f(shifted(5) + 1); + f(shifted(7) + 1); + f(shifted(1) - 1); + f(shifted(3) - 1); + f(shifted(5) - 1); + f(shifted(7) - 1); + printf("\n"); + } +} diff --git a/src/test/func/thread_alloc_external/thread_alloc_external.cc b/src/test/func/thread_alloc_external/thread_alloc_external.cc index f5427595b..740382480 100644 --- a/src/test/func/thread_alloc_external/thread_alloc_external.cc +++ b/src/test/func/thread_alloc_external/thread_alloc_external.cc @@ -1,37 +1,40 @@ +#include #include +// Specify using own #define SNMALLOC_EXTERNAL_THREAD_ALLOC -#include +namespace snmalloc +{ + using Alloc = snmalloc::LocalAllocator; +} + using namespace snmalloc; -class ThreadAllocUntyped +class ThreadAllocExternal { public: - static void* get() + static Alloc& get() { - static thread_local void* alloc = nullptr; - if (alloc != nullptr) - { - return alloc; - } - - alloc = current_alloc_pool()->acquire(); + static thread_local Alloc alloc; return alloc; } }; -#include +#include int main() { setup(); + ThreadAlloc::get().init(); - auto a = ThreadAlloc::get(); + auto& a = ThreadAlloc::get(); for (size_t i = 0; i < 1000; i++) { - auto r1 = a->alloc(i); + auto r1 = a.alloc(i); - a->dealloc(r1); + a.dealloc(r1); } + + ThreadAlloc::get().teardown(); } diff --git a/src/test/func/two_alloc_types/alloc1.cc b/src/test/func/two_alloc_types/alloc1.cc index 11a3b230d..08efc3898 100644 --- a/src/test/func/two_alloc_types/alloc1.cc +++ b/src/test/func/two_alloc_types/alloc1.cc @@ -1,15 +1,24 @@ -#undef SNMALLOC_USE_LARGE_CHUNKS -#define OPEN_ENCLAVE -#define OE_OK 0 -#define OPEN_ENCLAVE_SIMULATION -#define NO_BOOTSTRAP_ALLOCATOR -#define SNMALLOC_EXPOSE_PAGEMAP -#define SNMALLOC_NAME_MANGLE(a) enclave_##a +#define SNMALLOC_TRACING + // Redefine the namespace, so we can have two versions. #define snmalloc snmalloc_enclave + +#include +#include + +// Specify type of allocator +#define SNMALLOC_PROVIDE_OWN_CONFIG +namespace snmalloc +{ + using Alloc = LocalAllocator; +} + +#define SNMALLOC_NAME_MANGLE(a) enclave_##a #include "../../../override/malloc.cc" extern "C" void oe_allocator_init(void* base, void* end) { - snmalloc_enclave::PALOpenEnclave::setup_initial_range(base, end); + snmalloc::FixedGlobals fixed_handle; + fixed_handle.init( + CapPtr(base), address_cast(end) - address_cast(base)); } diff --git a/src/test/func/two_alloc_types/alloc2.cc b/src/test/func/two_alloc_types/alloc2.cc index 21316ebd9..9bdfbf8d0 100644 --- a/src/test/func/two_alloc_types/alloc2.cc +++ b/src/test/func/two_alloc_types/alloc2.cc @@ -1,10 +1,6 @@ -// Remove parameters feed from test harness -#undef SNMALLOC_USE_LARGE_CHUNKS -#undef SNMALLOC_USE_SMALL_CHUNKS +#define SNMALLOC_TRACING #define SNMALLOC_NAME_MANGLE(a) host_##a -#define NO_BOOTSTRAP_ALLOCATOR -#define SNMALLOC_EXPOSE_PAGEMAP // Redefine the namespace, so we can have two versions. #define snmalloc snmalloc_host #include "../../../override/malloc.cc" diff --git a/src/test/func/two_alloc_types/main.cc b/src/test/func/two_alloc_types/main.cc index 2daf44026..b722a7ec3 100644 --- a/src/test/func/two_alloc_types/main.cc +++ b/src/test/func/two_alloc_types/main.cc @@ -31,35 +31,20 @@ extern "C" void host_free(void*); extern "C" void* enclave_malloc(size_t); extern "C" void enclave_free(void*); -extern "C" void* -enclave_snmalloc_chunkmap_global_get(snmalloc::PagemapConfig const**); -extern "C" void* -host_snmalloc_chunkmap_global_get(snmalloc::PagemapConfig const**); - using namespace snmalloc; int main() { setup(); - MemoryProviderStateMixin< - DefaultPal, - DefaultArenaMap> - mp; - // 26 is large enough to produce a nested allocator. - // It is also large enough for the example to run in. - // For 1MiB superslabs, SUPERSLAB_BITS + 2 is not big enough for the example. - size_t large_class = 26 - SUPERSLAB_BITS; - size_t size = bits::one_at_bit(SUPERSLAB_BITS + large_class); - void* oe_base = mp.reserve(large_class).unsafe_capptr; - void* oe_end = (uint8_t*)oe_base + size; + // many other sizes would work. + size_t length = bits::one_at_bit(26); + auto oe_base = host_malloc(length); + + auto oe_end = pointer_offset(oe_base, length); oe_allocator_init(oe_base, oe_end); - std::cout << "Allocated region " << oe_base << " - " << oe_end << std::endl; - // Call these functions to trigger asserts if the cast-to-self doesn't work. - const PagemapConfig* c; - enclave_snmalloc_chunkmap_global_get(&c); - host_snmalloc_chunkmap_global_get(&c); + std::cout << "Allocated region " << oe_base << " - " << oe_end << std::endl; auto a = host_malloc(128); auto b = enclave_malloc(128); @@ -68,5 +53,7 @@ int main() std::cout << "Enclave alloc " << b << std::endl; host_free(a); + std::cout << "Host freed!" << std::endl; enclave_free(b); + std::cout << "Enclace freed!" << std::endl; } diff --git a/src/test/perf/contention/contention.cc b/src/test/perf/contention/contention.cc index bcc66d310..5be5641c5 100644 --- a/src/test/perf/contention/contention.cc +++ b/src/test/perf/contention/contention.cc @@ -75,13 +75,13 @@ size_t swapcount; void test_tasks_f(size_t id) { - Alloc* a = ThreadAlloc::get(); + auto& a = ThreadAlloc::get(); xoroshiro::p128r32 r(id + 5000); for (size_t n = 0; n < swapcount; n++) { size_t size = 16 + (r.next() % 1024); - size_t* res = (size_t*)(use_malloc ? malloc(size) : a->alloc(size)); + size_t* res = (size_t*)(use_malloc ? malloc(size) : a.alloc(size)); *res = size; size_t* out = @@ -93,14 +93,14 @@ void test_tasks_f(size_t id) if (use_malloc) free(out); else - a->dealloc(out, size); + a.dealloc(out, size); } } }; void test_tasks(size_t num_tasks, size_t count, size_t size) { - Alloc* a = ThreadAlloc::get(); + auto& a = ThreadAlloc::get(); contention = new std::atomic[size]; xoroshiro::p128r32 r; @@ -109,7 +109,7 @@ void test_tasks(size_t num_tasks, size_t count, size_t size) { size_t alloc_size = 16 + (r.next() % 1024); size_t* res = - (size_t*)(use_malloc ? malloc(alloc_size) : a->alloc(alloc_size)); + (size_t*)(use_malloc ? malloc(alloc_size) : a.alloc(alloc_size)); *res = alloc_size; contention[n] = res; } @@ -134,7 +134,7 @@ void test_tasks(size_t num_tasks, size_t count, size_t size) if (use_malloc) free(contention[n]); else - a->dealloc(contention[n], *contention[n]); + a.dealloc(contention[n], *contention[n]); } } @@ -142,7 +142,7 @@ void test_tasks(size_t num_tasks, size_t count, size_t size) } #ifndef NDEBUG - current_alloc_pool()->debug_check_empty(); + snmalloc::debug_check_empty(Globals::get_handle()); #endif }; diff --git a/src/test/perf/external_pointer/externalpointer.cc b/src/test/perf/external_pointer/externalpointer.cc index 0a88ea5d9..b927daf7a 100644 --- a/src/test/perf/external_pointer/externalpointer.cc +++ b/src/test/perf/external_pointer/externalpointer.cc @@ -13,7 +13,7 @@ namespace test // Pre allocate all the objects size_t* objects[count]; - NOINLINE void setup(xoroshiro::p128r64& r, Alloc* alloc) + NOINLINE void setup(xoroshiro::p128r64& r, Alloc& alloc) { for (size_t i = 0; i < count; i++) { @@ -31,26 +31,26 @@ namespace test if (size < 16) size = 16; // store object - objects[i] = (size_t*)alloc->alloc(size); + objects[i] = (size_t*)alloc.alloc(size); // Store allocators size for this object - *objects[i] = alloc->alloc_size(objects[i]); + *objects[i] = alloc.alloc_size(objects[i]); } } - NOINLINE void teardown(Alloc* alloc) + NOINLINE void teardown(Alloc& alloc) { // Deallocate everything for (size_t i = 0; i < count; i++) { - alloc->dealloc(objects[i]); + alloc.dealloc(objects[i]); } - current_alloc_pool()->debug_check_empty(); + snmalloc::debug_check_empty(Globals::get_handle()); } void test_external_pointer(xoroshiro::p128r64& r) { - auto alloc = ThreadAlloc::get(); + auto& alloc = ThreadAlloc::get(); #ifdef NDEBUG static constexpr size_t iterations = 10000000; #else @@ -75,7 +75,7 @@ namespace test size_t size = *external_ptr; size_t offset = (size >> 4) * (rand & 15); void* interior_ptr = pointer_offset(external_ptr, offset); - void* calced_external = alloc->external_pointer(interior_ptr); + void* calced_external = alloc.external_pointer(interior_ptr); if (calced_external != external_ptr) abort(); } diff --git a/src/test/perf/low_memory/low-memory.cc b/src/test/perf/low_memory/low-memory.cc index 22ba0ba04..aa024d451 100644 --- a/src/test/perf/low_memory/low-memory.cc +++ b/src/test/perf/low_memory/low-memory.cc @@ -19,7 +19,7 @@ class Queue Node* new_node(size_t size) { - auto result = (Node*)ThreadAlloc::get()->alloc(size); + auto result = (Node*)ThreadAlloc::get().alloc(size); result->next = nullptr; return result; } @@ -43,7 +43,7 @@ class Queue return false; Node* next = head->next; - ThreadAlloc::get()->dealloc(head); + ThreadAlloc::get().dealloc(head); head = next; return true; } @@ -107,58 +107,61 @@ int main(int argc, char** argv) { opt::Opt opt(argc, argv); - if constexpr (pal_supports) - { - register_for_pal_notifications(); - } - else - { - std::cout << "Pal does not support low-memory notification! Test not run" - << std::endl; - return 0; - } - -#ifdef NDEBUG -# if defined(WIN32) && !defined(SNMALLOC_VA_BITS_64) - std::cout << "32-bit windows not supported for this test." << std::endl; -# else - - bool interactive = opt.has("--interactive"); - - Queue allocations; - - std::cout - << "Expected use:" << std::endl - << " run first instances with --interactive. Wait for first to print " - << std::endl - << " 'No allocations left. Press any key to terminate'" << std::endl - << "watch working set, and start second instance working set of first " - << "should drop to almost zero," << std::endl - << "and second should climb to physical ram." << std::endl - << std::endl; - - setup(); - - for (size_t i = 0; i < 10; i++) - { - reach_pressure(allocations); - std::cout << "Pressure " << i << std::endl; - - reduce_pressure(allocations); - } - - // Deallocate everything - while (allocations.try_remove()) - ; - - if (interactive) - { - std::cout << "No allocations left. Press any key to terminate" << std::endl; - getchar(); - } -# endif -#else - std::cout << "Release test only." << std::endl; -#endif + // TODO reinstate + + // if constexpr (pal_supports) + // { + // register_for_pal_notifications(); + // } + // else + // { + // std::cout << "Pal does not support low-memory notification! Test not + // run" + // << std::endl; + // return 0; + // } + + // #ifdef NDEBUG + // # if defined(WIN32) && !defined(SNMALLOC_VA_BITS_64) + // std::cout << "32-bit windows not supported for this test." << std::endl; + // # else + + // bool interactive = opt.has("--interactive"); + + // Queue allocations; + + // std::cout + // << "Expected use:" << std::endl + // << " run first instances with --interactive. Wait for first to print " + // << std::endl + // << " 'No allocations left. Press any key to terminate'" << std::endl + // << "watch working set, and start second instance working set of first " + // << "should drop to almost zero," << std::endl + // << "and second should climb to physical ram." << std::endl + // << std::endl; + + // setup(); + + // for (size_t i = 0; i < 10; i++) + // { + // reach_pressure(allocations); + // std::cout << "Pressure " << i << std::endl; + + // reduce_pressure(allocations); + // } + + // // Deallocate everything + // while (allocations.try_remove()) + // ; + + // if (interactive) + // { + // std::cout << "No allocations left. Press any key to terminate" << + // std::endl; getchar(); + // } + // # endif + // #else + // std::cout << "Release test only." << std::endl; + // #endif return 0; } \ No newline at end of file diff --git a/src/test/perf/singlethread/singlethread.cc b/src/test/perf/singlethread/singlethread.cc index 61dee2f7b..2bf49809c 100644 --- a/src/test/perf/singlethread/singlethread.cc +++ b/src/test/perf/singlethread/singlethread.cc @@ -8,7 +8,7 @@ using namespace snmalloc; template void test_alloc_dealloc(size_t count, size_t size, bool write) { - auto* alloc = ThreadAlloc::get(); + auto& alloc = ThreadAlloc::get(); { MeasureTime m; @@ -20,7 +20,7 @@ void test_alloc_dealloc(size_t count, size_t size, bool write) // alloc 1.5x objects for (size_t i = 0; i < ((count * 3) / 2); i++) { - void* p = alloc->alloc(size); + void* p = alloc.alloc(size); SNMALLOC_CHECK(set.find(p) == set.end()); if (write) @@ -34,7 +34,7 @@ void test_alloc_dealloc(size_t count, size_t size, bool write) { auto it = set.begin(); void* p = *it; - alloc->dealloc(p, size); + alloc.dealloc(p, size); set.erase(it); SNMALLOC_CHECK(set.find(p) == set.end()); } @@ -42,7 +42,7 @@ void test_alloc_dealloc(size_t count, size_t size, bool write) // alloc 1x objects for (size_t i = 0; i < count; i++) { - void* p = alloc->alloc(size); + void* p = alloc.alloc(size); SNMALLOC_CHECK(set.find(p) == set.end()); if (write) @@ -55,12 +55,12 @@ void test_alloc_dealloc(size_t count, size_t size, bool write) while (!set.empty()) { auto it = set.begin(); - alloc->dealloc(*it, size); + alloc.dealloc(*it, size); set.erase(it); } } - current_alloc_pool()->debug_check_empty(); + snmalloc::debug_check_empty(Globals::get_handle()); } int main(int, char**) From 4b9ead80665b1da85bb4ef620232cb2cfc8b0c80 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 12 Jul 2021 19:50:36 +0100 Subject: [PATCH 002/302] Fix register_clean_up being called with ScopedAllocator. (#344) --- src/mem/scopedalloc.h | 5 ++++- .../thread_alloc_external/thread_alloc_external.cc | 11 +++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/mem/scopedalloc.h b/src/mem/scopedalloc.h index d0e179e20..345635a70 100644 --- a/src/mem/scopedalloc.h +++ b/src/mem/scopedalloc.h @@ -25,7 +25,10 @@ namespace snmalloc /** * Constructor. Claims an allocator from the global pool */ - ScopedAllocator() = default; + ScopedAllocator() + { + alloc.init(); + }; /** * Copying is not supported, it could easily lead to accidental sharing of diff --git a/src/test/func/thread_alloc_external/thread_alloc_external.cc b/src/test/func/thread_alloc_external/thread_alloc_external.cc index 740382480..16fed068c 100644 --- a/src/test/func/thread_alloc_external/thread_alloc_external.cc +++ b/src/test/func/thread_alloc_external/thread_alloc_external.cc @@ -37,4 +37,15 @@ int main() } ThreadAlloc::get().teardown(); + + // This checks that the scoped allocator does not call + // register clean up, as this configuration will fault + // if that occurs. + auto a2 = ScopedAllocator(); + for (size_t i = 0; i < 1000; i++) + { + auto r1 = a2->alloc(i); + + a2->dealloc(r1); + } } From 39c2df0b56edc765613764cc1d42f2de8060dc24 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Tue, 13 Jul 2021 13:25:01 +0100 Subject: [PATCH 003/302] pagemap: don't eagerly commit on LazyCommit PALs --- src/backend/pagemap.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/pagemap.h b/src/backend/pagemap.h index 688e51d36..843e8df6c 100644 --- a/src/backend/pagemap.h +++ b/src/backend/pagemap.h @@ -129,7 +129,7 @@ namespace snmalloc } // This means external pointer on Windows will be slow. - if constexpr (potentially_out_of_range) + if constexpr (potentially_out_of_range && !pal_supports) { commit_entry(&body[p >> SHIFT]); } From d0ecba5280717c813acfdde0c9acaf528d9dc5c4 Mon Sep 17 00:00:00 2001 From: Istvan Haller <31476121+ihaller@users.noreply.github.com> Date: Thu, 15 Jul 2021 15:06:47 +0100 Subject: [PATCH 004/302] Improved OEPal integration with the new snmalloc architecture (#346) * Improved OEPal integration with the new snmalloc architecture * Applied PR feedback --- src/mem/fixedglobalconfig.h | 7 +-- src/pal/pal.h | 4 +- src/pal/pal_open_enclave.h | 57 ++++++---------------- src/test/func/fixed_region/fixed_region.cc | 7 +-- src/test/func/two_alloc_types/alloc1.cc | 5 +- 5 files changed, 27 insertions(+), 53 deletions(-) diff --git a/src/mem/fixedglobalconfig.h b/src/mem/fixedglobalconfig.h index 146993d5b..4878453c7 100644 --- a/src/mem/fixedglobalconfig.h +++ b/src/mem/fixedglobalconfig.h @@ -11,20 +11,21 @@ namespace snmalloc /** * A single fixed address range allocator configuration */ + template class FixedGlobals : public CommonConfig { public: - using Backend = BackendAllocator, true>; + using Backend = BackendAllocator; private: - inline static Backend::GlobalState backend_state; + inline static typename Backend::GlobalState backend_state; inline static ChunkAllocatorState slab_allocator_state; inline static PoolState> alloc_pool; public: - static Backend::GlobalState& get_backend_state() + static typename Backend::GlobalState& get_backend_state() { return backend_state; } diff --git a/src/pal/pal.h b/src/pal/pal.h index 00111a953..d95ef0f3b 100644 --- a/src/pal/pal.h +++ b/src/pal/pal.h @@ -16,11 +16,11 @@ # include "pal_haiku.h" # include "pal_linux.h" # include "pal_netbsd.h" -# include "pal_noalloc.h" # include "pal_openbsd.h" # include "pal_solaris.h" # include "pal_windows.h" #endif +#include "pal_noalloc.h" #include "pal_plain.h" namespace snmalloc @@ -56,7 +56,7 @@ namespace snmalloc #if defined(SNMALLOC_MEMORY_PROVIDER) PALPlainMixin; #elif defined(OPEN_ENCLAVE) - PALPlainMixin; + PALOpenEnclave; #else DefaultPal; #endif diff --git a/src/pal/pal_open_enclave.h b/src/pal/pal_open_enclave.h index 3487d3f93..d6036eb11 100644 --- a/src/pal/pal_open_enclave.h +++ b/src/pal/pal_open_enclave.h @@ -1,10 +1,7 @@ #pragma once -#include "ds/address.h" -#include "ds/flaglock.h" -#include "pal_plain.h" +#include "pal_noalloc.h" -#include #ifdef OPEN_ENCLAVE extern "C" void* oe_memset_s(void* p, size_t p_size, int c, size_t size); extern "C" int oe_random(void* data, size_t size); @@ -12,55 +9,29 @@ extern "C" [[noreturn]] void oe_abort(); namespace snmalloc { - class PALOpenEnclave + class OpenEnclaveErrorHandler { - /// Base of OE heap - static inline void* heap_base = nullptr; - - /// Size of OE heap - static inline size_t heap_size; - - // This is infrequently used code, a spin lock simplifies the code - // considerably, and should never be on the fast path. - static inline std::atomic_flag spin_lock; - public: - /** - * This will be called by oe_allocator_init to set up enclave heap bounds. - */ - static void setup_initial_range(void* base, void* end) - { - heap_size = pointer_diff(base, end); - heap_base = base; - } - - /** - * Bitmap of PalFeatures flags indicating the optional features that this - * PAL supports. - */ - static constexpr uint64_t pal_features = Entropy; - - static constexpr size_t page_size = Aal::smallest_page_size; + static void print_stack_trace() {} [[noreturn]] static void error(const char* const str) { UNUSED(str); oe_abort(); } + }; - static std::pair - reserve_at_least(size_t request_size) noexcept - { - // First call returns the entire address space - // subsequent calls return {nullptr, 0} - FlagLock lock(spin_lock); - if (request_size > heap_size) - return {nullptr, 0}; + using OpenEnclaveBasePAL = PALNoAlloc; - auto result = std::make_pair(heap_base, heap_size); - heap_size = 0; - return result; - } + class PALOpenEnclave : public OpenEnclaveBasePAL + { + public: + /** + * Bitmap of PalFeatures flags indicating the optional features that this + * PAL supports. + */ + static constexpr uint64_t pal_features = + OpenEnclaveBasePAL::pal_features | Entropy; template static void zero(void* p, size_t size) noexcept diff --git a/src/test/func/fixed_region/fixed_region.cc b/src/test/func/fixed_region/fixed_region.cc index 99cf2aa6a..702aaee54 100644 --- a/src/test/func/fixed_region/fixed_region.cc +++ b/src/test/func/fixed_region/fixed_region.cc @@ -11,7 +11,8 @@ using namespace snmalloc; -using FixedAlloc = LocalAllocator; +using CustomGlobals = FixedGlobals>; +using FixedAlloc = LocalAllocator; int main() { @@ -30,8 +31,8 @@ int main() std::cout << "Allocated region " << oe_base.unsafe_ptr() << " - " << pointer_offset(oe_base, size).unsafe_ptr() << std::endl; - FixedGlobals fixed_handle; - FixedGlobals::init(oe_base, size); + CustomGlobals fixed_handle; + CustomGlobals::init(oe_base, size); FixedAlloc a(fixed_handle); size_t object_size = 128; diff --git a/src/test/func/two_alloc_types/alloc1.cc b/src/test/func/two_alloc_types/alloc1.cc index 08efc3898..2c15db55a 100644 --- a/src/test/func/two_alloc_types/alloc1.cc +++ b/src/test/func/two_alloc_types/alloc1.cc @@ -10,7 +10,8 @@ #define SNMALLOC_PROVIDE_OWN_CONFIG namespace snmalloc { - using Alloc = LocalAllocator; + using CustomGlobals = FixedGlobals>; + using Alloc = LocalAllocator; } #define SNMALLOC_NAME_MANGLE(a) enclave_##a @@ -18,7 +19,7 @@ namespace snmalloc extern "C" void oe_allocator_init(void* base, void* end) { - snmalloc::FixedGlobals fixed_handle; + snmalloc::CustomGlobals fixed_handle; fixed_handle.init( CapPtr(base), address_cast(end) - address_cast(base)); } From de8ef3efc7baa4b803d02a118a00278b7cf712cb Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 15 Jul 2021 18:31:28 +0100 Subject: [PATCH 005/302] Cleaner implementation of signed pointers. (#347) * Cleaner implementation of signed pointers. This encodes a back pointer in each node. The back pointer is stored in an encoded form so that it is hard to corrupt and trick the allocator into following incorrect pointers. This changes the encoding from previously being a Feistel network on the next pointer that was using the prev as part of the key, to now effectively using a doubly linked queue, where the back pointers are scrambled, so it is hard to forge them. This has the positive effects of - Not needing to store previous while building the list, as the append nows, curr and next at the point of writing into next, and does not need an additional previous. - The encoding is not affecting the actual next value, so more instructions can be executed in parallel by the CPU. Future extensions, store a changing key in the FreeListBuilder so it becomes harder to try to forge the previous token. This approach can also be applied to the remote list, and will in a subsequent PR. This enables the idea to be tested. * Remove unused header. * Apply suggestions from code review Co-authored-by: Nathaniel Wesley Filardo Co-authored-by: Nathaniel Wesley Filardo --- src/mem/corealloc.h | 2 - src/mem/freelist.h | 382 +++++++++++++++----------------------------- 2 files changed, 132 insertions(+), 252 deletions(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 6bf140f19..e307fb1a6 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -9,8 +9,6 @@ #include "sizeclasstable.h" #include "slaballocator.h" -#include - namespace snmalloc { template diff --git a/src/mem/freelist.h b/src/mem/freelist.h index 37809b418..48137915e 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -1,7 +1,36 @@ #pragma once /** * This file encapsulates the in disused object free lists - * that are used per slab of small objects. + * that are used per slab of small objects. The implementation + * can be configured to introduce randomness to the reallocation, + * and also provide signing to detect free list corruption. + * + * # Corruption + * + * The corruption detection works as follows + * + * FreeObject + * ----------------------------- + * | next | prev_encoded | ... | + * ----------------------------- + * A free object contains a pointer to next object in the free list, and + * a prev pointer, but the prev pointer is really a signature with the + * following property + * + * If n = c->next && n != 0, then n->prev_encoded = f(c,n). + * + * If f just returns the first parameter, then this degenerates to a doubly + * linked list. (Note that doing the degenerate case can be useful for + * debugging snmalloc bugs.) By making it a function of both pointers, it + * makes it harder for an adversary to mutate prev_encoded to a valid value. + * + * This provides protection against the free-list being corrupted by memory + * safety issues. + * + * # Randomness + * + * The randomness is introduced by building two free lists simulatenously, + * and randomly deciding which list to add an element to. */ #include "../ds/address.h" @@ -12,100 +41,43 @@ namespace snmalloc { - static constexpr std::size_t PRESERVE_BOTTOM_BITS = 30; - - static inline bool different_slab(address_t p1, address_t p2) - { - return (p1 ^ p2) >= bits::one_at_bit(PRESERVE_BOTTOM_BITS); - } - - template - static inline bool different_slab(address_t p1, CapPtr p2) - { - return different_slab(p1, address_cast(p2)); - } + struct Remote; - template - static inline bool - different_slab(CapPtr p1, CapPtr p2) + /** + * This function is used to sign back pointers in the free list. + * + * TODO - Needs review. Should we shift low bits out as they help + * guess the key. + * + * TODO - We now have space in the FreeListBuilder for a fresh key for each + * list. + */ + inline static uintptr_t + signed_prev(address_t curr, address_t next, LocalEntropy& entropy) { - return different_slab(address_cast(p1), address_cast(p2)); + auto c = curr; + auto n = next; + auto k = entropy.get_constant_key(); + return (c + k) * (n - k); } - class FreeObject; - - class EncodeFreeObjectReference - { - CapPtr reference; - - /** - * On architectures which use IntegerPointers, we can obfuscate our free - * lists and use this to drive some probabilistic checks for integrity. - * - * There are two definitions of encode() below, which use std::enable_if_t - * to gate on do_encode. - */ -#ifndef CHECK_CLIENT - static constexpr bool do_encode = false; -#else - static constexpr bool do_encode = aal_supports; -#endif - - public: -#ifdef CHECK_CLIENT - template - static std::enable_if_t> encode( - uint32_t local_key, CapPtr next_object, LocalEntropy& entropy) - { - // Simple involutional encoding. The bottom half of each word is - // multiplied by a function of both global and local keys (the latter, - // in practice, being the address of the previous list entry) and the - // resulting word's top part is XORed into the pointer value before it - // is stored. - auto next = address_cast(next_object); - constexpr address_t MASK = bits::one_at_bit(PRESERVE_BOTTOM_BITS) - 1; - // Mix in local_key - address_t p1 = local_key + entropy.get_constant_key(); - address_t p2 = (next & MASK) - entropy.get_constant_key(); - next ^= (p1 * p2) & ~MASK; - return CapPtr(reinterpret_cast(next)); - } -#endif - - template - static std::enable_if_t> encode( - uint32_t local_key, CapPtr next_object, LocalEntropy& entropy) - { - UNUSED(local_key); - UNUSED(entropy); - return next_object; - } - - void store( - CapPtr value, - uint32_t local_key, - LocalEntropy& entropy) - { - reference = encode(local_key, value, entropy); - } - - CapPtr read(uint32_t local_key, LocalEntropy& entropy) - { - return encode(local_key, reference, entropy); - } - }; - - struct Remote; /** * Free objects within each slab point directly to the next. - * The next_object pointer can be encoded to detect - * corruption caused by writes in a UAF or a double free. + * There is an optional second field that is effectively a + * back pointer in a doubly linked list, however, it is encoded + * to prevent corruption. */ class FreeObject { - public: - EncodeFreeObjectReference next_object; + CapPtr next_object; +#ifdef CHECK_CLIENT + // Encoded representation of a back pointer. + // Hard to fake, and provides consistency on + // the next pointers. + address_t prev_encoded; +#endif + public: static CapPtr make(CapPtr p) { return p.template as_static(); @@ -121,11 +93,45 @@ namespace snmalloc } /** - * Read the next pointer handling any required decoding of the pointer + * Assign next_object and update its prev_encoded if CHECK_CLIENT. + * + * Static so that it can be used on reference to a FreeObject. + * + * Returns a pointer to the next_object field of the next parameter as an + * optimization for repeated snoc operations (in which + * next->next_object is nullptr). */ - CapPtr read_next(uint32_t key, LocalEntropy& entropy) + static CapPtr* store( + CapPtr* curr, + CapPtr next, + LocalEntropy& entropy) { - return next_object.read(key, entropy); + *curr = next; +#ifdef CHECK_CLIENT + next->prev_encoded = + signed_prev(address_cast(curr), address_cast(next), entropy); +#else + UNUSED(entropy); +#endif + return &(next->next_object); + } + + /** + * Check the signature of this FreeObject + */ + void check_prev(address_t signed_prev) + { + UNUSED(signed_prev); + check_client( + signed_prev == prev_encoded, "Heap corruption - free list corrupted!"); + } + + /** + * Read the next pointer + */ + CapPtr read_next() + { + return next_object; } }; @@ -136,30 +142,19 @@ namespace snmalloc */ class FreeListIter { - CapPtr curr{1}; + CapPtr curr{nullptr}; #ifdef CHECK_CLIENT address_t prev{0}; #endif - uint32_t get_prev() - { -#ifdef CHECK_CLIENT - return prev & 0xffff'ffff; -#else - return 0; -#endif - } - public: constexpr FreeListIter( CapPtr head, address_t prev_value) : curr(head) + { #ifdef CHECK_CLIENT - , - prev(prev_value) + prev = prev_value; #endif - { - // SNMALLOC_ASSERT(head != nullptr); UNUSED(prev_value); } @@ -170,7 +165,7 @@ namespace snmalloc */ bool empty() { - return (address_cast(curr) & 1) == 1; + return curr == nullptr; } /** @@ -187,16 +182,17 @@ namespace snmalloc CapPtr take(LocalEntropy& entropy) { auto c = curr; - auto next = curr->read_next(get_prev(), entropy); + auto next = curr->read_next(); + Aal::prefetch(next.unsafe_ptr()); + curr = next; #ifdef CHECK_CLIENT - check_client( - !different_slab(curr, next), "Heap corruption - free list corrupted!"); - prev = address_cast(curr); + c->check_prev(prev); + prev = signed_prev(address_cast(c), address_cast(next), entropy); +#else + UNUSED(entropy); #endif - curr = next; - Aal::prefetch(next.unsafe_ptr()); return c; } }; @@ -224,46 +220,21 @@ namespace snmalloc * If RANDOM is set to false, then the code does not perform any * randomisation. */ - template + template class FreeListBuilder { static constexpr size_t LENGTH = RANDOM ? 2 : 1; // Pointer to the first element. - EncodeFreeObjectReference head[LENGTH]; + CapPtr head[LENGTH]; // Pointer to the reference to the last element. // In the empty case end[i] == &head[i] // This enables branch free enqueuing. - EncodeFreeObjectReference* end[LENGTH]; -#ifdef CHECK_CLIENT - // The bottom 32 bits of the previous pointer - uint32_t prev[LENGTH]; -#endif + CapPtr* end[LENGTH]; + public: S s; - uint32_t get_prev(uint32_t index) - { -#ifdef CHECK_CLIENT - return prev[index]; -#else - UNUSED(index); - return 0; -#endif - } - - uint32_t get_curr(uint32_t index) - { -#ifdef CHECK_CLIENT - return address_cast(end[index]) & 0xffff'ffff; -#else - UNUSED(index); - return 0; -#endif - } - - static constexpr uint32_t HEAD_KEY = 1; - public: constexpr FreeListBuilder() { @@ -283,63 +254,18 @@ namespace snmalloc return true; } - bool debug_different_slab(CapPtr n) - { - for (size_t i = 0; i < LENGTH; i++) - { - if (!different_slab(address_cast(end[i]), n)) - return false; - } - return true; - } - /** * Adds an element to the builder */ void add(CapPtr n, LocalEntropy& entropy) { - SNMALLOC_ASSERT(!debug_different_slab(n) || empty()); - uint32_t index; if constexpr (RANDOM) index = entropy.next_bit(); else index = 0; - end[index]->store(n, get_prev(index), entropy); -#ifdef CHECK_CLIENT - prev[index] = get_curr(index); -#endif - end[index] = &(n->next_object); - } - - /** - * Calculates the length of the queue. - * This is O(n) as it walks the queue. - * If this is needed in a non-debug setting then - * we should look at redesigning the queue. - */ - size_t debug_length(LocalEntropy& entropy) - { - size_t count = 0; - for (size_t i = 0; i < LENGTH; i++) - { - uint32_t local_prev = HEAD_KEY; - EncodeFreeObjectReference* iter = &head[i]; - CapPtr prev_obj = iter->read(local_prev, entropy); - UNUSED(prev_obj); - uint32_t local_curr = address_cast(&head[i]) & 0xffff'ffff; - while (end[i] != iter) - { - CapPtr next = iter->read(local_prev, entropy); - check_client(!different_slab(next, prev_obj), "Heap corruption"); - local_prev = local_curr; - local_curr = address_cast(next) & 0xffff'ffff; - count++; - iter = &next->next_object; - } - } - return count; + end[index] = FreeObject::store(end[index], n, entropy); } /** @@ -351,29 +277,21 @@ namespace snmalloc SNMALLOC_FAST_PATH void terminate_list(uint32_t index, LocalEntropy& entropy) { - auto term = CapPtr( - reinterpret_cast(end[index]) | 1); - end[index]->store(term, get_prev(index), entropy); + UNUSED(entropy); + *end[index] = nullptr; + } + + address_t get_fake_signed_prev(uint32_t index, LocalEntropy& entropy) + { + return signed_prev( + address_cast(&head[index]), address_cast(head[index]), entropy); } /** - * Adds a terminator at the end of a free list, - * but does not close the builder. Thus new elements - * can still be added. It returns a new iterator to the - * list. - * - * This also collapses the two queues into one, so that it can - * be iterated easily. - * - * This is used to iterate an list that is being constructed. - * - * It is used with preserve_queue enabled to check - * invariants in Debug builds. - * - * It is used with preserve_queue disabled by close. + * Close a free list, and set the iterator parameter + * to iterate it. */ - SNMALLOC_FAST_PATH void terminate( - FreeListIter& fl, LocalEntropy& entropy, bool preserve_queue = true) + SNMALLOC_FAST_PATH void close(FreeListIter& fl, LocalEntropy& entropy) { if constexpr (RANDOM) { @@ -383,62 +301,29 @@ namespace snmalloc // If second list is non-empty, perform append. if (end[1] != &head[1]) { + // The start token has been corrupted. + // TOCTTOU issue, but small window here. + head[1]->check_prev(get_fake_signed_prev(1, entropy)); + terminate_list(1, entropy); // Append 1 to 0 - auto mid = head[1].read(HEAD_KEY, entropy); - end[0]->store(mid, get_prev(0), entropy); - // Re-code first link in second list (if there is one). - // The first link in the second list will be encoded with initial key - // of the head, But that needs to be changed to the curr of the first - // list. - if (mid != nullptr) - { - auto mid_next = - mid->read_next(address_cast(&head[1]) & 0xffff'ffff, entropy); - mid->next_object.store(mid_next, get_curr(0), entropy); - } - - auto h = head[0].read(HEAD_KEY, entropy); - - // If we need to continue adding to the builder - // Set up the second list as empty, - // and extend the first list to cover all of the second. - if (preserve_queue && h != nullptr) - { -#ifdef CHECK_CLIENT - prev[0] = prev[1]; -#endif - end[0] = end[1]; -#ifdef CHECK_CLIENT - prev[1] = HEAD_KEY; -#endif - end[1] = &(head[1]); - } + FreeObject::store(end[0], head[1], entropy); SNMALLOC_ASSERT(end[1] != &head[0]); SNMALLOC_ASSERT(end[0] != &head[1]); - - fl = {h, address_cast(&head[0])}; - return; + } + else + { + terminate_list(0, entropy); } } else { - UNUSED(preserve_queue); + terminate_list(0, entropy); } - terminate_list(0, entropy); - fl = {head[0].read(HEAD_KEY, entropy), address_cast(&head[0])}; - } - - /** - * Close a free list, and set the iterator parameter - * to iterate it. - */ - SNMALLOC_FAST_PATH void close(FreeListIter& dst, LocalEntropy& entropy) - { - terminate(dst, entropy, false); + fl = {head[0], get_fake_signed_prev(0, entropy)}; init(); } @@ -450,9 +335,6 @@ namespace snmalloc for (size_t i = 0; i < LENGTH; i++) { end[i] = &head[i]; -#ifdef CHECK_CLIENT - prev[i] = HEAD_KEY; -#endif } } }; From c58b0a5f4da20a2231c622b4c67d52ccfdf32772 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 14 Jul 2021 12:03:07 +0100 Subject: [PATCH 006/302] Removes a register copy from x86 codegen. --- src/mem/entropy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mem/entropy.h b/src/mem/entropy.h index 8654532e6..80bd6f420 100644 --- a/src/mem/entropy.h +++ b/src/mem/entropy.h @@ -56,7 +56,7 @@ namespace snmalloc { uint64_t bottom_bit = bit_source & 1; bit_source = (bottom_bit << 63) | (bit_source >> 1); - return bottom_bit & 1; + return bit_source & 1; } /** From b501da69db887a01a9a71d4290a54e921ca363ce Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 14 Jul 2021 19:49:21 +0100 Subject: [PATCH 007/302] Implements protection on remote messages queues This extends the freelist protection to the remote message queues. They effectively perform doubly linked list entries for the message queue with the enqueue operation first linking in the previous pointer, and then then atomically setting the next. This ensures that the visible states always satisfy the invariant that the forward and backward pointers are correct for any visisble object. There is a key_global that is used for all remote deallocations. The remote cache uses the same protection to build the temporary lists before forwarding to the next allocator. The mpscq is integrated into the remoteallocator as it is no longer a reusable datastructure, but a special purpose implementation. --- src/ds/mpscq.h | 91 -------------------- src/mem/commonconfig.h | 2 +- src/mem/corealloc.h | 39 +++++---- src/mem/freelist.h | 171 ++++++++++++++++++++++++++++---------- src/mem/globalconfig.h | 3 + src/mem/localalloc.h | 8 +- src/mem/localcache.h | 10 ++- src/mem/metaslab.h | 59 ++++++------- src/mem/remoteallocator.h | 118 +++++++++++++++++--------- src/mem/remotecache.h | 83 +++++------------- 10 files changed, 292 insertions(+), 292 deletions(-) delete mode 100644 src/ds/mpscq.h diff --git a/src/ds/mpscq.h b/src/ds/mpscq.h deleted file mode 100644 index c3e8fb993..000000000 --- a/src/ds/mpscq.h +++ /dev/null @@ -1,91 +0,0 @@ -#pragma once - -#include "bits.h" -#include "helpers.h" - -#include -namespace snmalloc -{ - template< - class T, - template typename Ptr = Pointer, - template typename AtomicPtr = AtomicPointer> - class MPSCQ - { - private: - static_assert( - std::is_same>::value, - "T->next must be an AtomicPtr"); - - AtomicPtr back{nullptr}; - Ptr front{nullptr}; - - public: - constexpr MPSCQ() = default; - - void invariant() - { - SNMALLOC_ASSERT(back != nullptr); - SNMALLOC_ASSERT(front != nullptr); - } - - void init(Ptr stub) - { - stub->next.store(nullptr, std::memory_order_relaxed); - front = stub; - back.store(stub, std::memory_order_relaxed); - invariant(); - } - - Ptr destroy() - { - Ptr fnt = front; - back.store(nullptr, std::memory_order_relaxed); - front = nullptr; - return fnt; - } - - inline bool is_empty() - { - Ptr bk = back.load(std::memory_order_relaxed); - - return bk == front; - } - - void enqueue(Ptr first, Ptr last) - { - // Pushes a list of messages to the queue. Each message from first to - // last should be linked together through their next pointers. - invariant(); - last->next.store(nullptr, std::memory_order_relaxed); - std::atomic_thread_fence(std::memory_order_release); - Ptr prev = back.exchange(last, std::memory_order_relaxed); - prev->next.store(first, std::memory_order_relaxed); - } - - Ptr peek() - { - return front; - } - - std::pair, bool> dequeue() - { - // Returns the front message, or null if not possible to return a message. - invariant(); - Ptr first = front; - Ptr next = first->next.load(std::memory_order_relaxed); - - Aal::prefetch(&(next->next)); - if (next != nullptr) - { - front = next; - SNMALLOC_ASSERT(front != nullptr); - std::atomic_thread_fence(std::memory_order_acquire); - invariant(); - return {first, true}; - } - - return {nullptr, false}; - } - }; -} // namespace snmalloc diff --git a/src/mem/commonconfig.h b/src/mem/commonconfig.h index adfdb3395..9c24371c8 100644 --- a/src/mem/commonconfig.h +++ b/src/mem/commonconfig.h @@ -1,7 +1,7 @@ #pragma once #include "../ds/defines.h" -#include "remoteallocator.h" +#include "remotecache.h" namespace snmalloc { diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index e307fb1a6..4df9ca8d2 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -95,7 +95,7 @@ namespace snmalloc */ auto& message_queue() { - return public_state()->message_queue; + return *public_state(); } /** @@ -108,7 +108,7 @@ namespace snmalloc // Using an actual allocation removes a conditional from a critical path. auto dummy = CapPtr(small_alloc_one(sizeof(MIN_ALLOC_SIZE))) - .template as_static(); + .template as_static(); if (dummy == nullptr) { error("Critical error: Out-of-memory during initialisation."); @@ -141,6 +141,8 @@ namespace snmalloc { auto slab_end = pointer_offset(bumpptr, slab_size + 1 - rsize); + FreeListKey key(entropy.get_constant_key()); + FreeListBuilder b; SNMALLOC_ASSERT(b.empty()); @@ -187,14 +189,14 @@ namespace snmalloc auto curr_ptr = start_ptr; do { - b.add(FreeObject::make(curr_ptr.as_void()), entropy); + b.add(FreeObject::make(curr_ptr.as_void()), key); curr_ptr = curr_ptr->next; } while (curr_ptr != start_ptr); #else auto p = bumpptr; do { - b.add(Aal::capptr_bound(p, rsize), entropy); + b.add(Aal::capptr_bound(p, rsize), key); p = pointer_offset(p, rsize); } while (p < slab_end); #endif @@ -202,14 +204,15 @@ namespace snmalloc bumpptr = slab_end; SNMALLOC_ASSERT(!b.empty()); - b.close(fast_free_list, entropy); + b.close(fast_free_list, key); } ChunkRecord* clear_slab(Metaslab* meta, sizeclass_t sizeclass) { + FreeListKey key(entropy.get_constant_key()); FreeListIter fl; - meta->free_queue.close(fl, entropy); - void* p = finish_alloc_no_zero(fl.take(entropy), sizeclass); + meta->free_queue.close(fl, key); + void* p = finish_alloc_no_zero(fl.take(key), sizeclass); #ifdef CHECK_CLIENT // Check free list is well-formed on platforms with @@ -217,7 +220,7 @@ namespace snmalloc size_t count = 1; // Already taken one above. while (!fl.empty()) { - fl.take(entropy); + fl.take(key); count++; } // Check the list contains all the elements @@ -335,7 +338,7 @@ namespace snmalloc auto& entry = SharedStateHandle::Backend::get_meta_data( handle.get_backend_state(), snmalloc::address_cast(p)); - auto r = message_queue().dequeue(); + auto r = message_queue().dequeue(key_global); if (unlikely(!r.second)) break; @@ -360,7 +363,7 @@ namespace snmalloc * need_post will be set to true, if capacity is exceeded. */ void handle_dealloc_remote( - const MetaEntry& entry, CapPtr p, bool& need_post) + const MetaEntry& entry, CapPtr p, bool& need_post) { // TODO this needs to not double count stats // TODO this needs to not double revoke if using MTE @@ -381,7 +384,7 @@ namespace snmalloc need_post = true; attached_cache->remote_dealloc_cache .template dealloc( - entry.get_remote()->trunc_id(), p.as_void()); + entry.get_remote()->trunc_id(), p.as_void(), key_global); } } @@ -431,7 +434,7 @@ namespace snmalloc // stats().remote_post(); // TODO queue not in line! bool sent_something = attached_cache->remote_dealloc_cache.post( - handle, public_state()->trunc_id()); + handle, public_state()->trunc_id(), key_global); return sent_something; } @@ -472,8 +475,10 @@ namespace snmalloc auto cp = CapPtr(reinterpret_cast(p)); + FreeListKey key(entropy.get_constant_key()); + // Update the head and the next pointer in the free list. - meta->free_queue.add(cp, entropy); + meta->free_queue.add(cp, key, entropy); return likely(!meta->return_object()); } @@ -533,8 +538,10 @@ namespace snmalloc // Set meta slab to empty. meta->initialise(sizeclass); + FreeListKey key(entropy.get_constant_key()); + // take an allocation from the free list - auto p = fast_free_list.take(entropy); + auto p = fast_free_list.take(key); return finish_alloc(p, sizeclass); } @@ -550,12 +557,12 @@ namespace snmalloc if (destroy_queue) { - CapPtr p = message_queue().destroy(); + auto p = message_queue().destroy(); while (p != nullptr) { bool need_post = true; // Always going to post, so ignore. - auto n = p->non_atomic_next; + auto n = p->atomic_read_next(key_global); auto& entry = SharedStateHandle::Backend::get_meta_data( handle.get_backend_state(), snmalloc::address_cast(p)); handle_dealloc_remote(entry, p, need_post); diff --git a/src/mem/freelist.h b/src/mem/freelist.h index 48137915e..50dbc0446 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -41,7 +41,18 @@ namespace snmalloc { - struct Remote; + struct FreeListKey + { + address_t key; + + FreeListKey(uint64_t key_) + { + if constexpr (bits::BITS == 64) + key = static_cast(key_); + else + key = key_ & 0xffff'ffff; + } + }; /** * This function is used to sign back pointers in the free list. @@ -53,11 +64,11 @@ namespace snmalloc * list. */ inline static uintptr_t - signed_prev(address_t curr, address_t next, LocalEntropy& entropy) + signed_prev(address_t curr, address_t next, FreeListKey& key) { auto c = curr; auto n = next; - auto k = entropy.get_constant_key(); + auto k = key.key; return (c + k) * (n - k); } @@ -66,10 +77,19 @@ namespace snmalloc * There is an optional second field that is effectively a * back pointer in a doubly linked list, however, it is encoded * to prevent corruption. + * + * TODO: Consider put prev_encoded at the end of the object, would + * require size to be threaded through, but would provide more OOB + * detection. */ class FreeObject { - CapPtr next_object; + union + { + CapPtr next_object; + // TODO: Should really use C++20 atomic_ref rather than a union. + AtomicCapPtr atomic_next_object; + }; #ifdef CHECK_CLIENT // Encoded representation of a back pointer. // Hard to fake, and provides consistency on @@ -83,39 +103,66 @@ namespace snmalloc return p.template as_static(); } - /** - * Construct a FreeObject for local slabs from a Remote message. - */ - static CapPtr make(CapPtr p) - { - // TODO: Zero the difference between a FreeObject and a Remote - return p.template as_reinterpret(); - } - /** * Assign next_object and update its prev_encoded if CHECK_CLIENT. - * * Static so that it can be used on reference to a FreeObject. * * Returns a pointer to the next_object field of the next parameter as an * optimization for repeated snoc operations (in which * next->next_object is nullptr). */ - static CapPtr* store( + static CapPtr* store_next( CapPtr* curr, CapPtr next, - LocalEntropy& entropy) + FreeListKey& key) { - *curr = next; #ifdef CHECK_CLIENT next->prev_encoded = - signed_prev(address_cast(curr), address_cast(next), entropy); + signed_prev(address_cast(curr), address_cast(next), key); #else - UNUSED(entropy); + UNUSED(key); #endif + *curr = next; return &(next->next_object); } + /** + * Assign next_object and update its prev_encoded if CHECK_CLIENT + * + * Uses the atomic view of next, so can be used in the message queues. + */ + void atomic_store_next(CapPtr next, FreeListKey& key) + { +#ifdef CHECK_CLIENT + next->prev_encoded = + signed_prev(address_cast(this), address_cast(next), key); +#else + UNUSED(key); +#endif + // Signature needs to be visible before item is linked in + // so requires release semantics. + atomic_next_object.store(next, std::memory_order_release); + } + + void atomic_store_null() + { + atomic_next_object.store(nullptr, std::memory_order_relaxed); + } + + CapPtr atomic_read_next(FreeListKey& key) + { + auto n = atomic_next_object.load(std::memory_order_acquire); +#ifdef CHECK_CLIENT + if (n != nullptr) + { + n->check_prev(signed_prev(address_cast(this), address_cast(n), key)); + } +#else + UNUSED(key); +#endif + return n; + } + /** * Check the signature of this FreeObject */ @@ -135,6 +182,10 @@ namespace snmalloc } }; + static_assert( + sizeof(FreeObject) <= MIN_ALLOC_SIZE, + "Needs to be able to fit in smallest allocation."); + /** * Used to iterate a free list in object space. * @@ -179,7 +230,7 @@ namespace snmalloc /** * Moves the iterator on, and returns the current value. */ - CapPtr take(LocalEntropy& entropy) + CapPtr take(FreeListKey& key) { auto c = curr; auto next = curr->read_next(); @@ -188,9 +239,9 @@ namespace snmalloc curr = next; #ifdef CHECK_CLIENT c->check_prev(prev); - prev = signed_prev(address_cast(c), address_cast(next), entropy); + prev = signed_prev(address_cast(c), address_cast(next), key); #else - UNUSED(entropy); + UNUSED(key); #endif return c; @@ -220,25 +271,23 @@ namespace snmalloc * If RANDOM is set to false, then the code does not perform any * randomisation. */ - template + template class FreeListBuilder { static constexpr size_t LENGTH = RANDOM ? 2 : 1; // Pointer to the first element. - CapPtr head[LENGTH]; + std::array, LENGTH> head; // Pointer to the reference to the last element. // In the empty case end[i] == &head[i] // This enables branch free enqueuing. - CapPtr* end[LENGTH]; - - public: - S s; + std::array*, LENGTH> end{nullptr}; public: constexpr FreeListBuilder() { - init(); + if (INIT) + init(); } /** @@ -257,7 +306,8 @@ namespace snmalloc /** * Adds an element to the builder */ - void add(CapPtr n, LocalEntropy& entropy) + void + add(CapPtr n, FreeListKey& key, LocalEntropy& entropy) { uint32_t index; if constexpr (RANDOM) @@ -265,33 +315,45 @@ namespace snmalloc else index = 0; - end[index] = FreeObject::store(end[index], n, entropy); + end[index] = FreeObject::store_next(end[index], n, key); + } + + /** + * Adds an element to the builder, if we are guaranteed that + * RANDOM is false. This is useful in certain construction + * cases that do not need to introduce randomness, such as + * during the initialation construction of a free list, which + * uses its own algorithm, or during building remote deallocation + * lists, which will be randomised at the other end. + */ + template + std::enable_if_t + add(CapPtr n, FreeListKey& key) + { + static_assert(RANDOM_ == RANDOM, "Don't set template parameter"); + end[0] = FreeObject::store_next(end[0], n, key); } /** * Makes a terminator to a free list. - * - * Termination uses the bottom bit, this allows the next pointer - * to always be to the same slab. */ - SNMALLOC_FAST_PATH void - terminate_list(uint32_t index, LocalEntropy& entropy) + SNMALLOC_FAST_PATH void terminate_list(uint32_t index, FreeListKey& key) { - UNUSED(entropy); + UNUSED(key); *end[index] = nullptr; } - address_t get_fake_signed_prev(uint32_t index, LocalEntropy& entropy) + address_t get_fake_signed_prev(uint32_t index, FreeListKey& key) { return signed_prev( - address_cast(&head[index]), address_cast(head[index]), entropy); + address_cast(&head[index]), address_cast(head[index]), key); } /** * Close a free list, and set the iterator parameter * to iterate it. */ - SNMALLOC_FAST_PATH void close(FreeListIter& fl, LocalEntropy& entropy) + SNMALLOC_FAST_PATH void close(FreeListIter& fl, FreeListKey& key) { if constexpr (RANDOM) { @@ -303,27 +365,27 @@ namespace snmalloc { // The start token has been corrupted. // TOCTTOU issue, but small window here. - head[1]->check_prev(get_fake_signed_prev(1, entropy)); + head[1]->check_prev(get_fake_signed_prev(1, key)); - terminate_list(1, entropy); + terminate_list(1, key); // Append 1 to 0 - FreeObject::store(end[0], head[1], entropy); + FreeObject::store_next(end[0], head[1], key); SNMALLOC_ASSERT(end[1] != &head[0]); SNMALLOC_ASSERT(end[0] != &head[1]); } else { - terminate_list(0, entropy); + terminate_list(0, key); } } else { - terminate_list(0, entropy); + terminate_list(0, key); } - fl = {head[0], get_fake_signed_prev(0, entropy)}; + fl = {head[0], get_fake_signed_prev(0, key)}; init(); } @@ -337,5 +399,22 @@ namespace snmalloc end[i] = &head[i]; } } + + std::pair, CapPtr> + extract_segment() + { + SNMALLOC_ASSERT(!empty()); + SNMALLOC_ASSERT(!RANDOM); // TODO: Turn this into a static failure. + + auto first = head[0]; + // end[0] is pointing to the first field in the object, + // this is doing a CONTAINING_RECORD like cast to get back + // to the actual object. This isn't true if the builder is + // empty, but you are not allowed to call this in the empty case. + auto last = + CapPtr(reinterpret_cast(end[0])); + init(); + return {first, last}; + } }; } // namespace snmalloc diff --git a/src/mem/globalconfig.h b/src/mem/globalconfig.h index d87e75a2e..981f09a6a 100644 --- a/src/mem/globalconfig.h +++ b/src/mem/globalconfig.h @@ -73,6 +73,9 @@ namespace snmalloc if (initialised) return; + // Initialise key for remote deallocation lists + key_global = FreeListKey(get_entropy64()); + // Need to initialise pagemap. backend_state.init(); diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 9a674916f..c42685eb9 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -228,7 +228,7 @@ namespace snmalloc MetaEntry entry = SharedStateHandle::Backend::get_meta_data( handle.get_backend_state(), address_cast(p)); local_cache.remote_dealloc_cache.template dealloc( - entry.get_remote()->trunc_id(), CapPtr(p)); + entry.get_remote()->trunc_id(), CapPtr(p), key_global); post_remote_cache(); return; } @@ -249,7 +249,7 @@ namespace snmalloc */ auto& message_queue() { - return local_cache.remote_allocator->message_queue; + return local_cache.remote_allocator; } public: @@ -382,7 +382,9 @@ namespace snmalloc if (local_cache.remote_dealloc_cache.reserve_space(entry)) { local_cache.remote_dealloc_cache.template dealloc( - entry.get_remote()->trunc_id(), CapPtr(p)); + entry.get_remote()->trunc_id(), + CapPtr(p), + key_global); #ifdef SNMALLOC_TRACING std::cout << "Remote dealloc fast" << p << " size " << alloc_size(p) << std::endl; diff --git a/src/mem/localcache.h b/src/mem/localcache.h index ba364ad97..20709668e 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -76,6 +76,8 @@ namespace snmalloc typename SharedStateHandle> bool flush(DeallocFun dealloc, SharedStateHandle handle) { + FreeListKey key(entropy.get_constant_key()); + // Return all the free lists to the allocator. // Used during thread teardown for (size_t i = 0; i < NUM_SIZECLASSES; i++) @@ -84,25 +86,27 @@ namespace snmalloc // call. while (!small_fast_free_lists[i].empty()) { - auto p = small_fast_free_lists[i].take(entropy); + auto p = small_fast_free_lists[i].take(key); dealloc(finish_alloc_no_zero(p, i)); } } return remote_dealloc_cache.post( - handle, remote_allocator->trunc_id()); + handle, remote_allocator->trunc_id(), key_global); } template SNMALLOC_FAST_PATH void* alloc(size_t size, Slowpath slowpath) { + FreeListKey key(entropy.get_constant_key()); + sizeclass_t sizeclass = size_to_sizeclass(size); stats.alloc_request(size); stats.sizeclass_alloc(sizeclass); auto& fl = small_fast_free_lists[sizeclass]; if (likely(!fl.empty())) { - auto p = fl.take(entropy); + auto p = fl.take(key); return finish_alloc(p, sizeclass); } return slowpath(sizeclass, &fl); diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index f341064a3..9d774e2f9 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -18,14 +18,24 @@ namespace snmalloc sizeof(SlabLink) <= MIN_ALLOC_SIZE, "Need to be able to pack a SlabLink into any free small alloc"); - /** - * This struct is used inside FreeListBuilder to account for the - * alignment space that is wasted in sizeof. - * - * This is part of Metaslab abstraction. - */ - struct MetaslabEnd + // The Metaslab represent the status of a single slab. + // This can be either a short or a standard slab. + class alignas(CACHELINE_SIZE) Metaslab : public SlabLink { + public: + constexpr Metaslab() : SlabLink(true) {} + + /** + * Data-structure for building the free list for this slab. + * + * Spare 32bits are used for the fields in MetaslabEnd. + */ +#ifdef CHECK_CLIENT + FreeListBuilder free_queue; +#else + FreeListBuilder free_queue; +#endif + /** * The number of deallocation required until we hit a slow path. This * counts down in two different ways that are handled the same on the @@ -37,41 +47,22 @@ namespace snmalloc * to be detected, so that the statistics can be kept up to date, and * potentially return memory to the a global pool of slabs/chunks. */ - uint16_t needed = 0; + uint16_t needed_ = 0; /** * Flag that is used to indicate that the slab is currently not active. * I.e. it is not in a CoreAllocator cache for the appropriate sizeclass. */ - bool sleeping = false; - }; - - // The Metaslab represent the status of a single slab. - // This can be either a short or a standard slab. - class alignas(CACHELINE_SIZE) Metaslab : public SlabLink - { - public: - constexpr Metaslab() : SlabLink(true) {} - - /** - * Data-structure for building the free list for this slab. - * - * Spare 32bits are used for the fields in MetaslabEnd. - */ -#ifdef CHECK_CLIENT - FreeListBuilder free_queue; -#else - FreeListBuilder free_queue; -#endif + bool sleeping_ = false; uint16_t& needed() { - return free_queue.s.needed; + return needed_; } bool& sleeping() { - return free_queue.s.sleeping; + return sleeping_; } /** @@ -151,13 +142,17 @@ namespace snmalloc LocalEntropy& entropy, sizeclass_t sizeclass) { + FreeListKey key(entropy.get_constant_key()); + FreeListIter tmp_fl; - meta->free_queue.close(tmp_fl, entropy); - auto p = tmp_fl.take(entropy); + meta->free_queue.close(tmp_fl, key); + auto p = tmp_fl.take(key); fast_free_list = tmp_fl; #ifdef CHECK_CLIENT entropy.refresh_bits(); +#else + UNUSED(entropy); #endif // Treat stealing the free list as allocating it all. diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index c639e5c47..d3c1a738b 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -1,6 +1,5 @@ #pragma once -#include "../ds/mpscq.h" #include "../mem/allocconfig.h" #include "../mem/freelist.h" #include "../mem/metaslab.h" @@ -9,59 +8,102 @@ #include #include -#ifdef CHECK_CLIENT -# define SNMALLOC_DONT_CACHE_ALLOCATOR_PTR -#endif - namespace snmalloc { - /* - * A region of memory destined for a remote allocator's dealloc() via the - * message passing system. This structure is placed at the beginning of - * the allocation itself when it is queued for sending. + // Remotes need to be aligned enough that all the + // small size classes can fit in the bottom bits. + static constexpr size_t REMOTE_MIN_ALIGN = bits::min( + CACHELINE_SIZE, bits::next_pow2_const(NUM_SIZECLASSES + 1)); + + /** + * Global key for all remote lists. */ - struct Remote + inline static FreeListKey key_global(0xdeadbeef); + + struct alignas(REMOTE_MIN_ALIGN) RemoteAllocator { - using alloc_id_t = size_t; - union + using alloc_id_t = address_t; + + // Store the message queue on a separate cacheline. It is mutable data that + // is read by other threads. + alignas(CACHELINE_SIZE) AtomicCapPtr back{nullptr}; + // Store the two ends on different cache lines as access by different + // threads. + alignas(CACHELINE_SIZE) CapPtr front{nullptr}; + + constexpr RemoteAllocator() = default; + + void invariant() { - CapPtr non_atomic_next; - AtomicCapPtr next{nullptr}; - }; + SNMALLOC_ASSERT(back != nullptr); + SNMALLOC_ASSERT(front != nullptr); + } - constexpr Remote() : next(nullptr) {} + void init(CapPtr stub) + { + stub->atomic_store_null(); + front = stub; + back.store(stub, std::memory_order_relaxed); + invariant(); + } - /** Zero out a Remote tracking structure, return pointer to object base */ - template - SNMALLOC_FAST_PATH static CapPtr clear(CapPtr self) + CapPtr destroy() { - pal_zero(self, sizeof(Remote)); - return self.as_void(); + CapPtr fnt = front; + back.store(nullptr, std::memory_order_relaxed); + front = nullptr; + return fnt; } - }; - static_assert( - sizeof(Remote) <= MIN_ALLOC_SIZE, - "Needs to be able to fit in smallest allocation."); + inline bool is_empty() + { + CapPtr bk = back.load(std::memory_order_relaxed); - // Remotes need to be aligned enough that all the - // small size classes can fit in the bottom bits. - static constexpr size_t REMOTE_MIN_ALIGN = bits::min( - CACHELINE_SIZE, bits::next_pow2_const(NUM_SIZECLASSES + 1)); + return bk == front; + } - struct alignas(REMOTE_MIN_ALIGN) RemoteAllocator - { - using alloc_id_t = Remote::alloc_id_t; - // Store the message queue on a separate cacheline. It is mutable data that - // is read by other threads. + void enqueue( + CapPtr first, + CapPtr last, + FreeListKey& key) + { + // Pushes a list of messages to the queue. Each message from first to + // last should be linked together through their next pointers. + invariant(); + last->atomic_store_null(); + + // exchange needs to be a release, so nullptr in next is visible. + CapPtr prev = + back.exchange(last, std::memory_order_release); - MPSCQ message_queue; + prev->atomic_store_next(first, key); + } + + CapPtr peek() + { + return front; + } + + std::pair, bool> dequeue(FreeListKey& key) + { + // Returns the front message, or null if not possible to return a message. + invariant(); + CapPtr first = front; + CapPtr next = first->atomic_read_next(key); + + if (next != nullptr) + { + front = next; + invariant(); + return {first, true}; + } + + return {nullptr, false}; + } alloc_id_t trunc_id() { - return static_cast( - reinterpret_cast(&message_queue)) & - ~SIZECLASS_MASK; + return address_cast(this); } }; } // namespace snmalloc diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index a017b674a..af71c2205 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -1,6 +1,5 @@ #pragma once -#include "../ds/mpscq.h" #include "../mem/allocconfig.h" #include "../mem/freelist.h" #include "../mem/metaslab.h" @@ -17,40 +16,7 @@ namespace snmalloc */ struct RemoteDeallocCache { - /* - * A singly-linked list of Remote objects, supporting append and - * take-all operations. Intended only for the private use of this - * allocator; the Remote objects here will later be taken and pushed - * to the inter-thread message queues. - */ - struct RemoteList - { - /* - * A stub Remote object that will always be the head of this list; - * never taken for further processing. - */ - Remote head{}; - - /** - * Initially is null ptr, and needs to be non-null before anything runs on - * this. - */ - CapPtr last{nullptr}; - - void clear() - { - last = CapPtr(&head); - } - - bool empty() - { - return address_cast(last) == address_cast(&head); - } - - constexpr RemoteList() = default; - }; - - std::array list{}; + std::array, REMOTE_SLOTS> list; /** * The total amount of memory we are waiting for before we will dispatch @@ -98,19 +64,22 @@ namespace snmalloc } template - SNMALLOC_FAST_PATH void - dealloc(Remote::alloc_id_t target_id, CapPtr p) + SNMALLOC_FAST_PATH void dealloc( + RemoteAllocator::alloc_id_t target_id, + CapPtr p, + FreeListKey& key) { SNMALLOC_ASSERT(initialised); - auto r = p.template as_reinterpret(); + auto r = p.template as_reinterpret(); - RemoteList* l = &list[get_slot(target_id, 0)]; - l->last->non_atomic_next = r; - l->last = r; + list[get_slot(target_id, 0)].add(r, key); } template - bool post(SharedStateHandle handle, Remote::alloc_id_t id) + bool post( + SharedStateHandle handle, + RemoteAllocator::alloc_id_t id, + FreeListKey& key) { SNMALLOC_ASSERT(initialised); size_t post_round = 0; @@ -125,46 +94,37 @@ namespace snmalloc if (i == my_slot) continue; - RemoteList* l = &list[i]; - CapPtr first = l->head.non_atomic_next; - - if (!l->empty()) + if (!list[i].empty()) { + auto [first, last] = list[i].extract_segment(); MetaEntry entry = SharedStateHandle::Backend::get_meta_data( handle.get_backend_state(), address_cast(first)); - entry.get_remote()->message_queue.enqueue(first, l->last); - l->clear(); + entry.get_remote()->enqueue(first, last, key); sent_something = true; } } - RemoteList* resend = &list[my_slot]; - if (resend->empty()) + if (list[my_slot].empty()) break; // Entries could map back onto the "resend" list, // so take copy of the head, mark the last element, // and clear the original list. - CapPtr r = resend->head.non_atomic_next; - resend->last->non_atomic_next = nullptr; - resend->clear(); + FreeListIter resend; + list[my_slot].close(resend, key); post_round++; - while (r != nullptr) + while (!resend.empty()) { // Use the next N bits to spread out remote deallocs in our own // slot. + auto r = resend.take(key); MetaEntry entry = SharedStateHandle::Backend::get_meta_data( handle.get_backend_state(), address_cast(r)); auto i = entry.get_remote()->trunc_id(); - // TODO correct size for slot offset size_t slot = get_slot(i, post_round); - RemoteList* l = &list[slot]; - l->last->non_atomic_next = r; - l->last = r; - - r = r->non_atomic_next; + list[slot].add(r, key); } } @@ -190,8 +150,7 @@ namespace snmalloc #endif for (auto& l : list) { - SNMALLOC_ASSERT(l.last == nullptr || l.empty()); - l.clear(); + l.init(); } capacity = REMOTE_CACHE; } From 3e70963772d222aea6e78738f923b955ef02ddac Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Tue, 20 Jul 2021 12:04:21 +0100 Subject: [PATCH 008/302] ds/address: add some uintptr_t manipulation functions --- src/ds/address.h | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/src/ds/address.h b/src/ds/address.h index 0dde50245..d46da593b 100644 --- a/src/ds/address.h +++ b/src/ds/address.h @@ -16,12 +16,21 @@ namespace snmalloc */ using address_t = Aal::address_t; + /** + * Perform arithmetic on a uintptr_t. + */ + inline uintptr_t pointer_offset(uintptr_t base, size_t diff) + { + return base + diff; + } + /** * Perform pointer arithmetic and return the adjusted pointer. */ template inline U* pointer_offset(T* base, size_t diff) { + SNMALLOC_ASSERT(base != nullptr); /* Avoid UB */ return reinterpret_cast(reinterpret_cast(base) + diff); } @@ -38,6 +47,7 @@ namespace snmalloc template inline U* pointer_offset_signed(T* base, ptrdiff_t diff) { + SNMALLOC_ASSERT(base != nullptr); /* Avoid UB */ return reinterpret_cast(reinterpret_cast(base) + diff); } @@ -90,27 +100,37 @@ namespace snmalloc } /** - * Align a pointer down to a statically specified granularity, which must be a - * power of two. + * Align a uintptr_t down to a statically specified granularity, which must be + * a power of two. */ - template - inline T* pointer_align_down(void* p) + template + inline uintptr_t pointer_align_down(uintptr_t p) { static_assert(alignment > 0); static_assert(bits::is_pow2(alignment)); if constexpr (alignment == 1) - return static_cast(p); + return p; else { #if __has_builtin(__builtin_align_down) - return static_cast(__builtin_align_down(p, alignment)); + return __builtin_align_down(p, alignment); #else - return reinterpret_cast( - bits::align_down(reinterpret_cast(p), alignment)); + return bits::align_down(p, alignment); #endif } } + /** + * Align a pointer down to a statically specified granularity, which must be a + * power of two. + */ + template + inline T* pointer_align_down(void* p) + { + return reinterpret_cast( + pointer_align_down(reinterpret_cast(p))); + } + template inline CapPtr pointer_align_down(CapPtr p) { From 2ba0de76b3bde3888875188fbb76398f614e95aa Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Tue, 20 Jul 2021 12:13:56 +0100 Subject: [PATCH 009/302] mem/metaslab: use uintptr_t alignment functions It is UB to offset from `nullptr` (except perhaps with a 0 offset). Apparently clang is able to use this to reason, given `void* p`, that comparing `__builtin_align_down(p, x)` against `handle.fake_large_remote` (i.e., a `static inline constexpr` `nullptr`) must be the same as comparing `p` itself against `nullptr`. In `MetaEntry`'s constructor, converting the provided `RemoteAllocator*` to `uintptr_t` before offsetting avoids the UB. (While here, don't use `address_cast()`, as `address_t` will, on CHERI, be `ptraddr_t` and not `uintptr_t`.) Doing the alignment in `get_remote` at `uintptr_t` before casting to `RemoteAllocator*`, rather than converting and then aligning, prevents the reasoning above from eliminating the alignment. --- src/mem/metaslab.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 9d774e2f9..ae2b7eabd 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -180,8 +180,9 @@ namespace snmalloc MetaEntry(Metaslab* meta, RemoteAllocator* remote, sizeclass_t sizeclass) : meta(meta) { + /* remote might be nullptr; cast to uintptr_t before offsetting */ remote_and_sizeclass = - address_cast(pointer_offset(remote, sizeclass)); + pointer_offset(reinterpret_cast(remote), sizeclass); } [[nodiscard]] Metaslab* get_metaslab() const @@ -192,7 +193,7 @@ namespace snmalloc [[nodiscard]] RemoteAllocator* get_remote() const { return reinterpret_cast( - bits::align_down(remote_and_sizeclass, alignof(RemoteAllocator))); + pointer_align_down(remote_and_sizeclass)); } [[nodiscard]] sizeclass_t get_sizeclass() const From 8b1ffbc1668c81b9e19605ec052462002cf1bf13 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 16 Jul 2021 09:41:58 +0100 Subject: [PATCH 010/302] Expose reserve_at_least in all Pals --- docs/PORTING.md | 7 ++++--- src/pal/pal_bsd_aligned.h | 12 ------------ src/pal/pal_windows.h | 37 +++---------------------------------- 3 files changed, 7 insertions(+), 49 deletions(-) diff --git a/docs/PORTING.md b/docs/PORTING.md index 8238bf1cc..fdf1dec48 100644 --- a/docs/PORTING.md +++ b/docs/PORTING.md @@ -49,9 +49,10 @@ template static void* reserve_aligned(size_t size) noexcept; static std::pair reserve_at_least(size_t size) noexcept; ``` -Only one of these needs to be implemented, depending on whether the underlying -system can provide strongly aligned memory regions. -If the system guarantees only page alignment, implement the second. The Pal is +All platforms should provide `reserve_at_least` and can optionally provide +`reserve_aligned` if the underlying system can provide strongly aligned +memory regions. +If the system guarantees only page alignment, implement only the second. The Pal is free to overallocate based on the platform's desire and snmalloc will find suitably aligned blocks inside the region. `reserve_at_least` should not commit memory as snmalloc will commit the range of memory it requires of what diff --git a/src/pal/pal_bsd_aligned.h b/src/pal/pal_bsd_aligned.h index bac3e8284..1cdd23494 100644 --- a/src/pal/pal_bsd_aligned.h +++ b/src/pal/pal_bsd_aligned.h @@ -50,17 +50,5 @@ namespace snmalloc return p; } - - /** - * Explicitly deleted method for returning non-aligned memory. This causes - * incorrect use of `constexpr if` to fail on platforms with aligned - * allocation. Without this, this PAL and its subclasses exported both - * allocation functions and so callers would type-check if they called - * either in `constexpr if` branches and then fail on platforms such as - * Linux or Windows, which expose only unaligned or aligned allocations, - * respectively. - */ - static std::pair - reserve_at_least(size_t size) noexcept = delete; }; } // namespace snmalloc diff --git a/src/pal/pal_windows.h b/src/pal/pal_windows.h index 847a04ba6..c4f7219ac 100644 --- a/src/pal/pal_windows.h +++ b/src/pal/pal_windows.h @@ -152,38 +152,7 @@ namespace snmalloc ::memset(p, 0, size); } -# ifdef USE_SYSTEMATIC_TESTING - static size_t& systematic_bump_ptr() - { - static size_t bump_ptr = (size_t)0x4000'0000'0000; - return bump_ptr; - } - - static std::pair reserve_at_least(size_t size) noexcept - { - // Magic number for over-allocating chosen by the Pal - // These should be further refined based on experiments. - constexpr size_t min_size = - bits::is64() ? bits::one_at_bit(32) : bits::one_at_bit(28); - auto size_request = bits::max(size, min_size); - - DWORD flags = MEM_RESERVE; - - size_t retries = 1000; - void* p; - - do - { - p = VirtualAlloc( - (void*)systematic_bump_ptr(), size_request, flags, PAGE_READWRITE); - - systematic_bump_ptr() += size_request; - retries--; - } while (p == nullptr && retries > 0); - - return {p, size_request}; - } -# elif defined(PLATFORM_HAS_VIRTUALALLOC2) +# ifdef PLATFORM_HAS_VIRTUALALLOC2 template static void* reserve_aligned(size_t size) noexcept { @@ -214,7 +183,8 @@ namespace snmalloc } return ret; } -# else +# endif + static std::pair reserve_at_least(size_t size) noexcept { SNMALLOC_ASSERT(bits::is_pow2(size)); @@ -236,7 +206,6 @@ namespace snmalloc } error("Failed to allocate memory\n"); } -# endif /** * Source of Entropy From 686cb3321fc0a198e0628501ca341fcbcb4872bc Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 16 Jul 2021 09:51:29 +0100 Subject: [PATCH 011/302] Use SFINAE for varying API on pagemap --- src/backend/pagemap.h | 81 ++++++++++++++++++++----------------------- 1 file changed, 38 insertions(+), 43 deletions(-) diff --git a/src/backend/pagemap.h b/src/backend/pagemap.h index 843e8df6c..727e3b445 100644 --- a/src/backend/pagemap.h +++ b/src/backend/pagemap.h @@ -51,52 +51,47 @@ namespace snmalloc public: constexpr FlatPagemap() = default; - template - void init(ASM* a, address_t b = 0, size_t s = 0) + template + std::enable_if_t init(ASM* a, address_t b, size_t s) { - if constexpr (has_bounds) - { + static_assert( + has_bounds_ == has_bounds, "Don't set SFINAE template parameter!"); #ifdef SNMALLOC_TRACING - std::cout << "Pagemap.init " << (void*)b << " (" << s << ")" - << std::endl; + std::cout << "Pagemap.init " << (void*)b << " (" << s << ")" << std::endl; #endif - SNMALLOC_ASSERT(s != 0); - // Align the start and end. We won't store for the very ends as they - // are not aligned to a chunk boundary. - base = bits::align_up(b, bits::one_at_bit(GRANULARITY_BITS)); - auto end = bits::align_down(b + s, bits::one_at_bit(GRANULARITY_BITS)); - size = end - base; - body = a->template reserve( - bits::next_pow2((size >> SHIFT) * sizeof(T))) - .template as_static() - .unsafe_ptr(); - ; - } - else - { - // The parameters should not be set without has_bounds. - UNUSED(s); - UNUSED(b); - SNMALLOC_ASSERT(s == 0); - SNMALLOC_ASSERT(b == 0); - - static constexpr size_t COVERED_BITS = - bits::ADDRESS_BITS - GRANULARITY_BITS; - static constexpr size_t ENTRIES = bits::one_at_bit(COVERED_BITS); - auto new_body = (a->template reserve(ENTRIES * sizeof(T))) - .template as_static() - .unsafe_ptr(); - - // Ensure bottom page is committed - commit_entry(&new_body[0]); - - // Set up zero page - new_body[0] = body[0]; - - body = new_body; - // TODO this is pretty sparse, should we ignore huge pages for it? - // madvise(body, size, MADV_NOHUGEPAGE); - } + SNMALLOC_ASSERT(s != 0); + // Align the start and end. We won't store for the very ends as they + // are not aligned to a chunk boundary. + base = bits::align_up(b, bits::one_at_bit(GRANULARITY_BITS)); + auto end = bits::align_down(b + s, bits::one_at_bit(GRANULARITY_BITS)); + size = end - base; + body = a->template reserve( + bits::next_pow2((size >> SHIFT) * sizeof(T))) + .template as_static() + .unsafe_ptr(); + } + + template + std::enable_if_t init(ASM* a) + { + static_assert( + has_bounds_ == has_bounds, "Don't set SFINAE template parameter!"); + static constexpr size_t COVERED_BITS = + bits::ADDRESS_BITS - GRANULARITY_BITS; + static constexpr size_t ENTRIES = bits::one_at_bit(COVERED_BITS); + auto new_body = (a->template reserve(ENTRIES * sizeof(T))) + .template as_static() + .unsafe_ptr(); + + // Ensure bottom page is committed + commit_entry(&new_body[0]); + + // Set up zero page + new_body[0] = body[0]; + + body = new_body; + // TODO this is pretty sparse, should we ignore huge pages for it? + // madvise(body, size, MADV_NOHUGEPAGE); } /** From da01d5b4ca79a3b6a3f352486d6b01c681500894 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 16 Jul 2021 10:36:13 +0100 Subject: [PATCH 012/302] Remove address space usage from Pagemap. The pagemap allocates it self directly either from * the original fixed address range it is supplied, and returns the remaining space after the pagemap is removed; or * directly allocated from the PAL without using the address space manager. This change in layering is required for the next commit, which imposes the invariant that the pagemap has been committed for all spaced managed by the address space manager. --- src/backend/backend.h | 19 +++++-- src/backend/pagemap.h | 90 +++++++++++++++++++++++--------- src/ds/address.h | 7 +++ src/test/func/pagemap/pagemap.cc | 19 +++++-- 4 files changed, 101 insertions(+), 34 deletions(-) diff --git a/src/backend/backend.h b/src/backend/backend.h index 34541b303..c5eb247ee 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -50,9 +50,13 @@ namespace snmalloc FlatPagemap pagemap; public: - void init() + template + std::enable_if_t init() { - pagemap.init(&address_space); + static_assert( + fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); + + pagemap.init(); if constexpr (fixed_range) { @@ -60,10 +64,15 @@ namespace snmalloc } } - void init(CapPtr base, size_t length) + template + std::enable_if_t + init(CapPtr base, size_t length) { - address_space.add_range(base, length); - pagemap.init(&address_space, address_cast(base), length); + static_assert( + fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); + + auto [heap_base, heap_length] = pagemap.init(base, length); + address_space.add_range(heap_base, heap_length); if constexpr (!fixed_range) { diff --git a/src/backend/pagemap.h b/src/backend/pagemap.h index 727e3b445..06252d598 100644 --- a/src/backend/pagemap.h +++ b/src/backend/pagemap.h @@ -19,17 +19,30 @@ namespace snmalloc private: static constexpr size_t SHIFT = GRANULARITY_BITS; - // Before init is called will contain a single entry - // that is the default value. This is needed so that - // various calls do not have to check for nullptr. - // free(nullptr) - // and - // malloc_usable_size(nullptr) - // do not require an allocation to have ocurred before - // they are called. + /** + * Before init is called will contain a single entry + * that is the default value. This is needed so that + * various calls do not have to check for nullptr. + * free(nullptr) + * and + * malloc_usable_size(nullptr) + * do not require an allocation to have ocurred before + * they are called. + */ inline static const T default_value{}; + + /** + * The representation of the page map. + * + * Initially a single element to ensure nullptr operations + * work. + */ T* body{const_cast(&default_value)}; + /** + * If `has_bounds` is set, then these should contain the + * bounds of the heap that is being managed by this pagemap. + */ address_t base{0}; size_t size{0}; @@ -51,37 +64,66 @@ namespace snmalloc public: constexpr FlatPagemap() = default; - template - std::enable_if_t init(ASM* a, address_t b, size_t s) + /** + * Initialise the pagemap with bounds. + * + * Returns usable range after pagemap has been allocated + */ + template + std::enable_if_t, size_t>> + init(CapPtr b, size_t s) { static_assert( has_bounds_ == has_bounds, "Don't set SFINAE template parameter!"); #ifdef SNMALLOC_TRACING - std::cout << "Pagemap.init " << (void*)b << " (" << s << ")" << std::endl; + std::cout << "Pagemap.init " << b.unsafe_ptr() << " (" << s << ")" + << std::endl; #endif SNMALLOC_ASSERT(s != 0); + // TODO take account of pagemap size in the calculation of how big it + // needs to be. + // Align the start and end. We won't store for the very ends as they // are not aligned to a chunk boundary. - base = bits::align_up(b, bits::one_at_bit(GRANULARITY_BITS)); - auto end = bits::align_down(b + s, bits::one_at_bit(GRANULARITY_BITS)); - size = end - base; - body = a->template reserve( - bits::next_pow2((size >> SHIFT) * sizeof(T))) - .template as_static() - .unsafe_ptr(); + auto heap_base = pointer_align_up(b, bits::one_at_bit(GRANULARITY_BITS)); + auto end = pointer_align_down( + pointer_offset(b, s), bits::one_at_bit(GRANULARITY_BITS)); + size = pointer_diff(heap_base, end); + + // Put pagemap at start of range. + // TODO CHERI capability bound here! + body = b.as_reinterpret().unsafe_ptr(); + + // Advance by size of pagemap. + // TODO CHERI capability bound here! + heap_base = pointer_align_up( + pointer_offset(b, (size >> SHIFT) * sizeof(T)), + bits::one_at_bit(GRANULARITY_BITS)); + base = address_cast(heap_base); + SNMALLOC_ASSERT( + base == bits::align_up(base, bits::one_at_bit(GRANULARITY_BITS))); + return {heap_base, pointer_diff(heap_base, end)}; } - template - std::enable_if_t init(ASM* a) + /** + * Initialise the pagemap without bounds. + */ + template + std::enable_if_t init() { static_assert( has_bounds_ == has_bounds, "Don't set SFINAE template parameter!"); static constexpr size_t COVERED_BITS = bits::ADDRESS_BITS - GRANULARITY_BITS; static constexpr size_t ENTRIES = bits::one_at_bit(COVERED_BITS); - auto new_body = (a->template reserve(ENTRIES * sizeof(T))) - .template as_static() - .unsafe_ptr(); + + // TODO request additional space, and move to random offset. + + // TODO wasting space if size2 bigger than needed. + auto [new_body_untyped, size2] = + Pal::reserve_at_least(ENTRIES * sizeof(T)); + + auto new_body = reinterpret_cast(new_body_untyped); // Ensure bottom page is committed commit_entry(&new_body[0]); @@ -90,8 +132,6 @@ namespace snmalloc new_body[0] = body[0]; body = new_body; - // TODO this is pretty sparse, should we ignore huge pages for it? - // madvise(body, size, MADV_NOHUGEPAGE); } /** diff --git a/src/ds/address.h b/src/ds/address.h index d46da593b..15fc47bd1 100644 --- a/src/ds/address.h +++ b/src/ds/address.h @@ -194,6 +194,13 @@ namespace snmalloc #endif } + template + inline CapPtr + pointer_align_down(CapPtr p, size_t alignment) + { + return CapPtr(pointer_align_down(p.unsafe_ptr(), alignment)); + } + /** * Align a pointer up to a dynamically specified granularity, which must * be a power of two. diff --git a/src/test/func/pagemap/pagemap.cc b/src/test/func/pagemap/pagemap.cc index 5f02c0e70..7758b4fc4 100644 --- a/src/test/func/pagemap/pagemap.cc +++ b/src/test/func/pagemap/pagemap.cc @@ -76,11 +76,22 @@ void test_pagemap(bool bounded) // Initialise the pagemap if (bounded) { - pagemap_test_bound.init(&address_space, low, high); + auto size = bits::one_at_bit(30); + auto base = address_space.reserve(size); + std::cout << "Fixed base: " << base.unsafe_ptr() << " (" << size << ") " + << " end: " << pointer_offset(base, size).unsafe_ptr() + << std::endl; + auto [heap_base, heap_size] = pagemap_test_bound.init(base, size); + std::cout << "Heap base: " << heap_base.unsafe_ptr() << " (" << heap_size + << ") " + << " end: " << pointer_offset(heap_base, heap_size).unsafe_ptr() + << std::endl; + low = address_cast(heap_base); + high = low + heap_size; } else { - pagemap_test_unbound.init(&address_space); + pagemap_test_unbound.init(); } // Nullptr should still work after init. @@ -95,7 +106,7 @@ void test_pagemap(bool bounded) value.v++; if (value.v == T().v) value = 0; - if ((ptr % (1ULL << 26)) == 0) + if (((ptr - low) % (1ULL << 26)) == 0) std::cout << "." << std::flush; } @@ -110,7 +121,7 @@ void test_pagemap(bool bounded) if (value.v == T().v) value = 0; - if ((ptr % (1ULL << 26)) == 0) + if (((ptr - low) % (1ULL << 26)) == 0) std::cout << "." << std::flush; } std::cout << std::endl; From 02d2ab8f7e0751f8016d96b116a774388a2b4c48 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 16 Jul 2021 14:27:16 +0100 Subject: [PATCH 013/302] Pagemap requires registration of used space This PR exposes a pagemap interface to specify ranges that are being used. The overall invariant is that any memory in the address space manager has the pagemap committed. This means that individual operations do not need to commit entries. This is important for Windows that does not support lazy commit. It is also important if we want to PROT_NONE most of the pagemap to reduce the risk of memory safety issues getting access to the pagemap. There are minor changes to test to pull memory directly from the Pal. There are also bug fixes in the pagemap tests. --- src/backend/address_space.h | 164 +++++++++------------ src/backend/backend.h | 11 +- src/backend/pagemap.h | 55 +++---- src/mem/fixedglobalconfig.h | 2 +- src/test/func/fixed_region/fixed_region.cc | 16 +- src/test/func/pagemap/pagemap.cc | 32 ++-- src/test/func/two_alloc_types/alloc1.cc | 3 +- 7 files changed, 113 insertions(+), 170 deletions(-) diff --git a/src/backend/address_space.h b/src/backend/address_space.h index 77f45cbdb..d34b61ee3 100644 --- a/src/backend/address_space.h +++ b/src/backend/address_space.h @@ -42,8 +42,8 @@ namespace snmalloc * part of satisfying the request will be registered with the provided * arena_map for use in subsequent amplification. */ - template - CapPtr reserve(size_t size) + template + CapPtr reserve(size_t size, Pagemap& pagemap) { #ifdef SNMALLOC_TRACING std::cout << "ASM reserve request:" << size << std::endl; @@ -51,108 +51,88 @@ namespace snmalloc SNMALLOC_ASSERT(bits::is_pow2(size)); SNMALLOC_ASSERT(size >= sizeof(void*)); - if constexpr ((align == false) && !pal_supports) + /* + * For sufficiently large allocations with platforms that support + * aligned allocations and architectures that don't require + * StrictProvenance, try asking the platform first. + */ + if constexpr ( + pal_supports && !aal_supports) { - if constexpr (pal_supports) + if (size >= PAL::minimum_alloc_size) { - // TODO wasting size here. - size = bits::max(size, PAL::minimum_alloc_size); - return CapPtr( + auto base = CapPtr( PAL::template reserve_aligned(size)); - } - else - { - auto [block, size2] = PAL::reserve_at_least(size); - // TODO wasting size here. - UNUSED(size2); -#ifdef SNMALLOC_TRACING - std::cout << "Unaligned alloc here:" << block << " (" << size2 << ")" - << std::endl; -#endif - return CapPtr(block); + pagemap.register_range(address_cast(base), size); + return base; } } - else - { - /* - * For sufficiently large allocations with platforms that support - * aligned allocations and architectures that don't require - * StrictProvenance, try asking the platform first. - */ - if constexpr ( - pal_supports && - !aal_supports) - { - if (size >= PAL::minimum_alloc_size) - return CapPtr( - PAL::template reserve_aligned(size)); - } - CapPtr res; + CapPtr res; + { + FlagLock lock(spin_lock); + res = core.template reserve(size); + if (res == nullptr) { - FlagLock lock(spin_lock); - res = core.template reserve(size); - if (res == nullptr) + // Allocation failed ask OS for more memory + CapPtr block = nullptr; + size_t block_size = 0; + if constexpr (pal_supports) { - // Allocation failed ask OS for more memory - CapPtr block = nullptr; - size_t block_size = 0; - if constexpr (pal_supports) - { - /* - * We will have handled the case where size >= - * minimum_alloc_size above, so we are left to handle only small - * things here. - */ - block_size = PAL::minimum_alloc_size; - - void* block_raw = - PAL::template reserve_aligned(block_size); - - // It's a bit of a lie to convert without applying bounds, but the - // platform will have bounded block for us and it's better that - // the rest of our internals expect CBChunk bounds. - block = CapPtr(block_raw); - } - else if constexpr (!pal_supports) - { - // Need at least 2 times the space to guarantee alignment. - // Hold lock here as a race could cause additional requests to - // the PAL, and this could lead to suprious OOM. This is - // particularly bad if the PAL gives all the memory on first call. - auto block_and_size = PAL::reserve_at_least(size * 2); - block = CapPtr(block_and_size.first); - block_size = block_and_size.second; - - // Ensure block is pointer aligned. - if ( - pointer_align_up(block, sizeof(void*)) != block || - bits::align_up(block_size, sizeof(void*)) > block_size) - { - auto diff = - pointer_diff(block, pointer_align_up(block, sizeof(void*))); - block_size = block_size - diff; - block_size = bits::align_down(block_size, sizeof(void*)); - } - } - if (block == nullptr) + /* + * We will have handled the case where size >= + * minimum_alloc_size above, so we are left to handle only small + * things here. + */ + block_size = PAL::minimum_alloc_size; + + void* block_raw = PAL::template reserve_aligned(block_size); + + // It's a bit of a lie to convert without applying bounds, but the + // platform will have bounded block for us and it's better that + // the rest of our internals expect CBChunk bounds. + block = CapPtr(block_raw); + } + else if constexpr (!pal_supports) + { + // Need at least 2 times the space to guarantee alignment. + // Hold lock here as a race could cause additional requests to + // the PAL, and this could lead to suprious OOM. This is + // particularly bad if the PAL gives all the memory on first call. + auto block_and_size = PAL::reserve_at_least(size * 2); + block = CapPtr(block_and_size.first); + block_size = block_and_size.second; + + // Ensure block is pointer aligned. + if ( + pointer_align_up(block, sizeof(void*)) != block || + bits::align_up(block_size, sizeof(void*)) > block_size) { - return nullptr; + auto diff = + pointer_diff(block, pointer_align_up(block, sizeof(void*))); + block_size = block_size - diff; + block_size = bits::align_down(block_size, sizeof(void*)); } + } + if (block == nullptr) + { + return nullptr; + } - core.template add_range(block, block_size); + pagemap.register_range(address_cast(block), block_size); - // still holding lock so guaranteed to succeed. - res = core.template reserve(size); - } + core.template add_range(block, block_size); + + // still holding lock so guaranteed to succeed. + res = core.template reserve(size); } + } - // Don't need lock while committing pages. - if constexpr (committed) - core.template commit_block(res, size); + // Don't need lock while committing pages. + if constexpr (committed) + core.template commit_block(res, size); - return res; - } + return res; } /** @@ -162,8 +142,8 @@ namespace snmalloc * This is useful for allowing the space required for alignment to be * used, by smaller objects. */ - template - CapPtr reserve_with_left_over(size_t size) + template + CapPtr reserve_with_left_over(size_t size, Pagemap& pagemap) { SNMALLOC_ASSERT(size >= sizeof(void*)); @@ -171,7 +151,7 @@ namespace snmalloc size_t rsize = bits::next_pow2(size); - auto res = reserve(rsize); + auto res = reserve(rsize, pagemap); if (res != nullptr) { diff --git a/src/backend/backend.h b/src/backend/backend.h index c5eb247ee..0d4618afc 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -65,14 +65,13 @@ namespace snmalloc } template - std::enable_if_t - init(CapPtr base, size_t length) + std::enable_if_t init(void* base, size_t length) { static_assert( fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); auto [heap_base, heap_length] = pagemap.init(base, length); - address_space.add_range(heap_base, heap_length); + address_space.add_range(CapPtr(heap_base), heap_length); if constexpr (!fixed_range) { @@ -106,7 +105,7 @@ namespace snmalloc auto& a = h.address_space; // TODO Improve heuristics and params auto refill_size = bits::max(size, bits::one_at_bit(21)); - auto refill = a.template reserve(refill_size); + auto refill = a.template reserve(refill_size, h.pagemap); if (refill == nullptr) return nullptr; local_state->local_address_space.template add_range( @@ -122,7 +121,7 @@ namespace snmalloc else { auto& a = h.address_space; - p = a.template reserve_with_left_over(size); + p = a.template reserve_with_left_over(size, h.pagemap); } return p; @@ -183,7 +182,7 @@ namespace snmalloc a < address_cast(pointer_offset(p, size)); a += MIN_CHUNK_SIZE) { - h.pagemap.add(a, t); + h.pagemap.set(a, t); } return {p, meta}; } diff --git a/src/backend/pagemap.h b/src/backend/pagemap.h index 06252d598..4b9d43755 100644 --- a/src/backend/pagemap.h +++ b/src/backend/pagemap.h @@ -46,22 +46,23 @@ namespace snmalloc address_t base{0}; size_t size{0}; + public: /** - * Commit entry + * Ensure this range of pagemap is accessible */ - void commit_entry(void* p) + void register_range(address_t p, size_t length) { - auto entry_size = sizeof(T); - static_assert(sizeof(T) < OS_PAGE_SIZE); - // Rounding required for sub-page allocations. - auto page_start = pointer_align_down(p); - auto page_end = - pointer_align_up(pointer_offset(p, entry_size)); + // Calculate range in pagemap that is associated to this space. + auto first = &body[p >> SHIFT]; + auto last = &body[(p + length + bits::one_at_bit(SHIFT) - 1) >> SHIFT]; + + // Commit OS pages associated to the range. + auto page_start = pointer_align_down(first); + auto page_end = pointer_align_up(last); size_t using_size = pointer_diff(page_start, page_end); PAL::template notify_using(page_start, using_size); } - public: constexpr FlatPagemap() = default; /** @@ -70,14 +71,13 @@ namespace snmalloc * Returns usable range after pagemap has been allocated */ template - std::enable_if_t, size_t>> - init(CapPtr b, size_t s) + std::enable_if_t> + init(void* b, size_t s) { static_assert( has_bounds_ == has_bounds, "Don't set SFINAE template parameter!"); #ifdef SNMALLOC_TRACING - std::cout << "Pagemap.init " << b.unsafe_ptr() << " (" << s << ")" - << std::endl; + std::cout << "Pagemap.init " << b << " (" << s << ")" << std::endl; #endif SNMALLOC_ASSERT(s != 0); // TODO take account of pagemap size in the calculation of how big it @@ -92,7 +92,7 @@ namespace snmalloc // Put pagemap at start of range. // TODO CHERI capability bound here! - body = b.as_reinterpret().unsafe_ptr(); + body = reinterpret_cast(b); // Advance by size of pagemap. // TODO CHERI capability bound here! @@ -126,7 +126,8 @@ namespace snmalloc auto new_body = reinterpret_cast(new_body_untyped); // Ensure bottom page is committed - commit_entry(&new_body[0]); + // ASSUME: new memory is zeroed. + Pal::notify_using(new_body, OS_PAGE_SIZE); // Set up zero page new_body[0] = body[0]; @@ -166,7 +167,7 @@ namespace snmalloc // This means external pointer on Windows will be slow. if constexpr (potentially_out_of_range && !pal_supports) { - commit_entry(&body[p >> SHIFT]); + register_range(p, 1); } return body[p >> SHIFT]; @@ -188,27 +189,5 @@ namespace snmalloc body[p >> SHIFT] = t; } - - void add(address_t p, T t) - { -#ifdef SNMALLOC_TRACING - std::cout << "Pagemap.Add " << (void*)p << std::endl; -#endif - if constexpr (has_bounds) - { - if (p - base > size) - { - PAL::error("Internal error: Pagemap new write access out of range."); - } - p = p - base; - } - - // This could be the first time this page is used - // This will potentially be expensive on Windows, - // and we should revisit the performance here. - commit_entry(&body[p >> SHIFT]); - - body[p >> SHIFT] = t; - } }; } // namespace snmalloc diff --git a/src/mem/fixedglobalconfig.h b/src/mem/fixedglobalconfig.h index 4878453c7..01b268f5f 100644 --- a/src/mem/fixedglobalconfig.h +++ b/src/mem/fixedglobalconfig.h @@ -66,7 +66,7 @@ namespace snmalloc snmalloc::register_clean_up(); } - static void init(CapPtr base, size_t length) + static void init(void* base, size_t length) { get_backend_state().init(base, length); } diff --git a/src/test/func/fixed_region/fixed_region.cc b/src/test/func/fixed_region/fixed_region.cc index 702aaee54..a7db57a58 100644 --- a/src/test/func/fixed_region/fixed_region.cc +++ b/src/test/func/fixed_region/fixed_region.cc @@ -18,18 +18,14 @@ int main() { #ifndef SNMALLOC_PASS_THROUGH // Depends on snmalloc specific features - // Create a standard address space to get initial allocation - // this just bypasses having to understand the test platform. - AddressSpaceManager address_space; - // 28 is large enough to produce a nested allocator. // It is also large enough for the example to run in. // For 1MiB superslabs, SUPERSLAB_BITS + 4 is not big enough for the example. - size_t size = bits::one_at_bit(28); - auto oe_base = address_space.reserve(size); - auto oe_end = pointer_offset(oe_base, size).unsafe_ptr(); - std::cout << "Allocated region " << oe_base.unsafe_ptr() << " - " - << pointer_offset(oe_base, size).unsafe_ptr() << std::endl; + auto [oe_base, size] = Pal::reserve_at_least(bits::one_at_bit(28)); + Pal::notify_using(oe_base, size); + auto oe_end = pointer_offset(oe_base, size); + std::cout << "Allocated region " << oe_base << " - " + << pointer_offset(oe_base, size) << std::endl; CustomGlobals fixed_handle; CustomGlobals::init(oe_base, size); @@ -47,7 +43,7 @@ int main() if (r1 == nullptr) break; - if (oe_base.unsafe_ptr() > r1) + if (oe_base > r1) { std::cout << "Allocated: " << r1 << std::endl; abort(); diff --git a/src/test/func/pagemap/pagemap.cc b/src/test/func/pagemap/pagemap.cc index 7758b4fc4..dc5d5af63 100644 --- a/src/test/func/pagemap/pagemap.cc +++ b/src/test/func/pagemap/pagemap.cc @@ -47,14 +47,6 @@ void check_get( } } -void add(bool bounded, address_t address, T new_value) -{ - if (bounded) - pagemap_test_bound.add(address, new_value); - else - pagemap_test_unbound.add(address, new_value); -} - void set(bool bounded, address_t address, T new_value) { if (bounded) @@ -71,38 +63,36 @@ void test_pagemap(bool bounded) address_t high = bits::one_at_bit(30); // Nullptr needs to work before initialisation - CHECK_GET(true, 0, T()); + CHECK_GET(bounded, 0, T()); // Initialise the pagemap if (bounded) { - auto size = bits::one_at_bit(30); - auto base = address_space.reserve(size); - std::cout << "Fixed base: " << base.unsafe_ptr() << " (" << size << ") " - << " end: " << pointer_offset(base, size).unsafe_ptr() - << std::endl; + auto [base, size] = Pal::reserve_at_least(bits::one_at_bit(30)); + Pal::notify_using(base, size); + std::cout << "Fixed base: " << base << " (" << size << ") " + << " end: " << pointer_offset(base, size) << std::endl; auto [heap_base, heap_size] = pagemap_test_bound.init(base, size); - std::cout << "Heap base: " << heap_base.unsafe_ptr() << " (" << heap_size - << ") " - << " end: " << pointer_offset(heap_base, heap_size).unsafe_ptr() - << std::endl; + std::cout << "Heap base: " << heap_base << " (" << heap_size << ") " + << " end: " << pointer_offset(heap_base, heap_size) << std::endl; low = address_cast(heap_base); high = low + heap_size; } else { pagemap_test_unbound.init(); + pagemap_test_unbound.register_range(low, high - low); } // Nullptr should still work after init. - CHECK_GET(true, 0, T()); + CHECK_GET(bounded, 0, T()); // Store a pattern into page map T value = 1; for (uintptr_t ptr = low; ptr < high; ptr += bits::one_at_bit(GRANULARITY_BITS + 3)) { - add(false, ptr, value); + set(bounded, ptr, value); value.v++; if (value.v == T().v) value = 0; @@ -116,7 +106,7 @@ void test_pagemap(bool bounded) for (uintptr_t ptr = low; ptr < high; ptr += bits::one_at_bit(GRANULARITY_BITS + 3)) { - CHECK_GET(false, ptr, value); + CHECK_GET(bounded, ptr, value); value.v++; if (value.v == T().v) value = 0; diff --git a/src/test/func/two_alloc_types/alloc1.cc b/src/test/func/two_alloc_types/alloc1.cc index 2c15db55a..4e5f9d0bf 100644 --- a/src/test/func/two_alloc_types/alloc1.cc +++ b/src/test/func/two_alloc_types/alloc1.cc @@ -20,6 +20,5 @@ namespace snmalloc extern "C" void oe_allocator_init(void* base, void* end) { snmalloc::CustomGlobals fixed_handle; - fixed_handle.init( - CapPtr(base), address_cast(end) - address_cast(base)); + fixed_handle.init(base, address_cast(end) - address_cast(base)); } From 9df0101dfd91c67d4b6e1607abf39ba1b0bae056 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 16 Jul 2021 16:35:54 +0100 Subject: [PATCH 014/302] Enable guard pages in CHECK_CLIENT Change the behaviour to use PROT_NONE for reservations in CHECK_CLIENT mode. This means that we only provide access once data is actually being used. --- CMakeLists.txt | 20 +++----------------- src/ds/defines.h | 8 +------- src/ds/helpers.h | 2 +- src/mem/allocconfig.h | 2 +- src/mem/corealloc.h | 4 ++-- src/mem/freelist.h | 20 ++++++++++---------- src/mem/metaslab.h | 4 ++-- src/mem/sizeclasstable.h | 2 +- src/pal/pal_apple.h | 8 ++++---- src/pal/pal_bsd.h | 4 ++-- src/pal/pal_bsd_aligned.h | 8 +++++++- src/pal/pal_posix.h | 14 +++++++++++--- 12 files changed, 45 insertions(+), 51 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 878e5cbb6..f4f8e6735 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -50,13 +50,6 @@ int main() { } " SNMALLOC_PLATFORM_HAS_GETENTROPY) -if (NOT SNMALLOC_CI_BUILD) - option(USE_POSIX_COMMIT_CHECKS "Instrument Posix PAL to check for access to unused blocks of memory." Off) -else () - # This is enabled in every bit of CI to detect errors. - option(USE_POSIX_COMMIT_CHECKS "Instrument Posix PAL to check for access to unused blocks of memory." On) -endif () - # Provide as macro so other projects can reuse macro(warnings_high) if(MSVC) @@ -169,10 +162,6 @@ if(SNMALLOC_CI_BUILD) target_compile_definitions(snmalloc_lib INTERFACE -DSNMALLOC_CI_BUILD) endif() -if(USE_POSIX_COMMIT_CHECKS) - target_compile_definitions(snmalloc_lib INTERFACE -DUSE_POSIX_COMMIT_CHECKS) -endif() - if(SNMALLOC_PLATFORM_HAS_GETENTROPY) target_compile_definitions(snmalloc_lib INTERFACE -DSNMALLOC_PLATFORM_HAS_GETENTROPY) endif() @@ -277,13 +266,13 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) set(SHARED_FILES src/override/new.cc) add_shim(snmallocshim SHARED ${SHARED_FILES}) add_shim(snmallocshim-checks SHARED ${SHARED_FILES}) - target_compile_definitions(snmallocshim-checks PRIVATE CHECK_CLIENT) + target_compile_definitions(snmallocshim-checks PRIVATE SNMALLOC_CHECK_CLIENT) endif() if(SNMALLOC_RUST_SUPPORT) add_shim(snmallocshim-rust STATIC src/override/rust.cc) add_shim(snmallocshim-checks-rust STATIC src/override/rust.cc) - target_compile_definitions(snmallocshim-checks-rust PRIVATE CHECK_CLIENT) + target_compile_definitions(snmallocshim-checks-rust PRIVATE SNMALLOC_CHECK_CLIENT) endif() enable_testing() @@ -313,14 +302,11 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) add_executable(${TESTNAME} ${SRC}) - # For all tests enable commit checking. - target_compile_definitions(${TESTNAME} PRIVATE -DUSE_POSIX_COMMIT_CHECKS) - if (${FLAVOUR} STREQUAL "malloc") target_compile_definitions(${TESTNAME} PRIVATE SNMALLOC_PASS_THROUGH) endif() if (${FLAVOUR} STREQUAL "check") - target_compile_definitions(${TESTNAME} PRIVATE CHECK_CLIENT) + target_compile_definitions(${TESTNAME} PRIVATE SNMALLOC_CHECK_CLIENT) endif() target_link_libraries(${TESTNAME} snmalloc_lib) if (${TEST} MATCHES "release-.*") diff --git a/src/ds/defines.h b/src/ds/defines.h index 88ff5ea3c..f5d6646f7 100644 --- a/src/ds/defines.h +++ b/src/ds/defines.h @@ -114,12 +114,6 @@ namespace snmalloc # endif #endif -// // The CHECK_CLIENT macro is used to turn on minimal checking of the client -// // calling the API correctly. -// #if !defined(NDEBUG) && !defined(CHECK_CLIENT) -// # define CHECK_CLIENT -// #endif - inline SNMALLOC_FAST_PATH void check_client_error(const char* const str) { //[[clang::musttail]] @@ -132,7 +126,7 @@ check_client_impl(bool test, const char* const str) if (unlikely(!test)) check_client_error(str); } -#ifdef CHECK_CLIENT +#ifdef SNMALLOC_CHECK_CLIENT # define check_client(test, str) check_client_impl(test, str) #else # define check_client(test, str) diff --git a/src/ds/helpers.h b/src/ds/helpers.h index a3d3e8886..e34387d9a 100644 --- a/src/ds/helpers.h +++ b/src/ds/helpers.h @@ -73,7 +73,7 @@ namespace snmalloc } }; -#ifdef CHECK_CLIENT +#ifdef SNMALLOC_CHECK_CLIENT template class ModArray { diff --git a/src/mem/allocconfig.h b/src/mem/allocconfig.h index 153e05bf9..660bc941e 100644 --- a/src/mem/allocconfig.h +++ b/src/mem/allocconfig.h @@ -80,7 +80,7 @@ namespace snmalloc static constexpr size_t MIN_CHUNK_SIZE = bits::one_at_bit(MIN_CHUNK_BITS); // Minimum number of objects on a slab -#ifdef CHECK_CLIENT +#ifdef SNMALLOC_CHECK_CLIENT static constexpr size_t MIN_OBJECT_COUNT = 13; #else static constexpr size_t MIN_OBJECT_COUNT = 4; diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 4df9ca8d2..fb4ff91f8 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -146,7 +146,7 @@ namespace snmalloc FreeListBuilder b; SNMALLOC_ASSERT(b.empty()); -#ifdef CHECK_CLIENT +#ifdef SNMALLOC_CHECK_CLIENT // Structure to represent the temporary list elements struct PreAllocObject { @@ -214,7 +214,7 @@ namespace snmalloc meta->free_queue.close(fl, key); void* p = finish_alloc_no_zero(fl.take(key), sizeclass); -#ifdef CHECK_CLIENT +#ifdef SNMALLOC_CHECK_CLIENT // Check free list is well-formed on platforms with // integers as pointers. size_t count = 1; // Already taken one above. diff --git a/src/mem/freelist.h b/src/mem/freelist.h index 50dbc0446..e6a173b16 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -90,7 +90,7 @@ namespace snmalloc // TODO: Should really use C++20 atomic_ref rather than a union. AtomicCapPtr atomic_next_object; }; -#ifdef CHECK_CLIENT +#ifdef SNMALLOC_CHECK_CLIENT // Encoded representation of a back pointer. // Hard to fake, and provides consistency on // the next pointers. @@ -104,7 +104,7 @@ namespace snmalloc } /** - * Assign next_object and update its prev_encoded if CHECK_CLIENT. + * Assign next_object and update its prev_encoded if SNMALLOC_CHECK_CLIENT. * Static so that it can be used on reference to a FreeObject. * * Returns a pointer to the next_object field of the next parameter as an @@ -116,7 +116,7 @@ namespace snmalloc CapPtr next, FreeListKey& key) { -#ifdef CHECK_CLIENT +#ifdef SNMALLOC_CHECK_CLIENT next->prev_encoded = signed_prev(address_cast(curr), address_cast(next), key); #else @@ -127,13 +127,13 @@ namespace snmalloc } /** - * Assign next_object and update its prev_encoded if CHECK_CLIENT + * Assign next_object and update its prev_encoded if SNMALLOC_CHECK_CLIENT * * Uses the atomic view of next, so can be used in the message queues. */ void atomic_store_next(CapPtr next, FreeListKey& key) { -#ifdef CHECK_CLIENT +#ifdef SNMALLOC_CHECK_CLIENT next->prev_encoded = signed_prev(address_cast(this), address_cast(next), key); #else @@ -152,7 +152,7 @@ namespace snmalloc CapPtr atomic_read_next(FreeListKey& key) { auto n = atomic_next_object.load(std::memory_order_acquire); -#ifdef CHECK_CLIENT +#ifdef SNMALLOC_CHECK_CLIENT if (n != nullptr) { n->check_prev(signed_prev(address_cast(this), address_cast(n), key)); @@ -194,7 +194,7 @@ namespace snmalloc class FreeListIter { CapPtr curr{nullptr}; -#ifdef CHECK_CLIENT +#ifdef SNMALLOC_CHECK_CLIENT address_t prev{0}; #endif @@ -203,7 +203,7 @@ namespace snmalloc CapPtr head, address_t prev_value) : curr(head) { -#ifdef CHECK_CLIENT +#ifdef SNMALLOC_CHECK_CLIENT prev = prev_value; #endif UNUSED(prev_value); @@ -237,7 +237,7 @@ namespace snmalloc Aal::prefetch(next.unsafe_ptr()); curr = next; -#ifdef CHECK_CLIENT +#ifdef SNMALLOC_CHECK_CLIENT c->check_prev(prev); prev = signed_prev(address_cast(c), address_cast(next), key); #else @@ -251,7 +251,7 @@ namespace snmalloc /** * Used to build a free list in object space. * - * Adds signing of pointers in the CHECK_CLIENT mode + * Adds signing of pointers in the SNMALLOC_CHECK_CLIENT mode * * We use the template parameter, so that an enclosing * class can make use of the remaining bytes, which may not diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index ae2b7eabd..c8194dfcc 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -30,7 +30,7 @@ namespace snmalloc * * Spare 32bits are used for the fields in MetaslabEnd. */ -#ifdef CHECK_CLIENT +#ifdef SNMALLOC_CHECK_CLIENT FreeListBuilder free_queue; #else FreeListBuilder free_queue; @@ -149,7 +149,7 @@ namespace snmalloc auto p = tmp_fl.take(key); fast_free_list = tmp_fl; -#ifdef CHECK_CLIENT +#ifdef SNMALLOC_CHECK_CLIENT entropy.refresh_bits(); #else UNUSED(entropy); diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index a8607d5c5..13c3f69ee 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -166,7 +166,7 @@ namespace snmalloc */ inline uint16_t threshold_for_waking_slab(sizeclass_t sizeclass) { - // #ifdef CHECK_CLIENT + // #ifdef SNMALLOC_CHECK_CLIENT return sizeclass_metadata.waking[sizeclass]; // #else // UNUSED(sizeclass); diff --git a/src/pal/pal_apple.h b/src/pal/pal_apple.h index 685ae9353..0c24fa905 100644 --- a/src/pal/pal_apple.h +++ b/src/pal/pal_apple.h @@ -113,7 +113,7 @@ namespace snmalloc { SNMALLOC_ASSERT(is_aligned_block(p, size)); -# ifdef USE_POSIX_COMMIT_CHECKS +# if defined(SNMALLOC_CHECK_CLIENT) && !defined(NDEBUG) memset(p, 0x5a, size); # endif @@ -126,7 +126,7 @@ namespace snmalloc while (madvise(p, size, MADV_FREE_REUSABLE) == -1 && errno == EAGAIN) ; -# ifdef USE_POSIX_COMMIT_CHECKS +# ifdef SNMALLOC_CHECK_CLIENT // This must occur after `MADV_FREE_REUSABLE`. // // `mach_vm_protect` is observably slower in benchmarks. @@ -179,7 +179,7 @@ namespace snmalloc } } -# ifdef USE_POSIX_COMMIT_CHECKS +# ifdef SNMALLOC_CHECK_CLIENT // Mark pages as writable for `madvise` below. // // `mach_vm_protect` is observably slower in benchmarks. @@ -218,7 +218,7 @@ namespace snmalloc // must be initialized to 0 or addr is interepreted as a lower-bound. mach_vm_address_t addr = 0; -# ifdef USE_POSIX_COMMIT_CHECKS +# ifdef SNMALLOC_CHECK_CLIENT vm_prot_t prot = committed ? VM_PROT_READ | VM_PROT_WRITE : VM_PROT_NONE; # else vm_prot_t prot = VM_PROT_READ | VM_PROT_WRITE; diff --git a/src/pal/pal_bsd.h b/src/pal/pal_bsd.h index 1c3577623..38b357165 100644 --- a/src/pal/pal_bsd.h +++ b/src/pal/pal_bsd.h @@ -35,11 +35,11 @@ namespace snmalloc { SNMALLOC_ASSERT(is_aligned_block(p, size)); // Call this Pal to simulate the Windows decommit in CI. -#ifdef USE_POSIX_COMMIT_CHECKS +#if defined(SNMALLOC_CHECK_CLIENT) && !defined(NDEBUG) memset(p, 0x5a, size); #endif madvise(p, size, MADV_FREE); -#ifdef USE_POSIX_COMMIT_CHECKS +#ifdef SNMALLOC_CHECK_CLIENT mprotect(p, size, PROT_NONE); #endif } diff --git a/src/pal/pal_bsd_aligned.h b/src/pal/pal_bsd_aligned.h index 1cdd23494..c636e50f5 100644 --- a/src/pal/pal_bsd_aligned.h +++ b/src/pal/pal_bsd_aligned.h @@ -37,10 +37,16 @@ namespace snmalloc int log2align = static_cast(bits::next_pow2_bits(size)); +#ifdef SNMALLOC_CHECK_CLIENT + auto prot = committed ? PROT_READ | PROT_WRITE : PROT_NONE; +#else + auto prot = PROT_READ | PROT_WRITE; +#endif + void* p = mmap( nullptr, size, - PROT_READ | PROT_WRITE, + prot, MAP_PRIVATE | MAP_ANONYMOUS | MAP_ALIGNED(log2align), -1, 0); diff --git a/src/pal/pal_posix.h b/src/pal/pal_posix.h index dde665ae5..353848796 100644 --- a/src/pal/pal_posix.h +++ b/src/pal/pal_posix.h @@ -165,10 +165,12 @@ namespace snmalloc static void notify_not_using(void* p, size_t size) noexcept { SNMALLOC_ASSERT(is_aligned_block(p, size)); -#ifdef USE_POSIX_COMMIT_CHECKS +#ifdef SNMALLOC_CHECK_CLIENT // Fill memory so that when we switch the pages back on we don't make // assumptions on the content. +# if !defined(NDEBUG) memset(p, 0x5a, size); +# endif mprotect(p, size, PROT_NONE); #else UNUSED(p); @@ -189,7 +191,7 @@ namespace snmalloc SNMALLOC_ASSERT( is_aligned_block(p, size) || (zero_mem == NoZero)); -#ifdef USE_POSIX_COMMIT_CHECKS +#ifdef SNMALLOC_CHECK_CLIENT mprotect(p, size, PROT_READ | PROT_WRITE); #else UNUSED(p); @@ -260,6 +262,12 @@ namespace snmalloc constexpr size_t min_size = bits::is64() ? bits::one_at_bit(31) : bits::one_at_bit(27); +#ifdef SNMALLOC_CHECK_CLIENT + auto prot = PROT_NONE; +#else + auto prot = PROT_READ | PROT_WRITE; +#endif + for (size_t size_request = bits::max(size, min_size); size_request >= size; size_request = size_request / 2) @@ -267,7 +275,7 @@ namespace snmalloc void* p = mmap( nullptr, size_request, - PROT_READ | PROT_WRITE, + prot, MAP_PRIVATE | MAP_ANONYMOUS | DefaultMMAPFlags::flags, AnonFD::fd, 0); From 5d0ae71423d0f211bc048a354fdb3155a9e74fe9 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 19 Jul 2021 11:20:02 +0100 Subject: [PATCH 015/302] Remove at_least The Pal was providing policy for overallocating a block of memory to achieve alignment make that part of the backend. The backend should be responsible for layout policy. --- docs/PORTING.md | 6 +-- src/backend/address_space.h | 21 +++++++--- src/backend/backend.h | 2 +- src/backend/pagemap.h | 4 +- src/pal/pal_concept.h | 16 ++++---- src/pal/pal_posix.h | 40 +++++++------------ src/pal/pal_windows.h | 21 +--------- src/test/func/fixed_region/fixed_region.cc | 3 +- src/test/func/pagemap/pagemap.cc | 3 +- .../perf/external_pointer/externalpointer.cc | 2 + 10 files changed, 50 insertions(+), 68 deletions(-) diff --git a/docs/PORTING.md b/docs/PORTING.md index fdf1dec48..8e394511c 100644 --- a/docs/PORTING.md +++ b/docs/PORTING.md @@ -47,14 +47,14 @@ pages, rather than zeroing them synchronously in this call ```c++ template static void* reserve_aligned(size_t size) noexcept; -static std::pair reserve_at_least(size_t size) noexcept; +static void* reserve(size_t size) noexcept; ``` -All platforms should provide `reserve_at_least` and can optionally provide +All platforms should provide `reserve` and can optionally provide `reserve_aligned` if the underlying system can provide strongly aligned memory regions. If the system guarantees only page alignment, implement only the second. The Pal is free to overallocate based on the platform's desire and snmalloc -will find suitably aligned blocks inside the region. `reserve_at_least` should +will find suitably aligned blocks inside the region. `reserve` should not commit memory as snmalloc will commit the range of memory it requires of what is returned. diff --git a/src/backend/address_space.h b/src/backend/address_space.h index d34b61ee3..e8d148351 100644 --- a/src/backend/address_space.h +++ b/src/backend/address_space.h @@ -96,12 +96,21 @@ namespace snmalloc else if constexpr (!pal_supports) { // Need at least 2 times the space to guarantee alignment. - // Hold lock here as a race could cause additional requests to - // the PAL, and this could lead to suprious OOM. This is - // particularly bad if the PAL gives all the memory on first call. - auto block_and_size = PAL::reserve_at_least(size * 2); - block = CapPtr(block_and_size.first); - block_size = block_and_size.second; + size_t needed_size = size * 2; + // Magic number (27) for over-allocating a block of memory + // These should be further refined based on experiments. + constexpr size_t min_size = bits::one_at_bit(27); + for (size_t size_request = bits::max(needed_size, min_size); + size_request >= needed_size; + size_request = size_request / 2) + { + block = CapPtr(PAL::reserve(size_request)); + if (block != nullptr) + { + block_size = size_request; + break; + } + } // Ensure block is pointer aligned. if ( diff --git a/src/backend/backend.h b/src/backend/backend.h index 0d4618afc..8893a4aa5 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -11,7 +11,7 @@ namespace snmalloc * This class implements the standard backend for handling allocations. * It abstracts page table management and address space management. */ - template + template class BackendAllocator { public: diff --git a/src/backend/pagemap.h b/src/backend/pagemap.h index 4b9d43755..f1cda4a7d 100644 --- a/src/backend/pagemap.h +++ b/src/backend/pagemap.h @@ -119,9 +119,7 @@ namespace snmalloc // TODO request additional space, and move to random offset. - // TODO wasting space if size2 bigger than needed. - auto [new_body_untyped, size2] = - Pal::reserve_at_least(ENTRIES * sizeof(T)); + auto new_body_untyped = Pal::reserve(ENTRIES * sizeof(T)); auto new_body = reinterpret_cast(new_body_untyped); diff --git a/src/pal/pal_concept.h b/src/pal/pal_concept.h index bebd130d5..42e034bc3 100644 --- a/src/pal/pal_concept.h +++ b/src/pal/pal_concept.h @@ -52,11 +52,10 @@ namespace snmalloc * Absent any feature flags, the PAL must support a crude primitive allocator */ template - concept ConceptPAL_reserve_at_least = - requires(PAL p, void* vp, std::size_t sz) + concept ConceptPAL_reserve = + requires(PAL p, std::size_t sz) { - { PAL::reserve_at_least(sz) } noexcept - -> ConceptSame>; + { PAL::reserve(sz) } noexcept -> ConceptSame; }; /** @@ -102,10 +101,11 @@ namespace snmalloc (!pal_supports || ConceptPAL_mem_low_notify) && (pal_supports || - (pal_supports && - ConceptPAL_reserve_aligned) || - (!pal_supports && - ConceptPAL_reserve_at_least)); + ( + (!pal_supports || + ConceptPAL_reserve_aligned) && + ConceptPAL_reserve) + ); } // namespace snmalloc #endif diff --git a/src/pal/pal_posix.h b/src/pal/pal_posix.h index 353848796..8f9e2f707 100644 --- a/src/pal/pal_posix.h +++ b/src/pal/pal_posix.h @@ -253,44 +253,32 @@ namespace snmalloc * POSIX does not define a portable interface for specifying alignment * greater than a page. */ - static std::pair reserve_at_least(size_t size) noexcept + static void* reserve(size_t size) noexcept { - SNMALLOC_ASSERT(bits::is_pow2(size)); - - // Magic number for over-allocating chosen by the Pal - // These should be further refined based on experiments. - constexpr size_t min_size = - bits::is64() ? bits::one_at_bit(31) : bits::one_at_bit(27); - #ifdef SNMALLOC_CHECK_CLIENT auto prot = PROT_NONE; #else auto prot = PROT_READ | PROT_WRITE; #endif - for (size_t size_request = bits::max(size, min_size); - size_request >= size; - size_request = size_request / 2) - { - void* p = mmap( - nullptr, - size_request, - prot, - MAP_PRIVATE | MAP_ANONYMOUS | DefaultMMAPFlags::flags, - AnonFD::fd, - 0); + void* p = mmap( + nullptr, + size, + prot, + MAP_PRIVATE | MAP_ANONYMOUS | DefaultMMAPFlags::flags, + AnonFD::fd, + 0); - if (p != MAP_FAILED) - { + if (p != MAP_FAILED) + { #ifdef SNMALLOC_TRACING - std::cout << "Pal_posix reserved: " << p << " (" << size_request - << ")" << std::endl; + std::cout << "Pal_posix reserved: " << p << " (" << size << ")" + << std::endl; #endif - return {p, size_request}; - } + return p; } - OS::error("Out of memory"); + return nullptr; } /** diff --git a/src/pal/pal_windows.h b/src/pal/pal_windows.h index c4f7219ac..6fe35643f 100644 --- a/src/pal/pal_windows.h +++ b/src/pal/pal_windows.h @@ -185,26 +185,9 @@ namespace snmalloc } # endif - static std::pair reserve_at_least(size_t size) noexcept + static void* reserve(size_t size) noexcept { - SNMALLOC_ASSERT(bits::is_pow2(size)); - - // Magic number for over-allocating chosen by the Pal - // These should be further refined based on experiments. - constexpr size_t min_size = - bits::is64() ? bits::one_at_bit(32) : bits::one_at_bit(28); - for (size_t size_request = bits::max(size, min_size); - size_request >= size; - size_request = size_request / 2) - { - void* ret = - VirtualAlloc(nullptr, size_request, MEM_RESERVE, PAGE_READWRITE); - if (ret != nullptr) - { - return std::pair(ret, size_request); - } - } - error("Failed to allocate memory\n"); + return VirtualAlloc(nullptr, size, MEM_RESERVE, PAGE_READWRITE); } /** diff --git a/src/test/func/fixed_region/fixed_region.cc b/src/test/func/fixed_region/fixed_region.cc index a7db57a58..b2350a9ae 100644 --- a/src/test/func/fixed_region/fixed_region.cc +++ b/src/test/func/fixed_region/fixed_region.cc @@ -21,7 +21,8 @@ int main() // 28 is large enough to produce a nested allocator. // It is also large enough for the example to run in. // For 1MiB superslabs, SUPERSLAB_BITS + 4 is not big enough for the example. - auto [oe_base, size] = Pal::reserve_at_least(bits::one_at_bit(28)); + auto size = bits::one_at_bit(28); + auto oe_base = Pal::reserve(size); Pal::notify_using(oe_base, size); auto oe_end = pointer_offset(oe_base, size); std::cout << "Allocated region " << oe_base << " - " diff --git a/src/test/func/pagemap/pagemap.cc b/src/test/func/pagemap/pagemap.cc index dc5d5af63..61a23f1dd 100644 --- a/src/test/func/pagemap/pagemap.cc +++ b/src/test/func/pagemap/pagemap.cc @@ -68,7 +68,8 @@ void test_pagemap(bool bounded) // Initialise the pagemap if (bounded) { - auto [base, size] = Pal::reserve_at_least(bits::one_at_bit(30)); + auto size = bits::one_at_bit(30); + auto base = Pal::reserve(size); Pal::notify_using(base, size); std::cout << "Fixed base: " << base << " (" << size << ") " << " end: " << pointer_offset(base, size) << std::endl; diff --git a/src/test/perf/external_pointer/externalpointer.cc b/src/test/perf/external_pointer/externalpointer.cc index b927daf7a..ac66a1350 100644 --- a/src/test/perf/external_pointer/externalpointer.cc +++ b/src/test/perf/external_pointer/externalpointer.cc @@ -32,6 +32,8 @@ namespace test size = 16; // store object objects[i] = (size_t*)alloc.alloc(size); + if (objects[i] == nullptr) + abort(); // Store allocators size for this object *objects[i] = alloc.alloc_size(objects[i]); } From 8a8669f9572329494ae9ab98cf69ebaf7f522b88 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 19 Jul 2021 11:25:42 +0100 Subject: [PATCH 016/302] Add randomised start to pagemap layout The pagemap contains a lot of important data. This commit makes the checked mode overallocate, and then start the pagemap at a random offset within this range. --- src/backend/pagemap.h | 29 +++++++++++++++++++++++++---- src/mem/entropy.h | 2 ++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/backend/pagemap.h b/src/backend/pagemap.h index f1cda4a7d..22126c8ea 100644 --- a/src/backend/pagemap.h +++ b/src/backend/pagemap.h @@ -2,6 +2,7 @@ #include "../ds/bits.h" #include "../ds/helpers.h" +#include "../mem/entropy.h" #include "../pal/pal.h" #include @@ -95,6 +96,8 @@ namespace snmalloc body = reinterpret_cast(b); // Advance by size of pagemap. + // Note that base needs to be aligned to GRANULARITY for the rest of the + // code to work // TODO CHERI capability bound here! heap_base = pointer_align_up( pointer_offset(b, (size >> SHIFT) * sizeof(T)), @@ -102,6 +105,7 @@ namespace snmalloc base = address_cast(heap_base); SNMALLOC_ASSERT( base == bits::align_up(base, bits::one_at_bit(GRANULARITY_BITS))); + return {heap_base, pointer_diff(heap_base, end)}; } @@ -117,15 +121,32 @@ namespace snmalloc bits::ADDRESS_BITS - GRANULARITY_BITS; static constexpr size_t ENTRIES = bits::one_at_bit(COVERED_BITS); - // TODO request additional space, and move to random offset. + static constexpr size_t REQUIRED_SIZE = ENTRIES * sizeof(T); - auto new_body_untyped = Pal::reserve(ENTRIES * sizeof(T)); +#ifdef SNMALLOC_CHECK_CLIENT + // Allocate a power of two extra to allow the placement of the + // pagemap be difficult to guess. + size_t additional_size = bits::next_pow2(REQUIRED_SIZE) * 2; + size_t request_size = REQUIRED_SIZE + additional_size; +#else + size_t request_size = REQUIRED_SIZE; +#endif - auto new_body = reinterpret_cast(new_body_untyped); + auto new_body_untyped = PAL::reserve(request_size); +#ifdef SNMALLOC_CHECK_CLIENT + // Begin pagemap at random offset within the additionally allocated space. + static_assert(bits::is_pow2(sizeof(T)), "Next line assumes this."); + size_t offset = get_entropy64() & (additional_size - sizeof(T)); + auto new_body = + reinterpret_cast(pointer_offset(new_body_untyped, offset)); +#else + auto new_body = reinterpret_cast(new_body_untyped); +#endif // Ensure bottom page is committed // ASSUME: new memory is zeroed. - Pal::notify_using(new_body, OS_PAGE_SIZE); + PAL::template notify_using( + pointer_align_down(new_body), OS_PAGE_SIZE); // Set up zero page new_body[0] = body[0]; diff --git a/src/mem/entropy.h b/src/mem/entropy.h index 80bd6f420..d9b40aea3 100644 --- a/src/mem/entropy.h +++ b/src/mem/entropy.h @@ -1,3 +1,5 @@ +#pragma once + #include "../ds/address.h" #include "../pal/pal.h" From 0b7929327a4af7e6d249975cc713c5ba900ff504 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 19 Jul 2021 15:45:51 +0100 Subject: [PATCH 017/302] Add errors for failing to initialise the system. --- src/backend/pagemap.h | 5 +++++ src/mem/pool.h | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/src/backend/pagemap.h b/src/backend/pagemap.h index 22126c8ea..4f4ab6e3d 100644 --- a/src/backend/pagemap.h +++ b/src/backend/pagemap.h @@ -134,6 +134,11 @@ namespace snmalloc auto new_body_untyped = PAL::reserve(request_size); + if (new_body_untyped == nullptr) + { + PAL::error("Failed to initialisation snmalloc."); + } + #ifdef SNMALLOC_CHECK_CLIENT // Begin pagemap at random offset within the additionally allocated space. static_assert(bits::is_pow2(sizeof(T)), "Next line assumes this."); diff --git a/src/mem/pool.h b/src/mem/pool.h index e57dbc5ab..d718c7994 100644 --- a/src/mem/pool.h +++ b/src/mem/pool.h @@ -51,6 +51,12 @@ namespace snmalloc p = ChunkAllocator::alloc_meta_data( h, nullptr, std::forward(args)...); + if (p == nullptr) + { + SharedStateHandle::Backend::Pal::error( + "Failed to initialisation thread local allocator."); + } + FlagLock f(pool.lock); p->list_next = pool.list; pool.list = p; From 01259b024851a60fa3dcdc4c5784b27a780f9c33 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 19 Jul 2021 15:47:54 +0100 Subject: [PATCH 018/302] Make tests single test at a time for Debug. --- ci/scripts/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/test.sh b/ci/scripts/test.sh index ac439158c..2e13e8131 100755 --- a/ci/scripts/test.sh +++ b/ci/scripts/test.sh @@ -4,7 +4,7 @@ set -eo pipefail cd build if [ $SELF_HOST = false ]; then - ctest -j 4 --output-on-failure -C $BUILD_TYPE + ctest -j 1 --output-on-failure -C $BUILD_TYPE else sudo cp libsnmallocshim.so libsnmallocshim-checks.so /usr/local/lib/ ninja clean From 6b53c29600ff943fb3cb1984a24d5553d8458c6f Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 21 Jul 2021 14:52:33 +0100 Subject: [PATCH 019/302] Fix typo. --- src/mem/pool.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mem/pool.h b/src/mem/pool.h index d718c7994..bedbce58e 100644 --- a/src/mem/pool.h +++ b/src/mem/pool.h @@ -54,7 +54,7 @@ namespace snmalloc if (p == nullptr) { SharedStateHandle::Backend::Pal::error( - "Failed to initialisation thread local allocator."); + "Failed to initialise thread local allocator."); } FlagLock f(pool.lock); From e1ef8665a8863e92b3aff72ff73ca4fac94487d2 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 21 Jul 2021 16:21:33 +0100 Subject: [PATCH 020/302] Add protection to meta-data The meta-data in the CHECK_CLIENT mode is allocated from its own areas, where most of the pages have been disabled, and the location within this range is randomised. This protects from trivial OOB read/write hitting the meta-data. More complex controlled offsets are also mitigated due to the sparse level of pages being turned on (i.e. not PROT_NONE). --- src/backend/backend.h | 147 +++++++++++++++++++++++++++++++++--------- 1 file changed, 117 insertions(+), 30 deletions(-) diff --git a/src/backend/backend.h b/src/backend/backend.h index 8893a4aa5..e17b044b7 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -14,6 +14,19 @@ namespace snmalloc template class BackendAllocator { + // Size of local address space requests. Currently aimed at 2MiB large + // pages but should make this configurable (i.e. for OE, so we don't need as + // much space). + constexpr static size_t LOCAL_CACHE_BLOCK = bits::one_at_bit(21); + +#ifdef SNMALLOC_CHECK_CLIENT + // When protecting the meta-data, we use a smaller block for the meta-data + // that is randomised inside a larger block. This needs to be at least a + // page so that we can use guard pages. + constexpr static size_t LOCAL_CACHE_META_BLOCK = + bits::max(MIN_CHUNK_SIZE * 2, OS_PAGE_SIZE); +#endif + public: using Pal = PAL; @@ -27,8 +40,15 @@ namespace snmalloc { friend BackendAllocator; - // TODO Separate meta data and object AddressSpaceManagerCore local_address_space; + +#ifdef SNMALLOC_CHECK_CLIENT + /** + * Secondary local address space, so we can apply some randomisation + * and guard pages to protect the meta-data. + */ + AddressSpaceManagerCore local_meta_address_space; +#endif }; /** @@ -44,7 +64,6 @@ namespace snmalloc { friend BackendAllocator; - // TODO Separate meta data and object AddressSpaceManager address_space; FlatPagemap pagemap; @@ -81,6 +100,34 @@ namespace snmalloc }; private: +#ifdef SNMALLOC_CHECK_CLIENT + /** + * Returns a sub-range of [return, return+sub_size] that is contained in + * the range [base, base+full_size]. The first and last slot are not used + * so that the edges can be used for guard pages. + */ + static CapPtr + sub_range(CapPtr base, size_t full_size, size_t sub_size) + { + SNMALLOC_ASSERT(bits::is_pow2(full_size)); + SNMALLOC_ASSERT(bits::is_pow2(sub_size)); + SNMALLOC_ASSERT(full_size % sub_size == 0); + SNMALLOC_ASSERT(full_size / sub_size >= 4); + + size_t offset_mask = full_size - sub_size; + + // Don't use first or last block in the larger reservation + // Loop required to get uniform distribution. + size_t offset; + do + { + offset = get_entropy64() & offset_mask; + } while ((offset == 0) || (offset == offset_mask)); + + return pointer_offset(base, offset); + } +#endif + /** * Internal method for acquiring state from the local and global address * space managers. @@ -89,41 +136,75 @@ namespace snmalloc static CapPtr reserve(GlobalState& h, LocalState* local_state, size_t size) { - // TODO have two address spaces. - UNUSED(is_meta); +#ifdef SNMALLOC_CHECK_CLIENT + constexpr auto MAX_CACHED_SIZE = + is_meta ? LOCAL_CACHE_META_BLOCK : LOCAL_CACHE_BLOCK; +#else + constexpr auto MAX_CACHED_SIZE = LOCAL_CACHE_BLOCK; +#endif + + auto& global = h.address_space; CapPtr p; - if (local_state != nullptr) + if ((local_state != nullptr) && (size <= MAX_CACHED_SIZE)) { - p = - local_state->local_address_space.template reserve_with_left_over( - size); +#ifdef SNMALLOC_CHECK_CLIENT + auto& local = is_meta ? local_state->local_meta_address_space : + local_state->local_address_space; +#else + auto& local = local_state->local_address_space; +#endif + + p = local.template reserve_with_left_over(size); if (p != nullptr) - local_state->local_address_space.template commit_block(p, size); - else { - auto& a = h.address_space; - // TODO Improve heuristics and params - auto refill_size = bits::max(size, bits::one_at_bit(21)); - auto refill = a.template reserve(refill_size, h.pagemap); - if (refill == nullptr) - return nullptr; - local_state->local_address_space.template add_range( - refill, refill_size); - // This should succeed - p = local_state->local_address_space - .template reserve_with_left_over(size); - if (p != nullptr) - local_state->local_address_space.template commit_block( - p, size); + return p; } + + auto refill_size = LOCAL_CACHE_BLOCK; + auto refill = global.template reserve(refill_size, h.pagemap); + if (refill == nullptr) + return nullptr; + +#ifdef SNMALLOC_CHECK_CLIENT + if (is_meta) + { + refill = sub_range(refill, LOCAL_CACHE_BLOCK, LOCAL_CACHE_META_BLOCK); + refill_size = LOCAL_CACHE_META_BLOCK; + } +#endif + PAL::template notify_using(refill.unsafe_ptr(), refill_size); + local.template add_range(refill, refill_size); + + // This should succeed + return local.template reserve_with_left_over(size); } - else + +#ifdef SNMALLOC_CHECK_CLIENT + // During start up we need meta-data before we have a local allocator + // This code protects that meta-data with randomisation, and guard pages. + if (local_state == nullptr && is_meta) { - auto& a = h.address_space; - p = a.template reserve_with_left_over(size, h.pagemap); + size_t rsize = bits::max(OS_PAGE_SIZE, bits::next_pow2(size)); + size_t size_request = rsize * 64; + + p = global.template reserve(size_request, h.pagemap); + if (p == nullptr) + return nullptr; + + p = sub_range(p, size_request, rsize); + + PAL::template notify_using(p.unsafe_ptr(), rsize); + return p; } + // This path does not apply any guard pages to very large + // meta data requests. There are currently no meta data-requests + // this large. This assert checks for this assumption breaking. + SNMALLOC_ASSERT(!is_meta); +#endif + + p = global.template reserve_with_left_over(size, h.pagemap); return p; } @@ -159,6 +240,12 @@ namespace snmalloc SNMALLOC_ASSERT(bits::is_pow2(size)); SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE); + auto meta = reinterpret_cast( + reserve(h, local_state, sizeof(Metaslab)).unsafe_ptr()); + + if (meta == nullptr) + return {nullptr, nullptr}; + CapPtr p = reserve(h, local_state, size); #ifdef SNMALLOC_TRACING @@ -167,15 +254,15 @@ namespace snmalloc #endif if (p == nullptr) { + // TODO: This is leaking `meta`. Currently there is no facility for + // meta-data reuse, so will leave until we develop more expressive + // meta-data management. #ifdef SNMALLOC_TRACING std::cout << "Out of memory" << std::endl; #endif return {p, nullptr}; } - auto meta = reinterpret_cast( - reserve(h, local_state, sizeof(Metaslab)).unsafe_ptr()); - MetaEntry t(meta, remote, sizeclass); for (address_t a = address_cast(p); From 529b90b01b9aaec184fa105ff3777fdcd1eafe98 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 22 Jul 2021 09:51:49 +0100 Subject: [PATCH 021/302] Default some statistics --- src/mem/metaslab.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index c8194dfcc..d62db6ace 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -204,8 +204,8 @@ namespace snmalloc struct MetaslabCache : public CDLLNode<> { - uint16_t unused; - uint16_t length; + uint16_t unused = 0; + uint16_t length = 0; }; } // namespace snmalloc From d965ab2645bfa1324ec1fee7f4808f65f2574351 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 22 Jul 2021 09:53:19 +0100 Subject: [PATCH 022/302] Print stack traces for segfault Trap the segfault signal, and call the platform error. In most cases for CI this will print a stack trace. --- src/test/setup.h | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/src/test/setup.h b/src/test/setup.h index 20936cca7..bcf303561 100644 --- a/src/test/setup.h +++ b/src/test/setup.h @@ -1,12 +1,13 @@ -#if defined(WIN32) && defined(SNMALLOC_CI_BUILD) -# include -# include -# include -# include -# include +#if defined(SNMALLOC_CI_BUILD) +# if defined(WIN32) +# include +# include +# include +# include +# include // Has to come after the PAL. -# include -# pragma comment(lib, "dbghelp.lib") +# include +# pragma comment(lib, "dbghelp.lib") void print_stack_trace() { @@ -94,6 +95,20 @@ void setup() // Disable OS level dialog boxes during CI. SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX); } +# else +# include +# include +void error_handle(int signal) +{ + UNUSED(signal); + snmalloc::error("Seg Fault"); + _exit(1); +} +void setup() +{ + signal(SIGSEGV, error_handle); +} +# endif #else void setup() {} #endif From d6bae72b20105bf131002484c2f61e118c9e5199 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 22 Jul 2021 09:53:47 +0100 Subject: [PATCH 023/302] Improve debugging for fixed_region --- src/test/func/fixed_region/fixed_region.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/test/func/fixed_region/fixed_region.cc b/src/test/func/fixed_region/fixed_region.cc index b2350a9ae..ed5ca3d1b 100644 --- a/src/test/func/fixed_region/fixed_region.cc +++ b/src/test/func/fixed_region/fixed_region.cc @@ -1,5 +1,6 @@ #include "mem/fixedglobalconfig.h" #include "mem/globalconfig.h" +#include "test/setup.h" #include #include @@ -17,6 +18,7 @@ using FixedAlloc = LocalAllocator; int main() { #ifndef SNMALLOC_PASS_THROUGH // Depends on snmalloc specific features + setup(); // 28 is large enough to produce a nested allocator. // It is also large enough for the example to run in. @@ -34,11 +36,18 @@ int main() size_t object_size = 128; size_t count = 0; + size_t i = 0; while (true) { auto r1 = a.alloc(object_size); count += object_size; + i++; + if (i == 1024) + { + i = 0; + std::cout << "."; + } // Run until we exhaust the fixed region. // This should return null. if (r1 == nullptr) From 0cfa8f2cff371087e2d4a547c34532dbe65dddea Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 22 Jul 2021 19:13:00 +0100 Subject: [PATCH 024/302] Remove globalconfig.h includes. --- src/mem/threadalloc.h | 1 - src/override/malloc.cc | 1 + src/snmalloc.h | 3 +++ src/snmalloc_core.h | 3 +-- src/test/func/fixed_region/fixed_region.cc | 1 - src/test/func/thread_alloc_external/thread_alloc_external.cc | 3 +++ 6 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/mem/threadalloc.h b/src/mem/threadalloc.h index f2b6f4bd6..101e09230 100644 --- a/src/mem/threadalloc.h +++ b/src/mem/threadalloc.h @@ -1,7 +1,6 @@ #pragma once #include "../ds/helpers.h" -#include "globalconfig.h" #include "localalloc.h" #if defined(SNMALLOC_EXTERNAL_THREAD_ALLOC) diff --git a/src/override/malloc.cc b/src/override/malloc.cc index a04004268..acd640276 100644 --- a/src/override/malloc.cc +++ b/src/override/malloc.cc @@ -2,6 +2,7 @@ #include "../snmalloc_core.h" #ifndef SNMALLOC_PROVIDE_OWN_CONFIG +# include "../mem/globalconfig.h" // The default configuration for snmalloc is used if alternative not defined namespace snmalloc { diff --git a/src/snmalloc.h b/src/snmalloc.h index 87f3b5ff8..6abd703d6 100644 --- a/src/snmalloc.h +++ b/src/snmalloc.h @@ -3,6 +3,9 @@ // Core implementation of snmalloc independent of the configuration mode #include "snmalloc_core.h" +// Default implementation of global state +#include "mem/globalconfig.h" + // The default configuration for snmalloc namespace snmalloc { diff --git a/src/snmalloc_core.h b/src/snmalloc_core.h index 231d79be9..0c6a2d336 100644 --- a/src/snmalloc_core.h +++ b/src/snmalloc_core.h @@ -1,4 +1,3 @@ #pragma once -#include "mem/globalalloc.h" -#include "mem/globalconfig.h" \ No newline at end of file +#include "mem/globalalloc.h" \ No newline at end of file diff --git a/src/test/func/fixed_region/fixed_region.cc b/src/test/func/fixed_region/fixed_region.cc index ed5ca3d1b..1f0ca0bc6 100644 --- a/src/test/func/fixed_region/fixed_region.cc +++ b/src/test/func/fixed_region/fixed_region.cc @@ -1,5 +1,4 @@ #include "mem/fixedglobalconfig.h" -#include "mem/globalconfig.h" #include "test/setup.h" #include diff --git a/src/test/func/thread_alloc_external/thread_alloc_external.cc b/src/test/func/thread_alloc_external/thread_alloc_external.cc index 16fed068c..a5b1fe8d2 100644 --- a/src/test/func/thread_alloc_external/thread_alloc_external.cc +++ b/src/test/func/thread_alloc_external/thread_alloc_external.cc @@ -3,6 +3,9 @@ // Specify using own #define SNMALLOC_EXTERNAL_THREAD_ALLOC + +#include "mem/globalconfig.h" + namespace snmalloc { using Alloc = snmalloc::LocalAllocator; From 84a5fb9450b310551b2b86f12628e36f1aaf1aa1 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Fri, 23 Jul 2021 13:12:22 +0100 Subject: [PATCH 025/302] Correct REMOTE_MIN_ALIGN Take the maximum of... * CACHELINE_SIZE (for performance) * next_pow2(NUM_SIZECLASSES + 1) so that, when the pagemap points to a Remote, the (small) size class stuffed in the bottom bits can be removed by alignment * next_pow2(NUM_LARGE_CLASSES + 1) so that, when the pagemap isn't pointing to a Remote, when the associated chunk is (part of) a large allocation, aligning the Remote* results in 0. The last of these conditions will almost never be the deciding factor, as there are generally many more small size classes than large ones, but it shouldn't hurt to be safe. --- src/mem/remoteallocator.h | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index d3c1a738b..5b70ac1ef 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -10,10 +10,19 @@ namespace snmalloc { - // Remotes need to be aligned enough that all the - // small size classes can fit in the bottom bits. - static constexpr size_t REMOTE_MIN_ALIGN = bits::min( - CACHELINE_SIZE, bits::next_pow2_const(NUM_SIZECLASSES + 1)); + // Remotes need to be aligned enough that the bottom bits have enough room for + // all the size classes, both large and small. + // + // Including large classes in this calculation might seem remarkably strange, + // since large allocations don't have associated Remotes, that is, their + // remote is taken to be 0. However, if there are very few small size + // classes and many large classes, the attempt to align that 0 down by the + // alignment of a Remote might result in a nonzero value. + static constexpr size_t REMOTE_MIN_ALIGN = bits::max( + CACHELINE_SIZE, + bits::max( + bits::next_pow2_const(NUM_SIZECLASSES + 1), + bits::next_pow2_const(NUM_LARGE_CLASSES + 1))); /** * Global key for all remote lists. From b69505fc5db849fae03253d206e533284d4f1a64 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 23 Jul 2021 15:09:30 +0100 Subject: [PATCH 026/302] Make address space manager use pagemap for next pointers (#356) * Make address space manager use pagemap for next pointers The address space manager uses the pagemap entry to form linked lists of unused address space above MIN_CHUNK_SIZE. It continues to use references in the block below that threshold. In the CHECK_CLIENT mode this makes it hard to corrupt the ASM as only meta-data uses allocations below MIN_CHUNK_SIZE from the ASM. This allocations will be protected with guard pages by the backend. * address_space_core: use FreeChunk struct Purely stylistic, NFCI. This hides some somewhat gnarly reinterpret_cast-s in favor of more, but hopefully less gnarly, casts elsewhere. * Apply suggestions from code review Co-authored-by: Nathaniel Wesley Filardo --- src/backend/address_space.h | 14 ++- src/backend/address_space_core.h | 192 +++++++++++++++++++------------ src/backend/backend.h | 9 +- 3 files changed, 133 insertions(+), 82 deletions(-) diff --git a/src/backend/address_space.h b/src/backend/address_space.h index e8d148351..abccd83c2 100644 --- a/src/backend/address_space.h +++ b/src/backend/address_space.h @@ -71,7 +71,7 @@ namespace snmalloc CapPtr res; { FlagLock lock(spin_lock); - res = core.template reserve(size); + res = core.template reserve(size, pagemap); if (res == nullptr) { // Allocation failed ask OS for more memory @@ -130,10 +130,10 @@ namespace snmalloc pagemap.register_range(address_cast(block), block_size); - core.template add_range(block, block_size); + core.template add_range(block, block_size, pagemap); // still holding lock so guaranteed to succeed. - res = core.template reserve(size); + res = core.template reserve(size, pagemap); } } @@ -167,7 +167,8 @@ namespace snmalloc if (rsize > size) { FlagLock lock(spin_lock); - core.template add_range(pointer_offset(res, size), rsize - size); + core.template add_range( + pointer_offset(res, size), rsize - size, pagemap); } if constexpr (committed) @@ -187,10 +188,11 @@ namespace snmalloc * Add a range of memory to the address space. * Divides blocks into power of two sizes with natural alignment */ - void add_range(CapPtr base, size_t length) + template + void add_range(CapPtr base, size_t length, Pagemap& pagemap) { FlagLock lock(spin_lock); - core.add_range(base, length); + core.add_range(base, length, pagemap); } }; } // namespace snmalloc diff --git a/src/backend/address_space_core.h b/src/backend/address_space_core.h index 6f0ccc5d6..549c317dd 100644 --- a/src/backend/address_space_core.h +++ b/src/backend/address_space_core.h @@ -11,9 +11,6 @@ namespace snmalloc { /** - * TODO all comment in this file need revisiting. Core versus locking global - * version. - * * Implements a power of two allocator, where all blocks are aligned to the * same power of two as their size. This is what snmalloc uses to get * alignment of very large sizeclasses. @@ -23,33 +20,36 @@ namespace snmalloc */ class AddressSpaceManagerCore { + struct FreeChunk + { + CapPtr next; + }; + /** * Stores the blocks of address space * - * The first level of array indexes based on power of two size. - * - * The first entry ranges[n][0] is just a pointer to an address range - * of size 2^n. - * - * The second entry ranges[n][1] is a pointer to a linked list of blocks - * of this size. The final block in the list is not committed, so we commit - * on pop for this corner case. + * The array indexes based on power of two size. * - * Invariants - * ranges[n][1] != nullptr => ranges[n][0] != nullptr + * The entries for each size form a linked list. For sizes below + * MIN_CHUNK_SIZE they are linked through the first location in the + * block of memory. For sizes of, and above, MIN_CHUNK_SIZE they are + * linked using the pagemap. We only use the smaller than MIN_CHUNK_SIZE + * allocations for meta-data, so we can be sure that the next pointers + * never occur in a blocks that are ultimately used for object allocations. * * bits::BITS is used for simplicity, we do not use below the pointer size, * and large entries will be unlikely to be supported by the platform. */ - std::array, 2>, bits::BITS> ranges = {}; + std::array, bits::BITS> ranges = {}; /** * Checks a block satisfies its invariant. */ - inline void check_block(CapPtr base, size_t align_bits) + inline void check_block(CapPtr base, size_t align_bits) { SNMALLOC_ASSERT( - base == pointer_align_up(base, bits::one_at_bit(align_bits))); + address_cast(base) == + bits::align_up(address_cast(base), bits::one_at_bit(align_bits))); // All blocks need to be bigger than a pointer. SNMALLOC_ASSERT(bits::one_at_bit(align_bits) >= sizeof(void*)); UNUSED(base); @@ -57,45 +57,83 @@ namespace snmalloc } /** - * Adds a block to `ranges`. + * Set next pointer for a power of two address range. + * + * This abstracts the use of either + * - the pagemap; or + * - the first pointer word of the block + * to store the next pointer for the list of unused address space of a + * particular size. */ - template - void add_block(size_t align_bits, CapPtr base) + template + void set_next( + size_t align_bits, + CapPtr base, + CapPtr next, + Pagemap& pagemap) { - check_block(base, align_bits); - SNMALLOC_ASSERT(align_bits < 64); - if (ranges[align_bits][0] == nullptr) + if (align_bits >= MIN_CHUNK_BITS) { - // Prefer first slot if available. - ranges[align_bits][0] = base; + // The pagemap stores MetaEntrys, abuse the metaslab field to be the + // next block in the stack of blocks. + // + // The pagemap entries here have nullptr (i.e., fake_large_remote) as + // their remote, and so other accesses to the pagemap (by + // external_pointer, for example) will not attempt to follow this + // "Metaslab" pointer. + MetaEntry t(reinterpret_cast(next.unsafe_ptr()), nullptr, 0); + pagemap.set(address_cast(base), t); return; } - if (ranges[align_bits][1] != nullptr) + base->next = next; + } + + /** + * Get next pointer for a power of two address range. + * + * This abstracts the use of either + * - the pagemap; or + * - the first pointer word of the block + * to store the next pointer for the list of unused address space of a + * particular size. + */ + template + CapPtr get_next( + size_t align_bits, CapPtr base, Pagemap& pagemap) + { + if (align_bits >= MIN_CHUNK_BITS) { -#ifdef SNMALLOC_TRACING - std::cout << "Add range linking." << std::endl; -#endif - // Add to linked list. - commit_block(base, sizeof(void*)); - *(base.template as_static>().unsafe_ptr()) = - ranges[align_bits][1]; - check_block(ranges[align_bits][1], align_bits); + const MetaEntry& t = pagemap.template get(address_cast(base)); + return CapPtr( + reinterpret_cast(t.get_metaslab())); } - // Update head of list - ranges[align_bits][1] = base; - check_block(ranges[align_bits][1], align_bits); + return base->next; + } + + /** + * Adds a block to `ranges`. + */ + template + void add_block( + size_t align_bits, CapPtr base, Pagemap& pagemap) + { + check_block(base, align_bits); + SNMALLOC_ASSERT(align_bits < 64); + + set_next(align_bits, base, ranges[align_bits], pagemap); + ranges[align_bits] = base.as_static(); } /** * Find a block of the correct size. May split larger blocks * to satisfy this request. */ - template - CapPtr remove_block(size_t align_bits) + template + CapPtr remove_block(size_t align_bits, Pagemap& pagemap) { - CapPtr first = ranges[align_bits][0]; + CapPtr first = ranges[align_bits]; if (first == nullptr) { if (align_bits == (bits::BITS - 1)) @@ -105,37 +143,34 @@ namespace snmalloc } // Look for larger block and split up recursively - CapPtr bigger = remove_block(align_bits + 1); + CapPtr bigger = + remove_block(align_bits + 1, pagemap); if (bigger != nullptr) { + // This block is going to be broken up into sub CHUNK_SIZE blocks + // so we need to commit it to enable the next pointers to be used + // inside the block. + if ((align_bits + 1) == MIN_CHUNK_BITS) + { + commit_block(bigger, MIN_CHUNK_SIZE); + } + size_t left_over_size = bits::one_at_bit(align_bits); auto left_over = pointer_offset(bigger, left_over_size); - ranges[align_bits][0] = - Aal::capptr_bound(left_over, left_over_size); - check_block(left_over, align_bits); + + add_block( + align_bits, + Aal::capptr_bound(left_over, left_over_size), + pagemap); + check_block(left_over.as_static(), align_bits); } - check_block(bigger, align_bits + 1); + check_block(bigger.as_static(), align_bits + 1); return bigger; } - CapPtr second = ranges[align_bits][1]; - if (second != nullptr) - { - commit_block(second, sizeof(void*)); - auto psecond = - second.template as_static>().unsafe_ptr(); - auto next = *psecond; - ranges[align_bits][1] = next; - // Zero memory. Client assumes memory contains only zeros. - *psecond = nullptr; - check_block(second, align_bits); - check_block(next, align_bits); - return second; - } - check_block(first, align_bits); - ranges[align_bits][0] = nullptr; - return first; + ranges[align_bits] = get_next(align_bits, first, pagemap); + return first.as_void(); } public: @@ -143,9 +178,21 @@ namespace snmalloc * Add a range of memory to the address space. * Divides blocks into power of two sizes with natural alignment */ - template - void add_range(CapPtr base, size_t length) + template + void add_range(CapPtr base, size_t length, Pagemap& pagemap) { + // For start and end that are not chunk sized, we need to + // commit the pages to track the allocations. + auto base_chunk = pointer_align_up(base, MIN_CHUNK_SIZE); + auto end = pointer_offset(base, length); + auto end_chunk = pointer_align_down(end, MIN_CHUNK_SIZE); + auto start_length = pointer_diff(base, base_chunk); + auto end_length = pointer_diff(end_chunk, end); + if (start_length != 0) + commit_block(base, start_length); + if (end_length != 0) + commit_block(end_chunk, end_length); + // Find the minimum set of maximally aligned blocks in this range. // Each block's alignment and size are equal. while (length >= sizeof(void*)) @@ -154,9 +201,10 @@ namespace snmalloc size_t length_align_bits = (bits::BITS - 1) - bits::clz(length); size_t align_bits = bits::min(base_align_bits, length_align_bits); size_t align = bits::one_at_bit(align_bits); + auto b = base.as_static(); - check_block(base, align_bits); - add_block(align_bits, base); + check_block(b, align_bits); + add_block(align_bits, b, pagemap); base = pointer_offset(base, align); length -= align; @@ -188,8 +236,8 @@ namespace snmalloc * part of satisfying the request will be registered with the provided * arena_map for use in subsequent amplification. */ - template - CapPtr reserve(size_t size) + template + CapPtr reserve(size_t size, Pagemap& pagemap) { #ifdef SNMALLOC_TRACING std::cout << "ASM Core reserve request:" << size << std::endl; @@ -198,7 +246,7 @@ namespace snmalloc SNMALLOC_ASSERT(bits::is_pow2(size)); SNMALLOC_ASSERT(size >= sizeof(void*)); - return remove_block(bits::next_pow2_bits(size)); + return remove_block(bits::next_pow2_bits(size), pagemap); } /** @@ -208,8 +256,8 @@ namespace snmalloc * This is useful for allowing the space required for alignment to be * used, by smaller objects. */ - template - CapPtr reserve_with_left_over(size_t size) + template + CapPtr reserve_with_left_over(size_t size, Pagemap& pagemap) { SNMALLOC_ASSERT(size >= sizeof(void*)); @@ -217,13 +265,13 @@ namespace snmalloc size_t rsize = bits::next_pow2(size); - auto res = reserve(rsize); + auto res = reserve(rsize, pagemap); if (res != nullptr) { if (rsize > size) { - add_range(pointer_offset(res, size), rsize - size); + add_range(pointer_offset(res, size), rsize - size, pagemap); } } return res; diff --git a/src/backend/backend.h b/src/backend/backend.h index e17b044b7..cab9944ea 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -90,7 +90,8 @@ namespace snmalloc fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); auto [heap_base, heap_length] = pagemap.init(base, length); - address_space.add_range(CapPtr(heap_base), heap_length); + address_space.add_range( + CapPtr(heap_base), heap_length, pagemap); if constexpr (!fixed_range) { @@ -155,7 +156,7 @@ namespace snmalloc auto& local = local_state->local_address_space; #endif - p = local.template reserve_with_left_over(size); + p = local.template reserve_with_left_over(size, h.pagemap); if (p != nullptr) { return p; @@ -174,10 +175,10 @@ namespace snmalloc } #endif PAL::template notify_using(refill.unsafe_ptr(), refill_size); - local.template add_range(refill, refill_size); + local.template add_range(refill, refill_size, h.pagemap); // This should succeed - return local.template reserve_with_left_over(size); + return local.template reserve_with_left_over(size, h.pagemap); } #ifdef SNMALLOC_CHECK_CLIENT From 81bf3417327d4ee2e39037bd5af7365942c2eacd Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 26 Jul 2021 09:56:48 +0100 Subject: [PATCH 027/302] XOR encoded next_object This commit adds a simple XOR encoding to the next_object pointer in FreeObjects. This removes the trivial way of getting hold of a physical address from the system by observing the free list pointers in deallocated objects. --- src/ds/defines.h | 11 +++- src/mem/corealloc.h | 8 +-- src/mem/entropy.h | 31 +++++++--- src/mem/freelist.h | 123 +++++++++++++++++++++++++------------- src/mem/globalconfig.h | 6 +- src/mem/localcache.h | 4 +- src/mem/metaslab.h | 2 +- src/mem/remoteallocator.h | 10 ++-- src/mem/remotecache.h | 6 +- 9 files changed, 136 insertions(+), 65 deletions(-) diff --git a/src/ds/defines.h b/src/ds/defines.h index f5d6646f7..37095f8a1 100644 --- a/src/ds/defines.h +++ b/src/ds/defines.h @@ -130,4 +130,13 @@ check_client_impl(bool test, const char* const str) # define check_client(test, str) check_client_impl(test, str) #else # define check_client(test, str) -#endif \ No newline at end of file +#endif + +namespace snmalloc +{ +#ifdef SNMALLOC_CHECK_CLIENT + static constexpr bool CHECK_CLIENT = true; +#else + static constexpr bool CHECK_CLIENT = false; +#endif +} // namespace snmalloc diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index fb4ff91f8..96490a763 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -141,7 +141,7 @@ namespace snmalloc { auto slab_end = pointer_offset(bumpptr, slab_size + 1 - rsize); - FreeListKey key(entropy.get_constant_key()); + auto& key = entropy.get_free_list_key(); FreeListBuilder b; SNMALLOC_ASSERT(b.empty()); @@ -209,7 +209,7 @@ namespace snmalloc ChunkRecord* clear_slab(Metaslab* meta, sizeclass_t sizeclass) { - FreeListKey key(entropy.get_constant_key()); + auto& key = entropy.get_free_list_key(); FreeListIter fl; meta->free_queue.close(fl, key); void* p = finish_alloc_no_zero(fl.take(key), sizeclass); @@ -475,7 +475,7 @@ namespace snmalloc auto cp = CapPtr(reinterpret_cast(p)); - FreeListKey key(entropy.get_constant_key()); + auto& key = entropy.get_free_list_key(); // Update the head and the next pointer in the free list. meta->free_queue.add(cp, key, entropy); @@ -538,7 +538,7 @@ namespace snmalloc // Set meta slab to empty. meta->initialise(sizeclass); - FreeListKey key(entropy.get_constant_key()); + auto& key = entropy.get_free_list_key(); // take an allocation from the free list auto p = fast_free_list.take(key); diff --git a/src/mem/entropy.h b/src/mem/entropy.h index cf5f179f7..9a56cb289 100644 --- a/src/mem/entropy.h +++ b/src/mem/entropy.h @@ -29,14 +29,25 @@ namespace snmalloc #endif } + struct FreeListKey + { + address_t key; + address_t key_next; + + constexpr FreeListKey(uint64_t key, uint64_t key_next) + : key(static_cast(key)), + key_next(static_cast(key_next)) + {} + }; + class LocalEntropy { uint64_t bit_source{0}; uint64_t local_key{0}; uint64_t local_counter{0}; - address_t constant_key{0}; uint64_t fresh_bits{0}; uint64_t count{0}; + FreeListKey key{0, 0}; public: constexpr LocalEntropy() = default; @@ -47,9 +58,15 @@ namespace snmalloc local_key = get_entropy64(); local_counter = get_entropy64(); if constexpr (bits::BITS == 64) - constant_key = get_next(); + { + key.key = get_next(); + key.key_next = get_next(); + } else - constant_key = get_next() & 0xffff'ffff; + { + key.key = get_next() & 0xffff'ffff; + key.key_next = get_next() & 0xffff'ffff; + } bit_source = get_next(); } @@ -68,13 +85,11 @@ namespace snmalloc } /** - * A key that is not changed or used to create other keys - * - * This is for use when there is no storage for the key. + * A key for the free lists for this thread. */ - address_t get_constant_key() + const FreeListKey& get_free_list_key() { - return constant_key; + return key; } /** diff --git a/src/mem/freelist.h b/src/mem/freelist.h index e6a173b16..31500bb8d 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -41,19 +41,6 @@ namespace snmalloc { - struct FreeListKey - { - address_t key; - - FreeListKey(uint64_t key_) - { - if constexpr (bits::BITS == 64) - key = static_cast(key_); - else - key = key_ & 0xffff'ffff; - } - }; - /** * This function is used to sign back pointers in the free list. * @@ -64,7 +51,7 @@ namespace snmalloc * list. */ inline static uintptr_t - signed_prev(address_t curr, address_t next, FreeListKey& key) + signed_prev(address_t curr, address_t next, const FreeListKey& key) { auto c = curr; auto n = next; @@ -103,6 +90,30 @@ namespace snmalloc return p.template as_static(); } + /** + * Encode next + */ + inline static CapPtr encode_next( + address_t curr, CapPtr next, const FreeListKey& key) + { + // Note we can consider other encoding schemes here. + // * XORing curr and next. This doesn't require any key material + // * XORing (curr * key). This makes it harder to guess the underlying + // key, as each location effectively has its own key. + // Curr is not used in the current encoding scheme. + UNUSED(curr); + + if constexpr (CHECK_CLIENT && !aal_supports) + { + return CapPtr(address_cast(next) ^ key.key_next); + } + else + { + UNUSED(key); + return next; + } + } + /** * Assign next_object and update its prev_encoded if SNMALLOC_CHECK_CLIENT. * Static so that it can be used on reference to a FreeObject. @@ -114,7 +125,7 @@ namespace snmalloc static CapPtr* store_next( CapPtr* curr, CapPtr next, - FreeListKey& key) + const FreeListKey& key) { #ifdef SNMALLOC_CHECK_CLIENT next->prev_encoded = @@ -122,16 +133,23 @@ namespace snmalloc #else UNUSED(key); #endif - *curr = next; + *curr = encode_next(address_cast(curr), next, key); return &(next->next_object); } + static void + store_null(CapPtr* curr, const FreeListKey& key) + { + *curr = encode_next(address_cast(curr), nullptr, key); + } + /** * Assign next_object and update its prev_encoded if SNMALLOC_CHECK_CLIENT * * Uses the atomic view of next, so can be used in the message queues. */ - void atomic_store_next(CapPtr next, FreeListKey& key) + void + atomic_store_next(CapPtr next, const FreeListKey& key) { #ifdef SNMALLOC_CHECK_CLIENT next->prev_encoded = @@ -141,17 +159,24 @@ namespace snmalloc #endif // Signature needs to be visible before item is linked in // so requires release semantics. - atomic_next_object.store(next, std::memory_order_release); + atomic_next_object.store( + encode_next(address_cast(&next_object), next, key), + std::memory_order_release); } - void atomic_store_null() + void atomic_store_null(const FreeListKey& key) { - atomic_next_object.store(nullptr, std::memory_order_relaxed); + atomic_next_object.store( + encode_next(address_cast(&next_object), nullptr, key), + std::memory_order_relaxed); } - CapPtr atomic_read_next(FreeListKey& key) + CapPtr atomic_read_next(const FreeListKey& key) { - auto n = atomic_next_object.load(std::memory_order_acquire); + auto n = encode_next( + address_cast(&next_object), + atomic_next_object.load(std::memory_order_acquire), + key); #ifdef SNMALLOC_CHECK_CLIENT if (n != nullptr) { @@ -176,9 +201,9 @@ namespace snmalloc /** * Read the next pointer */ - CapPtr read_next() + CapPtr read_next(const FreeListKey& key) { - return next_object; + return encode_next(address_cast(&next_object), next_object, key); } }; @@ -230,10 +255,10 @@ namespace snmalloc /** * Moves the iterator on, and returns the current value. */ - CapPtr take(FreeListKey& key) + CapPtr take(const FreeListKey& key) { auto c = curr; - auto next = curr->read_next(); + auto next = curr->read_next(key); Aal::prefetch(next.unsafe_ptr()); curr = next; @@ -306,8 +331,10 @@ namespace snmalloc /** * Adds an element to the builder */ - void - add(CapPtr n, FreeListKey& key, LocalEntropy& entropy) + void add( + CapPtr n, + const FreeListKey& key, + LocalEntropy& entropy) { uint32_t index; if constexpr (RANDOM) @@ -328,7 +355,7 @@ namespace snmalloc */ template std::enable_if_t - add(CapPtr n, FreeListKey& key) + add(CapPtr n, const FreeListKey& key) { static_assert(RANDOM_ == RANDOM, "Don't set template parameter"); end[0] = FreeObject::store_next(end[0], n, key); @@ -337,23 +364,39 @@ namespace snmalloc /** * Makes a terminator to a free list. */ - SNMALLOC_FAST_PATH void terminate_list(uint32_t index, FreeListKey& key) + SNMALLOC_FAST_PATH void + terminate_list(uint32_t index, const FreeListKey& key) { - UNUSED(key); - *end[index] = nullptr; + FreeObject::store_null(end[index], key); + } + + /** + * Read head removing potential encoding + * + * Although, head does not require meta-data protection + * as it is not stored in an object allocation. For uniformity + * it is treated like the next_object field in a FreeObject + * and is thus subject to encoding if the next_object pointers + * encoded. + */ + CapPtr + read_head(uint32_t index, const FreeListKey& key) + { + return FreeObject::encode_next( + address_cast(&head[index]), head[index], key); } - address_t get_fake_signed_prev(uint32_t index, FreeListKey& key) + address_t get_fake_signed_prev(uint32_t index, const FreeListKey& key) { return signed_prev( - address_cast(&head[index]), address_cast(head[index]), key); + address_cast(&head[index]), address_cast(read_head(index, key)), key); } /** * Close a free list, and set the iterator parameter * to iterate it. */ - SNMALLOC_FAST_PATH void close(FreeListIter& fl, FreeListKey& key) + SNMALLOC_FAST_PATH void close(FreeListIter& fl, const FreeListKey& key) { if constexpr (RANDOM) { @@ -365,12 +408,12 @@ namespace snmalloc { // The start token has been corrupted. // TOCTTOU issue, but small window here. - head[1]->check_prev(get_fake_signed_prev(1, key)); + read_head(1, key)->check_prev(get_fake_signed_prev(1, key)); terminate_list(1, key); // Append 1 to 0 - FreeObject::store_next(end[0], head[1], key); + FreeObject::store_next(end[0], read_head(1, key), key); SNMALLOC_ASSERT(end[1] != &head[0]); SNMALLOC_ASSERT(end[0] != &head[1]); @@ -385,7 +428,7 @@ namespace snmalloc terminate_list(0, key); } - fl = {head[0], get_fake_signed_prev(0, key)}; + fl = {read_head(0, key), get_fake_signed_prev(0, key)}; init(); } @@ -401,12 +444,12 @@ namespace snmalloc } std::pair, CapPtr> - extract_segment() + extract_segment(const FreeListKey& key) { SNMALLOC_ASSERT(!empty()); SNMALLOC_ASSERT(!RANDOM); // TODO: Turn this into a static failure. - auto first = head[0]; + auto first = read_head(0, key); // end[0] is pointing to the first field in the object, // this is doing a CONTAINING_RECORD like cast to get back // to the actual object. This isn't true if the builder is diff --git a/src/mem/globalconfig.h b/src/mem/globalconfig.h index 981f09a6a..cc5dc76b1 100644 --- a/src/mem/globalconfig.h +++ b/src/mem/globalconfig.h @@ -6,6 +6,8 @@ #include "../mem/slaballocator.h" #include "commonconfig.h" +#include + namespace snmalloc { // Forward reference to thread local cleanup. @@ -73,8 +75,10 @@ namespace snmalloc if (initialised) return; + LocalEntropy entropy; + entropy.init(); // Initialise key for remote deallocation lists - key_global = FreeListKey(get_entropy64()); + key_global = FreeListKey(entropy.get_free_list_key()); // Need to initialise pagemap. backend_state.init(); diff --git a/src/mem/localcache.h b/src/mem/localcache.h index 20709668e..ae871c989 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -76,7 +76,7 @@ namespace snmalloc typename SharedStateHandle> bool flush(DeallocFun dealloc, SharedStateHandle handle) { - FreeListKey key(entropy.get_constant_key()); + auto& key = entropy.get_free_list_key(); // Return all the free lists to the allocator. // Used during thread teardown @@ -98,7 +98,7 @@ namespace snmalloc template SNMALLOC_FAST_PATH void* alloc(size_t size, Slowpath slowpath) { - FreeListKey key(entropy.get_constant_key()); + auto& key = entropy.get_free_list_key(); sizeclass_t sizeclass = size_to_sizeclass(size); stats.alloc_request(size); diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index d62db6ace..2c8852850 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -142,7 +142,7 @@ namespace snmalloc LocalEntropy& entropy, sizeclass_t sizeclass) { - FreeListKey key(entropy.get_constant_key()); + auto& key = entropy.get_free_list_key(); FreeListIter tmp_fl; meta->free_queue.close(tmp_fl, key); diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index 5b70ac1ef..6e8ea4735 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -27,7 +27,7 @@ namespace snmalloc /** * Global key for all remote lists. */ - inline static FreeListKey key_global(0xdeadbeef); + inline static FreeListKey key_global(0xdeadbeef, 0xdeadbeef); struct alignas(REMOTE_MIN_ALIGN) RemoteAllocator { @@ -50,7 +50,7 @@ namespace snmalloc void init(CapPtr stub) { - stub->atomic_store_null(); + stub->atomic_store_null(key_global); front = stub; back.store(stub, std::memory_order_relaxed); invariant(); @@ -74,12 +74,12 @@ namespace snmalloc void enqueue( CapPtr first, CapPtr last, - FreeListKey& key) + const FreeListKey& key) { // Pushes a list of messages to the queue. Each message from first to // last should be linked together through their next pointers. invariant(); - last->atomic_store_null(); + last->atomic_store_null(key); // exchange needs to be a release, so nullptr in next is visible. CapPtr prev = @@ -93,7 +93,7 @@ namespace snmalloc return front; } - std::pair, bool> dequeue(FreeListKey& key) + std::pair, bool> dequeue(const FreeListKey& key) { // Returns the front message, or null if not possible to return a message. invariant(); diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index af71c2205..25680b07a 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -67,7 +67,7 @@ namespace snmalloc SNMALLOC_FAST_PATH void dealloc( RemoteAllocator::alloc_id_t target_id, CapPtr p, - FreeListKey& key) + const FreeListKey& key) { SNMALLOC_ASSERT(initialised); auto r = p.template as_reinterpret(); @@ -79,7 +79,7 @@ namespace snmalloc bool post( SharedStateHandle handle, RemoteAllocator::alloc_id_t id, - FreeListKey& key) + const FreeListKey& key) { SNMALLOC_ASSERT(initialised); size_t post_round = 0; @@ -96,7 +96,7 @@ namespace snmalloc if (!list[i].empty()) { - auto [first, last] = list[i].extract_segment(); + auto [first, last] = list[i].extract_segment(key); MetaEntry entry = SharedStateHandle::Backend::get_meta_data( handle.get_backend_state(), address_cast(first)); entry.get_remote()->enqueue(first, last, key); From 2ef1eea3baef226ab1329913052cf75bcf8fc15b Mon Sep 17 00:00:00 2001 From: Istvan Haller Date: Thu, 5 Aug 2021 11:14:43 +0100 Subject: [PATCH 028/302] Improved the ability to extend GlobalState for custom Backend implementations --- src/backend/backend.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/backend/backend.h b/src/backend/backend.h index cab9944ea..f0311aebf 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -11,7 +11,10 @@ namespace snmalloc * This class implements the standard backend for handling allocations. * It abstracts page table management and address space management. */ - template + template< + SNMALLOC_CONCEPT(ConceptPAL) PAL, + bool fixed_range, + typename PageMapEntry = MetaEntry> class BackendAllocator { // Size of local address space requests. Currently aimed at 2MiB large @@ -64,9 +67,10 @@ namespace snmalloc { friend BackendAllocator; + protected: AddressSpaceManager address_space; - FlatPagemap pagemap; + FlatPagemap pagemap; public: template From e8374479f41b030065e477623f42b8568da232dc Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Thu, 5 Aug 2021 15:08:12 +0100 Subject: [PATCH 029/302] Snmalloc2 API cleanups for sandbox use. (#359) This is the set of changes required for snmalloc2 to be usable by the process sandboxing code and incorporates some API changes that reduce the amount of code required to embed snmalloc. Highlights: - Merge the config and back-end classes. - Everything in config is now global (all methods are static) - The GlobalState class is gone (all global state is managed by global methods on the config class) - LocalState is now a member of the config class, all methods are instance methods. - Not every configuration needs to use the lazy initialisation hooks. They now need to be provided only if they are used. If the configuration does not provide an `ensure_init` method, it is not called. If it does not provide an `is_initialised` method then the global initialisation state is not checked. - There is now an `snmalloc::Options` class that default initialises itself to the default behaviour. Every configuration must provide a `constexpr` instance of this class. Each flag can be separately overridden and new flags can be added without breaking any existing API consumers. The config classes are moved into the backend directory. --- src/backend/address_space_core.h | 2 + src/backend/backend.h | 109 ++++---- src/backend/commonconfig.h | 118 +++++++++ src/{mem => backend}/fixedglobalconfig.h | 42 +-- src/{mem => backend}/globalconfig.h | 41 ++- src/backend/pagemap.h | 36 ++- src/mem/commonconfig.h | 41 --- src/mem/corealloc.h | 179 ++++++++++--- src/mem/globalalloc.h | 72 ++++-- src/mem/localalloc.h | 243 +++++++++++++----- src/mem/localcache.h | 14 +- src/mem/metaslab.h | 21 ++ src/mem/pool.h | 26 +- src/mem/remotecache.h | 12 +- src/mem/slaballocator.h | 29 +-- src/mem/threadalloc.h | 2 +- src/override/malloc-extensions.cc | 8 +- src/override/malloc.cc | 2 +- src/snmalloc.h | 10 +- src/snmalloc_core.h | 5 +- src/test/func/fixed_region/fixed_region.cc | 5 +- src/test/func/malloc/malloc.cc | 2 +- src/test/func/memory/memory.cc | 6 +- src/test/func/statistics/stats.cc | 10 +- .../thread_alloc_external.cc | 2 +- src/test/func/two_alloc_types/alloc1.cc | 5 +- src/test/perf/contention/contention.cc | 2 +- .../perf/external_pointer/externalpointer.cc | 2 +- src/test/perf/singlethread/singlethread.cc | 2 +- 29 files changed, 672 insertions(+), 376 deletions(-) create mode 100644 src/backend/commonconfig.h rename src/{mem => backend}/fixedglobalconfig.h (52%) rename src/{mem => backend}/globalconfig.h (75%) delete mode 100644 src/mem/commonconfig.h diff --git a/src/backend/address_space_core.h b/src/backend/address_space_core.h index 549c317dd..d24b89f6f 100644 --- a/src/backend/address_space_core.h +++ b/src/backend/address_space_core.h @@ -1,6 +1,8 @@ #pragma once #include "../ds/address.h" #include "../ds/flaglock.h" +#include "../mem/allocconfig.h" +#include "../mem/metaslab.h" #include "../pal/pal.h" #include diff --git a/src/backend/backend.h b/src/backend/backend.h index f0311aebf..b9b62b9f9 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -3,6 +3,7 @@ #include "../mem/metaslab.h" #include "../pal/pal.h" #include "address_space.h" +#include "commonconfig.h" #include "pagemap.h" namespace snmalloc @@ -15,7 +16,7 @@ namespace snmalloc SNMALLOC_CONCEPT(ConceptPAL) PAL, bool fixed_range, typename PageMapEntry = MetaEntry> - class BackendAllocator + class BackendAllocator : public CommonConfig { // Size of local address space requests. Currently aimed at 2MiB large // pages but should make this configurable (i.e. for OE, so we don't need as @@ -54,55 +55,31 @@ namespace snmalloc #endif }; - /** - * Global state for the backend allocator - * - * This contains the various global datastructures required to store - * meta-data for each chunk of memory, and to provide well aligned chunks - * of memory. - * - * This type is required by snmalloc to exist as part of the Backend. - */ - class GlobalState - { - friend BackendAllocator; - - protected: - AddressSpaceManager address_space; + SNMALLOC_REQUIRE_CONSTINIT + static inline AddressSpaceManager address_space; - FlatPagemap pagemap; + SNMALLOC_REQUIRE_CONSTINIT + static inline FlatPagemap + pagemap; - public: - template - std::enable_if_t init() - { - static_assert( - fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); - - pagemap.init(); - - if constexpr (fixed_range) - { - abort(); - } - } + public: + template + static std::enable_if_t init() + { + static_assert(fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); - template - std::enable_if_t init(void* base, size_t length) - { - static_assert( - fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); + pagemap.init(); + } - auto [heap_base, heap_length] = pagemap.init(base, length); - address_space.add_range( - CapPtr(heap_base), heap_length, pagemap); + template + static std::enable_if_t init(void* base, size_t length) + { + static_assert(fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); - if constexpr (!fixed_range) - { - abort(); - } - } - }; + auto [heap_base, heap_length] = pagemap.init(base, length); + address_space.add_range( + CapPtr(heap_base), heap_length, pagemap); + } private: #ifdef SNMALLOC_CHECK_CLIENT @@ -138,8 +115,7 @@ namespace snmalloc * space managers. */ template - static CapPtr - reserve(GlobalState& h, LocalState* local_state, size_t size) + static CapPtr reserve(LocalState* local_state, size_t size) { #ifdef SNMALLOC_CHECK_CLIENT constexpr auto MAX_CACHED_SIZE = @@ -148,7 +124,7 @@ namespace snmalloc constexpr auto MAX_CACHED_SIZE = LOCAL_CACHE_BLOCK; #endif - auto& global = h.address_space; + auto& global = address_space; CapPtr p; if ((local_state != nullptr) && (size <= MAX_CACHED_SIZE)) @@ -160,14 +136,14 @@ namespace snmalloc auto& local = local_state->local_address_space; #endif - p = local.template reserve_with_left_over(size, h.pagemap); + p = local.template reserve_with_left_over(size, pagemap); if (p != nullptr) { return p; } auto refill_size = LOCAL_CACHE_BLOCK; - auto refill = global.template reserve(refill_size, h.pagemap); + auto refill = global.template reserve(refill_size, pagemap); if (refill == nullptr) return nullptr; @@ -179,10 +155,10 @@ namespace snmalloc } #endif PAL::template notify_using(refill.unsafe_ptr(), refill_size); - local.template add_range(refill, refill_size, h.pagemap); + local.template add_range(refill, refill_size, pagemap); // This should succeed - return local.template reserve_with_left_over(size, h.pagemap); + return local.template reserve_with_left_over(size, pagemap); } #ifdef SNMALLOC_CHECK_CLIENT @@ -193,7 +169,7 @@ namespace snmalloc size_t rsize = bits::max(OS_PAGE_SIZE, bits::next_pow2(size)); size_t size_request = rsize * 64; - p = global.template reserve(size_request, h.pagemap); + p = global.template reserve(size_request, pagemap); if (p == nullptr) return nullptr; @@ -209,7 +185,7 @@ namespace snmalloc SNMALLOC_ASSERT(!is_meta); #endif - p = global.template reserve_with_left_over(size, h.pagemap); + p = global.template reserve_with_left_over(size, pagemap); return p; } @@ -219,11 +195,16 @@ namespace snmalloc * * Backend allocator may use guard pages and separate area of * address space to protect this from corruption. + * + * The template argument is the type of the metadata being allocated. This + * allows the backend to allocate different types of metadata in different + * places or with different policies. */ + template static CapPtr - alloc_meta_data(GlobalState& h, LocalState* local_state, size_t size) + alloc_meta_data(LocalState* local_state, size_t size) { - return reserve(h, local_state, size); + return reserve(local_state, size); } /** @@ -236,7 +217,6 @@ namespace snmalloc * where metaslab, is the second element of the pair return. */ static std::pair, Metaslab*> alloc_chunk( - GlobalState& h, LocalState* local_state, size_t size, RemoteAllocator* remote, @@ -246,12 +226,12 @@ namespace snmalloc SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE); auto meta = reinterpret_cast( - reserve(h, local_state, sizeof(Metaslab)).unsafe_ptr()); + reserve(local_state, sizeof(Metaslab)).unsafe_ptr()); if (meta == nullptr) return {nullptr, nullptr}; - CapPtr p = reserve(h, local_state, size); + CapPtr p = reserve(local_state, size); #ifdef SNMALLOC_TRACING std::cout << "Alloc chunk: " << p.unsafe_ptr() << " (" << size << ")" @@ -274,7 +254,7 @@ namespace snmalloc a < address_cast(pointer_offset(p, size)); a += MIN_CHUNK_SIZE) { - h.pagemap.set(a, t); + pagemap.set(a, t); } return {p, meta}; } @@ -286,20 +266,19 @@ namespace snmalloc * to access a location that is not backed by a chunk. */ template - static const MetaEntry& get_meta_data(GlobalState& h, address_t p) + static const MetaEntry& get_meta_data(address_t p) { - return h.pagemap.template get(p); + return pagemap.template get(p); } /** * Set the metadata associated with a chunk. */ - static void - set_meta_data(GlobalState& h, address_t p, size_t size, MetaEntry t) + static void set_meta_data(address_t p, size_t size, MetaEntry t) { for (address_t a = p; a < p + size; a += MIN_CHUNK_SIZE) { - h.pagemap.set(a, t); + pagemap.set(a, t); } } }; diff --git a/src/backend/commonconfig.h b/src/backend/commonconfig.h new file mode 100644 index 000000000..689a13946 --- /dev/null +++ b/src/backend/commonconfig.h @@ -0,0 +1,118 @@ +#pragma once + +#include "../ds/defines.h" +#include "../mem/remotecache.h" + +namespace snmalloc +{ + // Forward reference to thread local cleanup. + void register_clean_up(); + + /** + * Options for a specific snmalloc configuration. Every globals object must + * have one `constexpr` instance of this class called `Options`. This should + * be constructed to explicitly override any of the defaults. A + * configuration that does not need to override anything would simply declare + * this as a field of the global object: + * + * ```c++ + * constexpr static snmalloc::Flags Options{}; + * ``` + * + * A global configuration that wished to use out-of-line message queues but + * accept the defaults for everything else would instead do this: + * + * ```c++ + * constexpr static snmalloc::Flags Options{.IsQueueInline = false}; + * ``` + * + * To maintain backwards source compatibility in future versions, any new + * option added here should have its default set to be whatever snmalloc was + * doing before the new option was added. + */ + struct Flags + { + /** + * Should allocators have inline message queues? If this is true then + * the `CoreAllocator` is responsible for allocating the + * `RemoteAllocator` that contains its message queue. If this is false + * then the `RemoteAllocator` must be separately allocated and provided + * to the `CoreAllocator` before it is used. + * + * Setting this to `false` currently requires also setting + * `LocalAllocSupportsLazyInit` to false so that the `CoreAllocator` can + * be provided to the `LocalAllocator` fully initialised but in the + * future it may be possible to allocate the `RemoteAllocator` via + * `alloc_meta_data` or a similar API in the back end. + */ + bool IsQueueInline = true; + + /** + * Does the `CoreAllocator` own a `Backend::LocalState` object? If this is + * true then the `CoreAllocator` is responsible for allocating and + * deallocating a local state object, otherwise the surrounding code is + * responsible for creating it. + * + * Use cases that set this to false will probably also need to set + * `LocalAllocSupportsLazyInit` to false so that they can provide the local + * state explicitly during allocator creation. + */ + bool CoreAllocOwnsLocalState = true; + + /** + * Are `CoreAllocator` allocated by the pool allocator? If not then the + * code embedding this snmalloc configuration is responsible for allocating + * `CoreAllocator` instances. + * + * Users setting this flag must also set `LocalAllocSupportsLazyInit` to + * false currently because there is no alternative mechanism for allocating + * core allocators. This may change in future versions. + */ + bool CoreAllocIsPoolAllocated = true; + + /** + * Do `LocalAllocator` instances in this configuration support lazy + * initialisation? If so, then the first exit from a fast path will + * trigger allocation of a `CoreAllocator` and associated state. If not + * then the code embedding this configuration of snmalloc is responsible + * for allocating core allocators. + */ + bool LocalAllocSupportsLazyInit = true; + }; + + /** + * Class containing definitions that are likely to be used by all except for + * the most unusual back-end implementations. This can be subclassed as a + * convenience for back-end implementers, but is not required. + */ + class CommonConfig + { + public: + /** + * Special remote that should never be used as a real remote. + * This is used to initialise allocators that should always hit the + * remote path for deallocation. Hence moving a branch off the critical + * path. + */ + SNMALLOC_REQUIRE_CONSTINIT + inline static RemoteAllocator unused_remote; + + /** + * Special remote that is used in meta-data for large allocations. + * + * nullptr is considered a large allocations for this purpose to move + * of the critical path. + * + * Bottom bits of the remote pointer are used for a sizeclass, we need + * size bits to represent the non-large sizeclasses, we can then get + * the large sizeclass by having the fake large_remote considerably + * more aligned. + */ + SNMALLOC_REQUIRE_CONSTINIT + inline static constexpr RemoteAllocator* fake_large_remote{nullptr}; + + static_assert( + &unused_remote != fake_large_remote, + "Compilation should ensure these are different"); + }; +} // namespace snmalloc diff --git a/src/mem/fixedglobalconfig.h b/src/backend/fixedglobalconfig.h similarity index 52% rename from src/mem/fixedglobalconfig.h rename to src/backend/fixedglobalconfig.h index 01b268f5f..2e4473e1c 100644 --- a/src/mem/fixedglobalconfig.h +++ b/src/backend/fixedglobalconfig.h @@ -12,50 +12,27 @@ namespace snmalloc * A single fixed address range allocator configuration */ template - class FixedGlobals : public CommonConfig + class FixedGlobals final : public BackendAllocator { - public: - using Backend = BackendAllocator; - private: - inline static typename Backend::GlobalState backend_state; - + using Backend = BackendAllocator; inline static ChunkAllocatorState slab_allocator_state; inline static PoolState> alloc_pool; public: - static typename Backend::GlobalState& get_backend_state() - { - return backend_state; - } - - ChunkAllocatorState& get_slab_allocator_state() + static ChunkAllocatorState& + get_slab_allocator_state(typename Backend::LocalState*) { return slab_allocator_state; } - PoolState>& pool() + static PoolState>& pool() { return alloc_pool; } - static constexpr bool IsQueueInline = true; - - // Performs initialisation for this configuration - // of allocators. Will be called at most once - // before any other datastructures are accessed. - void ensure_init() noexcept - { -#ifdef SNMALLOC_TRACING - std::cout << "Run init_impl" << std::endl; -#endif - } - - static bool is_initialised() - { - return true; - } + static constexpr Flags Options{}; // This needs to be a forward reference as the // thread local state will need to know about this. @@ -68,12 +45,7 @@ namespace snmalloc static void init(void* base, size_t length) { - get_backend_state().init(base, length); - } - - constexpr static FixedGlobals get_handle() - { - return {}; + Backend::init(base, length); } }; } diff --git a/src/mem/globalconfig.h b/src/backend/globalconfig.h similarity index 75% rename from src/mem/globalconfig.h rename to src/backend/globalconfig.h index cc5dc76b1..f084703d4 100644 --- a/src/mem/globalconfig.h +++ b/src/backend/globalconfig.h @@ -23,15 +23,15 @@ namespace snmalloc } #endif - class Globals : public CommonConfig + /** + * The default configuration for a global snmalloc. This allocates memory + * from the operating system and expects to manage memory anywhere in the + * address space. + */ + class Globals final : public BackendAllocator { - public: - using Backend = BackendAllocator; - private: - SNMALLOC_REQUIRE_CONSTINIT - inline static Backend::GlobalState backend_state; - + using Backend = BackendAllocator; SNMALLOC_REQUIRE_CONSTINIT inline static ChunkAllocatorState slab_allocator_state; @@ -45,27 +45,23 @@ namespace snmalloc inline static std::atomic_flag initialisation_lock{}; public: - Backend::GlobalState& get_backend_state() - { - return backend_state; - } - - ChunkAllocatorState& get_slab_allocator_state() + static ChunkAllocatorState& + get_slab_allocator_state(Backend::LocalState* = nullptr) { return slab_allocator_state; } - PoolState>& pool() + static PoolState>& pool() { return alloc_pool; } - static constexpr bool IsQueueInline = true; + static constexpr Flags Options{}; // Performs initialisation for this configuration // of allocators. Needs to be idempotent, // and concurrency safe. - void ensure_init() + static void ensure_init() { FlagLock lock{initialisation_lock}; #ifdef SNMALLOC_TRACING @@ -81,7 +77,7 @@ namespace snmalloc key_global = FreeListKey(entropy.get_free_list_key()); // Need to initialise pagemap. - backend_state.init(); + Backend::init(); #ifdef USE_SNMALLOC_STATS atexit(snmalloc::print_stats); @@ -90,7 +86,7 @@ namespace snmalloc initialised = true; } - bool is_initialised() + static bool is_initialised() { return initialised; } @@ -99,16 +95,9 @@ namespace snmalloc // thread local state will need to know about this. // This may allocate, so should only be called once // a thread local allocator is available. - void register_clean_up() + static void register_clean_up() { snmalloc::register_clean_up(); } - - // This is an empty structure as all the state is global - // for this allocator configuration. - static constexpr Globals get_handle() - { - return {}; - } }; } // namespace snmalloc diff --git a/src/backend/pagemap.h b/src/backend/pagemap.h index 4f4ab6e3d..d640ae1b1 100644 --- a/src/backend/pagemap.h +++ b/src/backend/pagemap.h @@ -66,6 +66,36 @@ namespace snmalloc constexpr FlatPagemap() = default; + /** + * For pagemaps that cover an entire fixed address space, return the size + * that they must be. This allows the caller to allocate the correct + * amount of memory to be passed to `init`. This is not available for + * fixed-range pagemaps, whose size depends on dynamic configuration. + */ + template + static constexpr std::enable_if_t required_size() + { + static_assert( + has_bounds_ == has_bounds, "Don't set SFINAE template parameter!"); + constexpr size_t COVERED_BITS = bits::ADDRESS_BITS - GRANULARITY_BITS; + constexpr size_t ENTRIES = bits::one_at_bit(COVERED_BITS); + return ENTRIES * sizeof(T); + } + + /** + * Initialise with pre-allocated memory. + * + * This is currently disabled for bounded pagemaps but may be reenabled if + * `required_size` is enabled for the has-bounds case. + */ + template + std::enable_if_t init(T* address) + { + static_assert( + has_bounds_ == has_bounds, "Don't set SFINAE template parameter!"); + body = address; + } + /** * Initialise the pagemap with bounds. * @@ -117,11 +147,7 @@ namespace snmalloc { static_assert( has_bounds_ == has_bounds, "Don't set SFINAE template parameter!"); - static constexpr size_t COVERED_BITS = - bits::ADDRESS_BITS - GRANULARITY_BITS; - static constexpr size_t ENTRIES = bits::one_at_bit(COVERED_BITS); - - static constexpr size_t REQUIRED_SIZE = ENTRIES * sizeof(T); + static constexpr size_t REQUIRED_SIZE = required_size(); #ifdef SNMALLOC_CHECK_CLIENT // Allocate a power of two extra to allow the placement of the diff --git a/src/mem/commonconfig.h b/src/mem/commonconfig.h deleted file mode 100644 index 9c24371c8..000000000 --- a/src/mem/commonconfig.h +++ /dev/null @@ -1,41 +0,0 @@ -#pragma once - -#include "../ds/defines.h" -#include "remotecache.h" - -namespace snmalloc -{ - // Forward reference to thread local cleanup. - void register_clean_up(); - - class CommonConfig - { - public: - /** - * Special remote that should never be used as a real remote. - * This is used to initialise allocators that should always hit the - * remote path for deallocation. Hence moving a branch off the critical - * path. - */ - SNMALLOC_REQUIRE_CONSTINIT - inline static RemoteAllocator unused_remote; - - /** - * Special remote that is used in meta-data for large allocations. - * - * nullptr is considered a large allocations for this purpose to move - * of the critical path. - * - * Bottom bits of the remote pointer are used for a sizeclass, we need - * size bits to represent the non-large sizeclasses, we can then get - * the large sizeclass by having the fake large_remote considerably - * more aligned. - */ - SNMALLOC_REQUIRE_CONSTINIT - inline static constexpr RemoteAllocator* fake_large_remote{nullptr}; - - static_assert( - &unused_remote != fake_large_remote, - "Compilation should ensure these are different"); - }; -} // namespace snmalloc diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 96490a763..0097b1247 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -11,10 +11,45 @@ namespace snmalloc { + /** + * Empty class used as the superclass for `CoreAllocator` when it does not + * opt into pool allocation. This class exists because `std::conditional` + * (or other equivalent features in C++) can choose between options for + * superclasses but they cannot choose whether a class has a superclass. + * Setting the superclass to an empty class is equivalent to no superclass. + */ + class NotPoolAllocated + {}; + + /** + * The core, stateful, part of a memory allocator. Each `LocalAllocator` + * owns one `CoreAllocator` once it is initialised. + * + * The template parameter provides all of the global configuration for this + * instantiation of snmalloc. This includes three options that apply to this + * class: + * + * - `CoreAllocIsPoolAllocated` defines whether this `CoreAlloc` + * configuration should support pool allocation. This defaults to true but + * a configuration that allocates allocators eagerly may opt out. + * - `CoreAllocOwnsLocalState` defines whether the `CoreAllocator` owns the + * associated `LocalState` object. If this is true (the default) then + * `CoreAllocator` embeds the LocalState object. If this is set to false + * then a `LocalState` object must be provided to the constructor. This + * allows external code to provide explicit configuration of the address + * range managed by this object. + * - `IsQueueInline` (defaults to true) defines whether the message queue + * (`RemoteAllocator`) for this class is inline or provided externally. If + * provided externally, then it must be set explicitly with + * `init_message_queue`. + */ template - class CoreAllocator : public Pooled> + class CoreAllocator : public std::conditional_t< + SharedStateHandle::Options.CoreAllocIsPoolAllocated, + Pooled>, + NotPoolAllocated> { - template + template friend class LocalAllocator; /** @@ -33,16 +68,27 @@ namespace snmalloc * allocator */ std::conditional_t< - SharedStateHandle::IsQueueInline, + SharedStateHandle::Options.IsQueueInline, RemoteAllocator, RemoteAllocator*> remote_alloc; + /** + * The type used local state. This is defined by the back end. + */ + using LocalState = typename SharedStateHandle::LocalState; + /** * A local area of address space managed by this allocator. - * Used to reduce calls on the global address space. + * Used to reduce calls on the global address space. This is inline if the + * core allocator owns the local state or indirect if it is owned + * externally. */ - typename SharedStateHandle::Backend::LocalState backend_state; + std::conditional_t< + SharedStateHandle::Options.CoreAllocOwnsLocalState, + LocalState, + LocalState*> + backend_state; /** * This is the thread local structure associated to this @@ -50,12 +96,6 @@ namespace snmalloc */ LocalCache* attached_cache; - /** - * This contains the way to access all the global state and - * configuration for the system setup. - */ - SharedStateHandle handle; - /** * The message queue needs to be accessible from other threads * @@ -64,7 +104,7 @@ namespace snmalloc */ auto* public_state() { - if constexpr (SharedStateHandle::IsQueueInline) + if constexpr (SharedStateHandle::Options.IsQueueInline) { return &remote_alloc; } @@ -260,8 +300,10 @@ namespace snmalloc // TODO delay the clear to the next user of the slab, or teardown so // don't touch the cache lines at this point in check_client. auto chunk_record = clear_slab(meta, sizeclass); - ChunkAllocator::dealloc( - handle, chunk_record, sizeclass_to_slab_sizeclass(sizeclass)); + ChunkAllocator::dealloc( + get_backend_local_state(), + chunk_record, + sizeclass_to_slab_sizeclass(sizeclass)); } else { @@ -335,8 +377,8 @@ namespace snmalloc for (size_t i = 0; i < REMOTE_BATCH; i++) { auto p = message_queue().peek(); - auto& entry = SharedStateHandle::Backend::get_meta_data( - handle.get_backend_state(), snmalloc::address_cast(p)); + auto& entry = + SharedStateHandle::get_meta_data(snmalloc::address_cast(p)); auto r = message_queue().dequeue(key_global); @@ -388,9 +430,11 @@ namespace snmalloc } } - public: - CoreAllocator(LocalCache* cache, SharedStateHandle handle) - : attached_cache(cache), handle(handle) + /** + * Initialiser, shared code between the constructors for different + * configurations. + */ + void init() { #ifdef SNMALLOC_TRACING std::cout << "Making an allocator." << std::endl; @@ -398,13 +442,16 @@ namespace snmalloc // Entropy must be first, so that all data-structures can use the key // it generates. // This must occur before any freelists are constructed. - entropy.init(); + entropy.init(); // Ignoring stats for now. // stats().start(); - init_message_queue(); - message_queue().invariant(); + if constexpr (SharedStateHandle::Options.IsQueueInline) + { + init_message_queue(); + message_queue().invariant(); + } #ifndef NDEBUG for (sizeclass_t i = 0; i < NUM_SIZECLASSES; i++) @@ -423,6 +470,44 @@ namespace snmalloc #endif } + public: + /** + * Constructor for the case that the core allocator owns the local state. + * SFINAE disabled if the allocator does not own the local state. + */ + template< + typename Config = SharedStateHandle, + typename = std::enable_if_t> + CoreAllocator(LocalCache* cache) : attached_cache(cache) + { + init(); + } + + /** + * Constructor for the case that the core allocator does not owns the local + * state. SFINAE disabled if the allocator does own the local state. + */ + template< + typename Config = SharedStateHandle, + typename = std::enable_if_t> + CoreAllocator(LocalCache* cache, LocalState* backend = nullptr) + : backend_state(backend), attached_cache(cache) + { + init(); + } + + /** + * If the message queue is not inline, provide it. This will then + * configure the message queue for use. + */ + template + std::enable_if_t init_message_queue(RemoteAllocator* q) + { + remote_alloc = q; + init_message_queue(); + message_queue().invariant(); + } + /** * Post deallocations onto other threads. * @@ -432,9 +517,9 @@ namespace snmalloc SNMALLOC_FAST_PATH bool post() { // stats().remote_post(); // TODO queue not in line! - bool sent_something = - attached_cache->remote_dealloc_cache.post( - handle, public_state()->trunc_id(), key_global); + bool sent_something = attached_cache->remote_dealloc_cache + .post( + public_state()->trunc_id(), key_global); return sent_something; } @@ -454,8 +539,7 @@ namespace snmalloc SNMALLOC_FAST_PATH void dealloc_local_object(void* p) { - auto entry = SharedStateHandle::Backend::get_meta_data( - handle.get_backend_state(), snmalloc::address_cast(p)); + auto entry = SharedStateHandle::get_meta_data(snmalloc::address_cast(p)); if (likely(dealloc_local_object_fast(entry, p, entropy))) return; @@ -506,6 +590,24 @@ namespace snmalloc return small_alloc_slow(sizeclass, fast_free_list, rsize); } + /** + * Accessor for the local state. This hides whether the local state is + * stored inline or provided externally from the rest of the code. + */ + SNMALLOC_FAST_PATH + LocalState& get_backend_local_state() + { + if constexpr (SharedStateHandle::Options.CoreAllocOwnsLocalState) + { + return backend_state; + } + else + { + SNMALLOC_ASSERT(backend_state); + return *backend_state; + } + } + template SNMALLOC_SLOW_PATH void* small_alloc_slow( sizeclass_t sizeclass, FreeListIter& fast_free_list, size_t rsize) @@ -519,13 +621,13 @@ namespace snmalloc std::cout << "slab size " << slab_size << std::endl; #endif - auto [slab, meta] = snmalloc::ChunkAllocator::alloc_chunk( - handle, - backend_state, - sizeclass, - slab_sizeclass, - slab_size, - public_state()); + auto [slab, meta] = + snmalloc::ChunkAllocator::alloc_chunk( + get_backend_local_state(), + sizeclass, + slab_sizeclass, + slab_size, + public_state()); if (slab == nullptr) { @@ -563,8 +665,8 @@ namespace snmalloc { bool need_post = true; // Always going to post, so ignore. auto n = p->atomic_read_next(key_global); - auto& entry = SharedStateHandle::Backend::get_meta_data( - handle.get_backend_state(), snmalloc::address_cast(p)); + auto& entry = + SharedStateHandle::get_meta_data(snmalloc::address_cast(p)); handle_dealloc_remote(entry, p, need_post); p = n; } @@ -577,8 +679,9 @@ namespace snmalloc handle_message_queue([]() {}); } - auto posted = attached_cache->flush( - [&](auto p) { dealloc_local_object(p); }, handle); + auto posted = + attached_cache->flush( + [&](auto p) { dealloc_local_object(p); }); // We may now have unused slabs, return to the global allocator. for (sizeclass_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++) diff --git a/src/mem/globalalloc.h b/src/mem/globalalloc.h index 00265989e..cc4ecd53b 100644 --- a/src/mem/globalalloc.h +++ b/src/mem/globalalloc.h @@ -6,9 +6,13 @@ namespace snmalloc { template - inline static void aggregate_stats(SharedStateHandle handle, Stats& stats) + inline static void aggregate_stats(Stats& stats) { - auto* alloc = Pool>::iterate(handle); + static_assert( + SharedStateHandle::Options.CoreAllocIsPoolAllocated, + "Global statistics are available only for pool-allocated configurations"); + auto* alloc = Pool>::template iterate< + SharedStateHandle>(); while (alloc != nullptr) { @@ -16,45 +20,51 @@ namespace snmalloc if (a != nullptr) stats.add(*a); stats.add(alloc->stats()); - alloc = Pool>::iterate(handle, alloc); + alloc = Pool>::template iterate< + SharedStateHandle>(alloc); } } #ifdef USE_SNMALLOC_STATS template - inline static void print_all_stats( - SharedStateHandle handle, std::ostream& o, uint64_t dumpid = 0) + inline static void print_all_stats(std::ostream& o, uint64_t dumpid = 0) { - auto alloc = Pool>::iterate(handle); + static_assert( + SharedStateHandle::Options.CoreAllocIsPoolAllocated, + "Global statistics are available only for pool-allocated configurations"); + auto alloc = Pool>::template iterate< + SharedStateHandle>(); while (alloc != nullptr) { auto stats = alloc->stats(); if (stats != nullptr) - stats->template print(o, dumpid, alloc->id()); - alloc = Pool>::iterate(handle, alloc); + stats->template print(o, dumpid, alloc->id()); + alloc = Pool>::template iterate< + SharedStateHandle>(alloc); } } #else template - inline static void - print_all_stats(SharedStateHandle handle, void*& o, uint64_t dumpid = 0) + inline static void print_all_stats(void*& o, uint64_t dumpid = 0) { UNUSED(o); UNUSED(dumpid); - UNUSED(handle); } #endif template - inline static void cleanup_unused(SharedStateHandle handle) + inline static void cleanup_unused() { #ifndef SNMALLOC_PASS_THROUGH + static_assert( + SharedStateHandle::Options.CoreAllocIsPoolAllocated, + "Global cleanup is available only for pool-allocated configurations"); // Call this periodically to free and coalesce memory allocated by // allocators that are not currently in use by any thread. // One atomic operation to extract the stack, another to restore it. // Handling the message queue for each stack is non-atomic. - auto* first = Pool>::extract(handle); + auto* first = Pool>::extract(); auto* alloc = first; decltype(alloc) last; @@ -64,10 +74,10 @@ namespace snmalloc { alloc->flush(); last = alloc; - alloc = Pool>::extract(handle, alloc); + alloc = Pool>::extract(alloc); } - Pool>::restore(handle, first, last); + Pool>::restore(first, last); } #endif } @@ -78,13 +88,16 @@ namespace snmalloc raise an error all the allocators are not empty. */ template - inline static void - debug_check_empty(SharedStateHandle handle, bool* result = nullptr) + inline static void debug_check_empty(bool* result = nullptr) { #ifndef SNMALLOC_PASS_THROUGH + static_assert( + SharedStateHandle::Options.CoreAllocIsPoolAllocated, + "Global status is available only for pool-allocated configurations"); // This is a debugging function. It checks that all memory from all // allocators has been freed. - auto* alloc = Pool>::iterate(handle); + auto* alloc = Pool>::template iterate< + SharedStateHandle>(); # ifdef SNMALLOC_TRACING std::cout << "debug check empty: first " << alloc << std::endl; @@ -98,7 +111,8 @@ namespace snmalloc std::cout << "debug_check_empty: Check all allocators!" << std::endl; # endif done = true; - alloc = Pool>::iterate(handle); + alloc = Pool>::template iterate< + SharedStateHandle>(); okay = true; while (alloc != nullptr) @@ -120,7 +134,8 @@ namespace snmalloc # ifdef SNMALLOC_TRACING std::cout << "debug check empty: okay = " << okay << std::endl; # endif - alloc = Pool>::iterate(handle, alloc); + alloc = Pool>::template iterate< + SharedStateHandle>(alloc); } } @@ -133,11 +148,13 @@ namespace snmalloc // Redo check so abort is on allocator with allocation left. if (!okay) { - alloc = Pool>::iterate(handle); + alloc = Pool>::template iterate< + SharedStateHandle>(); while (alloc != nullptr) { alloc->debug_is_empty(nullptr); - alloc = Pool>::iterate(handle, alloc); + alloc = Pool>::template iterate< + SharedStateHandle>(alloc); } } #else @@ -146,9 +163,13 @@ namespace snmalloc } template - inline static void debug_in_use(SharedStateHandle handle, size_t count) + inline static void debug_in_use(size_t count) { - auto alloc = Pool>::iterate(handle); + static_assert( + SharedStateHandle::Options.CoreAllocIsPoolAllocated, + "Global status is available only for pool-allocated configurations"); + auto alloc = Pool>::template iterate< + SharedStateHandle>(); while (alloc != nullptr) { if (alloc->debug_is_in_use()) @@ -159,7 +180,8 @@ namespace snmalloc } count--; } - alloc = Pool>::iterate(handle, alloc); + alloc = Pool>::template iterate< + SharedStateHandle>(alloc); if (count != 0) { diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index c42685eb9..ad5807169 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -38,24 +38,34 @@ namespace snmalloc OnePastEnd }; - // This class contains the fastest path code for the allocator. + /** + * A local allocator contains the fast-path allocation routines and + * encapsulates all of the behaviour of an allocator that is local to some + * context, typically a thread. This delegates to a `CoreAllocator` for all + * slow-path operations, including anything that requires claiming new chunks + * of address space. + * + * The template parameter defines the configuration of this allocator and is + * passed through to the associated `CoreAllocator`. The `Options` structure + * of this defines one property that directly affects the behaviour of the + * local allocator: `LocalAllocSupportsLazyInit`, which defaults to true, + * defines whether the local allocator supports lazy initialisation. If this + * is true then the local allocator will construct a core allocator the first + * time it needs to perform a slow-path operation. If this is false then the + * core allocator must be provided externally by invoking the `init` method + * on this class *before* any allocation-related methods are called. + */ template class LocalAllocator { using CoreAlloc = CoreAllocator; private: - /** - * Contains a way to access all the shared state for this allocator. - * This may have no dynamic state, and be purely static. - */ - SharedStateHandle handle; - // Free list per small size class. These are used for // allocation on the fast path. This part of the code is inspired by // mimalloc. // Also contains remote deallocation cache. - LocalCache local_cache; + LocalCache local_cache{&SharedStateHandle::unused_remote}; // Underlying allocator for most non-fast path operations. CoreAlloc* core_alloc{nullptr}; @@ -90,40 +100,59 @@ namespace snmalloc /** * This initialises the fast allocator by acquiring a core allocator, and * setting up its local copy of data structures. + * + * If the allocator does not support lazy initialisation then this assumes + * that initialisation has already taken place and invokes the action + * immediately. */ template SNMALLOC_SLOW_PATH decltype(auto) lazy_init(Action action, Args... args) { SNMALLOC_ASSERT(core_alloc == nullptr); - - // Initialise the thread local allocator - init(); - - // register_clean_up must be called after init. register clean up may be - // implemented with allocation, so need to ensure we have a valid - // allocator at this point. - if (!post_teardown) - // Must be called at least once per thread. - // A pthread implementation only calls the thread destruction handle - // if the key has been set. - handle.register_clean_up(); - - // Perform underlying operation - auto r = action(core_alloc, args...); - - // After performing underlying operation, in the case of teardown already - // having begun, we must flush any state we just acquired. - if (post_teardown) + if constexpr (!SharedStateHandle::Options.LocalAllocSupportsLazyInit) { + SNMALLOC_CHECK( + false && + "lazy_init called on an allocator that doesn't support lazy " + "initialisation"); + // Unreachable, but needed to keep the type checker happy in deducing + // the return type of this function. + return static_cast(nullptr); + } + else + { + // Initialise the thread local allocator + if constexpr (SharedStateHandle::Options.CoreAllocOwnsLocalState) + { + init(); + } + + // register_clean_up must be called after init. register clean up may + // be implemented with allocation, so need to ensure we have a valid + // allocator at this point. + if (!post_teardown) + // Must be called at least once per thread. + // A pthread implementation only calls the thread destruction handle + // if the key has been set. + SharedStateHandle::register_clean_up(); + + // Perform underlying operation + auto r = action(core_alloc, args...); + + // After performing underlying operation, in the case of teardown + // already having begun, we must flush any state we just acquired. + if (post_teardown) + { #ifdef SNMALLOC_TRACING - std::cout << "post_teardown flush()" << std::endl; + std::cout << "post_teardown flush()" << std::endl; #endif - // We didn't have an allocator because the thread is being torndown. - // We need to return any local state, so we don't leak it. - flush(); - } + // We didn't have an allocator because the thread is being torndown. + // We need to return any local state, so we don't leak it. + flush(); + } - return r; + return r; + } } /** @@ -144,13 +173,12 @@ namespace snmalloc return check_init([&](CoreAlloc* core_alloc) { // Grab slab of correct size // Set remote as large allocator remote. - auto [chunk, meta] = ChunkAllocator::alloc_chunk( - handle, - core_alloc->backend_state, + auto [chunk, meta] = ChunkAllocator::alloc_chunk( + core_alloc->get_backend_local_state(), bits::next_pow2_bits(size), // TODO large_size_to_chunk_sizeclass(size), large_size_to_chunk_size(size), - handle.fake_large_remote); + SharedStateHandle::fake_large_remote); // set up meta data so sizeclass is correct, and hence alloc size, and // external pointer. #ifdef SNMALLOC_TRACING @@ -164,7 +192,7 @@ namespace snmalloc if (zero_mem == YesZero) { - SharedStateHandle::Backend::Pal::template zero( + SharedStateHandle::Pal::template zero( chunk.unsafe_ptr(), size); } @@ -225,8 +253,7 @@ namespace snmalloc std::cout << "Remote dealloc post" << p << " size " << alloc_size(p) << std::endl; #endif - MetaEntry entry = SharedStateHandle::Backend::get_meta_data( - handle.get_backend_state(), address_cast(p)); + MetaEntry entry = SharedStateHandle::get_meta_data(address_cast(p)); local_cache.remote_dealloc_cache.template dealloc( entry.get_remote()->trunc_id(), CapPtr(p), key_global); post_remote_cache(); @@ -252,30 +279,84 @@ namespace snmalloc return local_cache.remote_allocator; } - public: - constexpr LocalAllocator() - : handle(SharedStateHandle::get_handle()), - local_cache(&handle.unused_remote) - {} + /** + * SFINAE helper. Matched only if `T` implements `is_initialised`. Calls + * it if it exists. + */ + template + SNMALLOC_FAST_PATH auto call_is_initialised(T*, int) + -> decltype(T::is_initialised()) + { + return T::is_initialised(); + } + + /** + * SFINAE helper. Matched only if `T` does not implement `is_initialised`. + * Unconditionally returns true if invoked. + */ + template + SNMALLOC_FAST_PATH auto call_is_initialised(T*, long) + { + return true; + } - LocalAllocator(SharedStateHandle handle) - : handle(handle), local_cache(&handle.unused_remote) + /** + * Call `SharedStateHandle::is_initialised()` if it is implemented, + * unconditionally returns true otherwise. + */ + SNMALLOC_FAST_PATH + bool is_initialised() + { + return call_is_initialised(nullptr, 0); + } + + /** + * SFINAE helper. Matched only if `T` implements `ensure_init`. Calls it + * if it exists. + */ + template + SNMALLOC_FAST_PATH auto call_ensure_init(T*, int) + -> decltype(T::ensure_init()) + { + T::ensure_init(); + } + + /** + * SFINAE helper. Matched only if `T` does not implement `ensure_init`. + * Does nothing if called. + */ + template + SNMALLOC_FAST_PATH auto call_ensure_init(T*, long) {} - // This is effectively the constructor for the LocalAllocator, but due to - // not wanting initialisation checks on the fast path, it is initialised - // lazily. - void init() + /** + * Call `SharedStateHandle::ensure_init()` if it is implemented, do nothing + * otherwise. + */ + SNMALLOC_FAST_PATH + void ensure_init() + { + call_ensure_init(nullptr, 0); + } + + public: + constexpr LocalAllocator() = default; + + /** + * Initialise the allocator. For allocators that support local + * initialisation, this is called with a core allocator that this class + * allocates (from a pool allocator) the first time it encounters a slow + * path. If this class is configured without lazy initialisation support + * then this must be called externally + */ + void init(CoreAlloc* c) { // Initialise the global allocator structures - handle.ensure_init(); + ensure_init(); // Should only be called if the allocator has not been initialised. SNMALLOC_ASSERT(core_alloc == nullptr); - // Grab an allocator for this thread. - auto c = Pool::acquire(handle, &(this->local_cache), handle); - // Attach to it. c->attach(&local_cache); core_alloc = c; @@ -286,6 +367,18 @@ namespace snmalloc // local_cache.stats.sta rt(); } + // This is effectively the constructor for the LocalAllocator, but due to + // not wanting initialisation checks on the fast path, it is initialised + // lazily. + void init() + { + // Initialise the global allocator structures + ensure_init(); + // Grab an allocator for this thread. + init(Pool::template acquire( + &(this->local_cache))); + } + // Return all state in the fast allocator and release the underlying // core allocator. This is used during teardown to empty the thread // local state. @@ -303,7 +396,10 @@ namespace snmalloc // Detach underlying allocator core_alloc->attached_cache = nullptr; // Return underlying allocator to the system. - Pool::release(handle, core_alloc); + if constexpr (SharedStateHandle::Options.CoreAllocOwnsLocalState) + { + Pool::template release(core_alloc); + } // Set up thread local allocator to look like // it is new to hit slow paths. @@ -311,7 +407,7 @@ namespace snmalloc #ifdef SNMALLOC_TRACING std::cout << "flush(): core_alloc=" << core_alloc << std::endl; #endif - local_cache.remote_allocator = &handle.unused_remote; + local_cache.remote_allocator = &SharedStateHandle::unused_remote; local_cache.remote_dealloc_cache.capacity = 0; } } @@ -365,8 +461,8 @@ namespace snmalloc // before init, that maps null to a remote_deallocator that will never be // in thread local state. - const MetaEntry& entry = SharedStateHandle::Backend::get_meta_data( - handle.get_backend_state(), address_cast(p)); + const MetaEntry& entry = + SharedStateHandle::get_meta_data(address_cast(p)); if (likely(local_cache.remote_allocator == entry.get_remote())) { if (likely(CoreAlloc::dealloc_local_object_fast( @@ -376,7 +472,7 @@ namespace snmalloc return; } - if (likely(entry.get_remote() != handle.fake_large_remote)) + if (likely(entry.get_remote() != SharedStateHandle::fake_large_remote)) { // Check if we have space for the remote deallocation if (local_cache.remote_dealloc_cache.reserve_space(entry)) @@ -416,7 +512,8 @@ namespace snmalloc ChunkRecord* slab_record = reinterpret_cast(entry.get_metaslab()); slab_record->chunk = CapPtr(p); - ChunkAllocator::dealloc(handle, slab_record, slab_sizeclass); + ChunkAllocator::dealloc( + core_alloc->get_backend_local_state(), slab_record, slab_sizeclass); return; } @@ -460,10 +557,9 @@ namespace snmalloc // To handle this case we require the uninitialised pagemap contain an // entry for the first chunk of memory, that states it represents a large // object, so we can pull the check for null off the fast path. - MetaEntry entry = SharedStateHandle::Backend::get_meta_data( - handle.get_backend_state(), address_cast(p_raw)); + MetaEntry entry = SharedStateHandle::get_meta_data(address_cast(p_raw)); - if (likely(entry.get_remote() != handle.fake_large_remote)) + if (likely(entry.get_remote() != SharedStateHandle::fake_large_remote)) return sizeclass_to_size(entry.get_sizeclass()); // Sizeclass zero is for large is actually zero @@ -484,13 +580,12 @@ namespace snmalloc void* external_pointer(void* p_raw) { // TODO bring back the CHERI bits. Wes to review if required. - if (likely(handle.is_initialised())) + if (likely(is_initialised())) { MetaEntry entry = - SharedStateHandle::Backend::template get_meta_data( - handle.get_backend_state(), address_cast(p_raw)); + SharedStateHandle::template get_meta_data(address_cast(p_raw)); auto sizeclass = entry.get_sizeclass(); - if (likely(entry.get_remote() != handle.fake_large_remote)) + if (likely(entry.get_remote() != SharedStateHandle::fake_large_remote)) { auto rsize = sizeclass_to_size(sizeclass); auto offset = @@ -533,5 +628,15 @@ namespace snmalloc // We don't know the Start, so return MIN_PTR return nullptr; } + + /** + * Accessor, returns the local cache. If embedding code is allocating the + * core allocator for use by this local allocator then it needs to access + * this field. + */ + LocalCache& get_local_cache() + { + return local_cache; + } }; -} // namespace snmalloc \ No newline at end of file +} // namespace snmalloc diff --git a/src/mem/localcache.h b/src/mem/localcache.h index ae871c989..7e7687f05 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -32,7 +32,7 @@ namespace snmalloc auto r = finish_alloc_no_zero(p, sizeclass); if constexpr (zero_mem == YesZero) - SharedStateHandle::Backend::Pal::zero(r, sizeclass_to_size(sizeclass)); + SharedStateHandle::Pal::zero(r, sizeclass_to_size(sizeclass)); // TODO: Should this be zeroing the FreeObject state, in the non-zeroing // case? @@ -72,9 +72,9 @@ namespace snmalloc template< size_t allocator_size, - typename DeallocFun, - typename SharedStateHandle> - bool flush(DeallocFun dealloc, SharedStateHandle handle) + typename SharedStateHandle, + typename DeallocFun> + bool flush(DeallocFun dealloc) { auto& key = entropy.get_free_list_key(); @@ -91,8 +91,8 @@ namespace snmalloc } } - return remote_dealloc_cache.post( - handle, remote_allocator->trunc_id(), key_global); + return remote_dealloc_cache.post( + remote_allocator->trunc_id(), key_global); } template @@ -113,4 +113,4 @@ namespace snmalloc } }; -} // namespace snmalloc \ No newline at end of file +} // namespace snmalloc diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 2c8852850..ff2da8791 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -177,6 +177,16 @@ namespace snmalloc public: constexpr MetaEntry() = default; + /** + * Constructor, provides the remote and sizeclass embedded in a single + * pointer-sized word. This format is not guaranteed to be stable and so + * the second argument of this must always be the return value from + * `get_remote_and_sizeclass`. + */ + MetaEntry(Metaslab* meta, uintptr_t remote_and_sizeclass) + : meta(meta), remote_and_sizeclass(remote_and_sizeclass) + {} + MetaEntry(Metaslab* meta, RemoteAllocator* remote, sizeclass_t sizeclass) : meta(meta) { @@ -190,6 +200,17 @@ namespace snmalloc return meta; } + /** + * Return the remote and sizeclass in an implementation-defined encoding. + * This is not guaranteed to be stable across snmalloc releases and so the + * only safe use for this is to pass it to the two-argument constructor of + * this class. + */ + uintptr_t get_remote_and_sizeclass() + { + return remote_and_sizeclass; + } + [[nodiscard]] RemoteAllocator* get_remote() const { return reinterpret_cast( diff --git a/src/mem/pool.h b/src/mem/pool.h index bedbce58e..d3c25b4f9 100644 --- a/src/mem/pool.h +++ b/src/mem/pool.h @@ -37,9 +37,9 @@ namespace snmalloc { public: template - static T* acquire(SharedStateHandle h, Args&&... args) + static T* acquire(Args&&... args) { - PoolState& pool = h.pool(); + PoolState& pool = SharedStateHandle::pool(); T* p = pool.stack.pop(); if (p != nullptr) @@ -48,12 +48,12 @@ namespace snmalloc return p; } - p = ChunkAllocator::alloc_meta_data( - h, nullptr, std::forward(args)...); + p = ChunkAllocator::alloc_meta_data( + nullptr, std::forward(args)...); if (p == nullptr) { - SharedStateHandle::Backend::Pal::error( + SharedStateHandle::Pal::error( "Failed to initialise thread local allocator."); } @@ -71,21 +71,21 @@ namespace snmalloc * Do not return objects from `extract`. */ template - static void release(SharedStateHandle h, T* p) + static void release(T* p) { // The object's destructor is not run. If the object is "reallocated", it // is returned without the constructor being run, so the object is reused // without re-initialisation. p->reset_in_use(); - h.pool().stack.push(p); + SharedStateHandle::pool().stack.push(p); } template - static T* extract(SharedStateHandle h, T* p = nullptr) + static T* extract(T* p = nullptr) { // Returns a linked list of all objects in the stack, emptying the stack. if (p == nullptr) - return h.pool().stack.pop_all(); + return SharedStateHandle::pool().stack.pop_all(); return p->next; } @@ -96,18 +96,18 @@ namespace snmalloc * Do not return objects from `acquire`. */ template - static void restore(SharedStateHandle h, T* first, T* last) + static void restore(T* first, T* last) { // Pushes a linked list of objects onto the stack. Use to put a linked // list returned by extract back onto the stack. - h.pool().stack.push(first, last); + SharedStateHandle::pool().stack.push(first, last); } template - static T* iterate(SharedStateHandle h, T* p = nullptr) + static T* iterate(T* p = nullptr) { if (p == nullptr) - return h.pool().list; + return SharedStateHandle::pool().list; return p->list_next; } diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index 25680b07a..a062b9a6f 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -76,10 +76,7 @@ namespace snmalloc } template - bool post( - SharedStateHandle handle, - RemoteAllocator::alloc_id_t id, - const FreeListKey& key) + bool post(RemoteAllocator::alloc_id_t id, const FreeListKey& key) { SNMALLOC_ASSERT(initialised); size_t post_round = 0; @@ -97,8 +94,8 @@ namespace snmalloc if (!list[i].empty()) { auto [first, last] = list[i].extract_segment(key); - MetaEntry entry = SharedStateHandle::Backend::get_meta_data( - handle.get_backend_state(), address_cast(first)); + MetaEntry entry = + SharedStateHandle::get_meta_data(address_cast(first)); entry.get_remote()->enqueue(first, last, key); sent_something = true; } @@ -120,8 +117,7 @@ namespace snmalloc // Use the next N bits to spread out remote deallocs in our own // slot. auto r = resend.take(key); - MetaEntry entry = SharedStateHandle::Backend::get_meta_data( - handle.get_backend_state(), address_cast(r)); + MetaEntry entry = SharedStateHandle::get_meta_data(address_cast(r)); auto i = entry.get_remote()->trunc_id(); size_t slot = get_slot(i, post_round); list[slot].add(r, key); diff --git a/src/mem/slaballocator.h b/src/mem/slaballocator.h index 30b3dafce..13dc161f2 100644 --- a/src/mem/slaballocator.h +++ b/src/mem/slaballocator.h @@ -75,14 +75,14 @@ namespace snmalloc public: template static std::pair, Metaslab*> alloc_chunk( - SharedStateHandle h, - typename SharedStateHandle::Backend::LocalState& backend_state, + typename SharedStateHandle::LocalState& local_state, sizeclass_t sizeclass, sizeclass_t slab_sizeclass, // TODO sizeclass_t size_t slab_size, RemoteAllocator* remote) { - ChunkAllocatorState& state = h.get_slab_allocator_state(); + ChunkAllocatorState& state = + SharedStateHandle::get_slab_allocator_state(&local_state); // Pop a slab auto chunk_record = state.chunk_stack[slab_sizeclass].pop(); @@ -98,15 +98,14 @@ namespace snmalloc << std::endl; #endif MetaEntry entry{meta, remote, sizeclass}; - SharedStateHandle::Backend::set_meta_data( - h.get_backend_state(), address_cast(slab), slab_size, entry); + SharedStateHandle::set_meta_data(address_cast(slab), slab_size, entry); return {slab, meta}; } // Allocate a fresh slab as there are no available ones. // First create meta-data - auto [slab, meta] = SharedStateHandle::Backend::alloc_chunk( - h.get_backend_state(), &backend_state, slab_size, remote, sizeclass); + auto [slab, meta] = SharedStateHandle::alloc_chunk( + &local_state, slab_size, remote, sizeclass); #ifdef SNMALLOC_TRACING std::cout << "Create slab:" << slab.unsafe_ptr() << " slab_sizeclass " << slab_sizeclass << " size " << slab_size << std::endl; @@ -122,10 +121,12 @@ namespace snmalloc } template - SNMALLOC_SLOW_PATH static void - dealloc(SharedStateHandle h, ChunkRecord* p, size_t slab_sizeclass) + SNMALLOC_SLOW_PATH static void dealloc( + typename SharedStateHandle::LocalState& local_state, + ChunkRecord* p, + size_t slab_sizeclass) { - auto& state = h.get_slab_allocator_state(); + auto& state = SharedStateHandle::get_slab_allocator_state(&local_state); #ifdef SNMALLOC_TRACING std::cout << "Return slab:" << p->chunk.unsafe_ptr() << " slab_sizeclass " << slab_sizeclass << " size " @@ -144,15 +145,13 @@ namespace snmalloc */ template static U* alloc_meta_data( - SharedStateHandle h, - typename SharedStateHandle::Backend::LocalState* local_state, - Args&&... args) + typename SharedStateHandle::LocalState* local_state, Args&&... args) { // Cache line align size_t size = bits::align_up(sizeof(U), 64); - CapPtr p = SharedStateHandle::Backend::alloc_meta_data( - h.get_backend_state(), local_state, size); + CapPtr p = + SharedStateHandle::template alloc_meta_data(local_state, size); if (p == nullptr) return nullptr; diff --git a/src/mem/threadalloc.h b/src/mem/threadalloc.h index 101e09230..dc9cee6f4 100644 --- a/src/mem/threadalloc.h +++ b/src/mem/threadalloc.h @@ -156,6 +156,6 @@ namespace snmalloc */ void _malloc_thread_cleanup() { - ThreadAlloc::get().teardown(); + snmalloc::ThreadAlloc::get().teardown(); } #endif diff --git a/src/override/malloc-extensions.cc b/src/override/malloc-extensions.cc index 9245cc923..d672e05be 100644 --- a/src/override/malloc-extensions.cc +++ b/src/override/malloc-extensions.cc @@ -6,10 +6,8 @@ using namespace snmalloc; void get_malloc_info_v1(malloc_info_v1* stats) { - auto unused_chunks = - Globals::get_handle().get_slab_allocator_state().unused_memory(); - auto peak = - Globals::get_handle().get_slab_allocator_state().peak_memory_usage(); + auto unused_chunks = Globals::get_slab_allocator_state().unused_memory(); + auto peak = Globals::get_slab_allocator_state().peak_memory_usage(); stats->current_memory_usage = peak - unused_chunks; stats->peak_memory_usage = peak; -} \ No newline at end of file +} diff --git a/src/override/malloc.cc b/src/override/malloc.cc index acd640276..0a9ad3630 100644 --- a/src/override/malloc.cc +++ b/src/override/malloc.cc @@ -2,7 +2,7 @@ #include "../snmalloc_core.h" #ifndef SNMALLOC_PROVIDE_OWN_CONFIG -# include "../mem/globalconfig.h" +# include "../backend/globalconfig.h" // The default configuration for snmalloc is used if alternative not defined namespace snmalloc { diff --git a/src/snmalloc.h b/src/snmalloc.h index 6abd703d6..e1d90d402 100644 --- a/src/snmalloc.h +++ b/src/snmalloc.h @@ -3,8 +3,13 @@ // Core implementation of snmalloc independent of the configuration mode #include "snmalloc_core.h" +// If you define SNMALLOC_PROVIDE_OWN_CONFIG then you must provide your own +// definition of `snmalloc::Alloc` and include `snmalloc_front.h` before +// including any files that include `snmalloc.h` and consume the global +// allocation APIs. +#ifndef SNMALLOC_PROVIDE_OWN_CONFIG // Default implementation of global state -#include "mem/globalconfig.h" +# include "backend/globalconfig.h" // The default configuration for snmalloc namespace snmalloc @@ -13,4 +18,5 @@ namespace snmalloc } // User facing API surface, needs to know what `Alloc` is. -#include "snmalloc_front.h" \ No newline at end of file +# include "snmalloc_front.h" +#endif diff --git a/src/snmalloc_core.h b/src/snmalloc_core.h index 0c6a2d336..d8b14aa03 100644 --- a/src/snmalloc_core.h +++ b/src/snmalloc_core.h @@ -1,3 +1,6 @@ #pragma once -#include "mem/globalalloc.h" \ No newline at end of file +#include "backend/address_space.h" +#include "backend/commonconfig.h" +#include "backend/pagemap.h" +#include "mem/globalalloc.h" diff --git a/src/test/func/fixed_region/fixed_region.cc b/src/test/func/fixed_region/fixed_region.cc index 1f0ca0bc6..db75ccf96 100644 --- a/src/test/func/fixed_region/fixed_region.cc +++ b/src/test/func/fixed_region/fixed_region.cc @@ -1,4 +1,4 @@ -#include "mem/fixedglobalconfig.h" +#include "backend/fixedglobalconfig.h" #include "test/setup.h" #include @@ -29,9 +29,8 @@ int main() std::cout << "Allocated region " << oe_base << " - " << pointer_offset(oe_base, size) << std::endl; - CustomGlobals fixed_handle; CustomGlobals::init(oe_base, size); - FixedAlloc a(fixed_handle); + FixedAlloc a; size_t object_size = 128; size_t count = 0; diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index e87e0eeed..c8fc74a8e 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -232,6 +232,6 @@ int main(int argc, char** argv) abort(); } - snmalloc::debug_check_empty(Globals::get_handle()); + snmalloc::debug_check_empty(); return 0; } diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index 9c6cd359a..f4afd8fc3 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -183,7 +183,7 @@ void test_calloc() alloc.dealloc(p, size); } - snmalloc::debug_check_empty(Globals::get_handle()); + snmalloc::debug_check_empty(); } void test_double_alloc() @@ -228,7 +228,7 @@ void test_double_alloc() } } } - snmalloc::debug_check_empty(Globals::get_handle()); + snmalloc::debug_check_empty(); } void test_external_pointer() @@ -264,7 +264,7 @@ void test_external_pointer() alloc.dealloc(p1, size); } - snmalloc::debug_check_empty(Globals::get_handle()); + snmalloc::debug_check_empty(); }; void check_offset(void* base, void* interior) diff --git a/src/test/func/statistics/stats.cc b/src/test/func/statistics/stats.cc index 36b47889d..156612c27 100644 --- a/src/test/func/statistics/stats.cc +++ b/src/test/func/statistics/stats.cc @@ -8,7 +8,7 @@ int main() auto r = a.alloc(16); - snmalloc::debug_check_empty(snmalloc::Globals::get_handle(), &result); + snmalloc::debug_check_empty(&result); if (result != false) { abort(); @@ -16,7 +16,7 @@ int main() a.dealloc(r); - snmalloc::debug_check_empty(snmalloc::Globals::get_handle(), &result); + snmalloc::debug_check_empty(&result); if (result != true) { abort(); @@ -24,7 +24,7 @@ int main() r = a.alloc(16); - snmalloc::debug_check_empty(snmalloc::Globals::get_handle(), &result); + snmalloc::debug_check_empty(&result); if (result != false) { abort(); @@ -32,10 +32,10 @@ int main() a.dealloc(r); - snmalloc::debug_check_empty(snmalloc::Globals::get_handle(), &result); + snmalloc::debug_check_empty(&result); if (result != true) { abort(); } #endif -} \ No newline at end of file +} diff --git a/src/test/func/thread_alloc_external/thread_alloc_external.cc b/src/test/func/thread_alloc_external/thread_alloc_external.cc index a5b1fe8d2..effb62fa0 100644 --- a/src/test/func/thread_alloc_external/thread_alloc_external.cc +++ b/src/test/func/thread_alloc_external/thread_alloc_external.cc @@ -4,7 +4,7 @@ // Specify using own #define SNMALLOC_EXTERNAL_THREAD_ALLOC -#include "mem/globalconfig.h" +#include "backend/globalconfig.h" namespace snmalloc { diff --git a/src/test/func/two_alloc_types/alloc1.cc b/src/test/func/two_alloc_types/alloc1.cc index 4e5f9d0bf..feb47bd09 100644 --- a/src/test/func/two_alloc_types/alloc1.cc +++ b/src/test/func/two_alloc_types/alloc1.cc @@ -3,7 +3,7 @@ // Redefine the namespace, so we can have two versions. #define snmalloc snmalloc_enclave -#include +#include #include // Specify type of allocator @@ -19,6 +19,5 @@ namespace snmalloc extern "C" void oe_allocator_init(void* base, void* end) { - snmalloc::CustomGlobals fixed_handle; - fixed_handle.init(base, address_cast(end) - address_cast(base)); + snmalloc::CustomGlobals::init(base, address_cast(end) - address_cast(base)); } diff --git a/src/test/perf/contention/contention.cc b/src/test/perf/contention/contention.cc index 5be5641c5..74f9cd200 100644 --- a/src/test/perf/contention/contention.cc +++ b/src/test/perf/contention/contention.cc @@ -142,7 +142,7 @@ void test_tasks(size_t num_tasks, size_t count, size_t size) } #ifndef NDEBUG - snmalloc::debug_check_empty(Globals::get_handle()); + snmalloc::debug_check_empty(); #endif }; diff --git a/src/test/perf/external_pointer/externalpointer.cc b/src/test/perf/external_pointer/externalpointer.cc index ac66a1350..90aa1bacb 100644 --- a/src/test/perf/external_pointer/externalpointer.cc +++ b/src/test/perf/external_pointer/externalpointer.cc @@ -47,7 +47,7 @@ namespace test alloc.dealloc(objects[i]); } - snmalloc::debug_check_empty(Globals::get_handle()); + snmalloc::debug_check_empty(); } void test_external_pointer(xoroshiro::p128r64& r) diff --git a/src/test/perf/singlethread/singlethread.cc b/src/test/perf/singlethread/singlethread.cc index 2bf49809c..cd15a12b8 100644 --- a/src/test/perf/singlethread/singlethread.cc +++ b/src/test/perf/singlethread/singlethread.cc @@ -60,7 +60,7 @@ void test_alloc_dealloc(size_t count, size_t size, bool write) } } - snmalloc::debug_check_empty(Globals::get_handle()); + snmalloc::debug_check_empty(); } int main(int, char**) From cd70a7856b06d20fbebdc53c563c03364367fa4a Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Thu, 5 Aug 2021 16:09:37 +0100 Subject: [PATCH 030/302] Fix fallout from the merge. - CI merge issues: - The malloc shim libraries are renamed. - CMake gets very unhappy if you don't enable the C language and tries to link with the C compiler instead of the C++ compiler if you do enable it. - The Ubuntu packages for QEMU install a `binfmt_misc` activator for PowerPC64 little-endian, but set the page size to 4 KiB. We then tried to run the tests (which expect 64 KiB pages) and became very confused when `mmap` returned 4 KiB-aligned memory. - Test failures: - Fix all of the issues UBsan found. - Underflow in `pointer_offset` when used to add negative offsets. - `CoreAlloc`'s `LocalState` accessed on a null `CoreAlloc` pointer. - Out of bounds access in the sizeclass list on attempts to access more memory than fits in the VA space. - - There was an integer overflow in `AddressSpace` that could cause it to try to allocate a zero-sized object, get a null pointer, and then try to do something with 0 - {size of the real allocation}. - The malloc tests weren't setting `errno` to 0 before doing calling `malloc`, which should set `errno` on failure, and then checking that `errno` was 0. - Don't call `PAL::error` on PAL allocation failure, return `nullptr`. The PALs were inconsistent about that and the new code expects to be able to report address-space exhaustion. - The malloc checks can behave differently with 0-sized allocations on different platforms but were very fragile about their expectations. - The malloc test didn't report failure for all of the ways that it could fail and so was spuriously passing on some platforms. - The perf test for external pointer is currently very slow on Windows. The number of loops have been reduced and a timeout added for the Windows CI runs. - The logic to capture `errno` across calls was using `decltype(errno)`, which on some platforms where `errno` is a macro evaluated to `int&` and so they captured a reference rather than the value and failed to reset `errno`. - The Apple PAL can set `errno` on `notify_using` if it's called with memory that was not previously passed to `notify_not_using` but was not adequately protected against this and so would sometimes cause `malloc` to set `errno` to `EINVAL`. --- .github/workflows/main.yml | 16 +++++++++++----- src/backend/address_space.h | 7 ++++++- src/ds/address.h | 3 ++- src/mem/localalloc.h | 7 +++++-- src/mem/slaballocator.h | 7 +++++++ src/pal/pal_apple.h | 3 ++- src/pal/pal_bsd_aligned.h | 2 +- src/pal/pal_posix.h | 3 ++- src/pal/pal_windows.h | 4 ---- src/test/func/malloc/malloc.cc | 15 ++++++++++----- .../perf/external_pointer/externalpointer.cc | 4 +++- 11 files changed, 49 insertions(+), 22 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9c63a7083..feabadeb7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -50,7 +50,7 @@ jobs: - os: ubuntu-latest variant: Clang 10 libstdc++ (Build only) dependencies: "sudo apt install ninja-build" - extra-cmake-flags: "-DCMAKE_CXX_COMPILER=clang++-10 -DCMAKE_CXX_FLAGS=-stdlib=libstdc++" + extra-cmake-flags: "-DCMAKE_CXX_COMPILER=clang++-10 -DCMAKE_C_COMPILER=clang-10 -DCMAKE_CXX_FLAGS=-stdlib=libstdc++" build-only: yes # Don't abort runners if a single one fails fail-fast: false @@ -78,13 +78,11 @@ jobs: if: ${{ matrix.self-host }} working-directory: ${{github.workspace}}/build run: | - sudo cp libsnmallocshim.so libsnmallocshim-16mib.so libsnmallocshim-oe.so /usr/local/lib/ + sudo cp libsnmallocshim.so libsnmallocshim-checks.so /usr/local/lib/ ninja clean LD_PRELOAD=/usr/local/lib/libsnmallocshim.so ninja ninja clean - LD_PRELOAD=/usr/local/lib/libsnmallocshim-16mib.so ninja - ninja clean - LD_PRELOAD=/usr/local/lib/libsnmallocshim-oe.so ninja + LD_PRELOAD=/usr/local/lib/libsnmallocshim-checks.so ninja qemu-crossbuild: strategy: @@ -117,6 +115,13 @@ jobs: sudo add-apt-repository "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main" sudo apt update sudo apt install libstdc++-9-dev-${{ matrix.arch.name }}-cross qemu-user ninja-build clang-13 lld-13 + # The default PowerPC qemu configuration uses the wrong page size. + # Wrap it in a script that fixes this. + sudo update-binfmts --disable qemu-ppc64le + sudo sh -c 'echo ":qemu-ppc64le:M:0:\x7f\x45\x4c\x46\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x15\x00:\xff\xff\xff\xff\xff\xff\xff\xfc\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\x00:`pwd`/ppc64.sh:" > /proc/sys/fs/binfmt_misc/register' + echo '#!/bin/sh' > ppc64.sh + echo '/usr/bin/qemu-ppc64le -p 65536 $@' >> ppc64.sh + chmod +x ppc64.sh - name: Configure run: > RTLD_NAME=${{ matrix.arch.rtld }} @@ -179,6 +184,7 @@ jobs: - name: Test working-directory: ${{ github.workspace }}/build run: ctest -j 2 --interactive-debug-mode 0 --output-on-failure -C ${{ matrix.build-type }} + timeout-minutes: 20 # Job to run clang-format and report errors diff --git a/src/backend/address_space.h b/src/backend/address_space.h index abccd83c2..fbb8be944 100644 --- a/src/backend/address_space.h +++ b/src/backend/address_space.h @@ -96,7 +96,12 @@ namespace snmalloc else if constexpr (!pal_supports) { // Need at least 2 times the space to guarantee alignment. - size_t needed_size = size * 2; + bool overflow; + size_t needed_size = bits::umul(size, 2, overflow); + if (overflow) + { + return nullptr; + } // Magic number (27) for over-allocating a block of memory // These should be further refined based on experiments. constexpr size_t min_size = bits::one_at_bit(27); diff --git a/src/ds/address.h b/src/ds/address.h index 15fc47bd1..bfa1ac534 100644 --- a/src/ds/address.h +++ b/src/ds/address.h @@ -31,7 +31,8 @@ namespace snmalloc inline U* pointer_offset(T* base, size_t diff) { SNMALLOC_ASSERT(base != nullptr); /* Avoid UB */ - return reinterpret_cast(reinterpret_cast(base) + diff); + return reinterpret_cast( + reinterpret_cast(base) + static_cast(diff)); } template diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index ad5807169..0d9e747eb 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -512,8 +512,11 @@ namespace snmalloc ChunkRecord* slab_record = reinterpret_cast(entry.get_metaslab()); slab_record->chunk = CapPtr(p); - ChunkAllocator::dealloc( - core_alloc->get_backend_local_state(), slab_record, slab_sizeclass); + check_init([&](CoreAlloc* core_alloc) { + ChunkAllocator::dealloc( + core_alloc->get_backend_local_state(), slab_record, slab_sizeclass); + return nullptr; + }); return; } diff --git a/src/mem/slaballocator.h b/src/mem/slaballocator.h index 13dc161f2..367509be5 100644 --- a/src/mem/slaballocator.h +++ b/src/mem/slaballocator.h @@ -83,6 +83,13 @@ namespace snmalloc { ChunkAllocatorState& state = SharedStateHandle::get_slab_allocator_state(&local_state); + + if (slab_sizeclass >= NUM_SLAB_SIZES) + { + // Your address space is not big enough for this allocation! + return {nullptr, nullptr}; + } + // Pop a slab auto chunk_record = state.chunk_stack[slab_sizeclass].pop(); diff --git a/src/pal/pal_apple.h b/src/pal/pal_apple.h index 0c24fa905..a36e10c91 100644 --- a/src/pal/pal_apple.h +++ b/src/pal/pal_apple.h @@ -160,6 +160,7 @@ namespace snmalloc template static void notify_using(void* p, size_t size) noexcept { + KeepErrno e; SNMALLOC_ASSERT( is_aligned_block(p, size) || (zero_mem == NoZero)); @@ -239,7 +240,7 @@ namespace snmalloc if (unlikely(kr != KERN_SUCCESS)) { - error("Failed to allocate memory\n"); + return nullptr; } return reinterpret_cast(addr); diff --git a/src/pal/pal_bsd_aligned.h b/src/pal/pal_bsd_aligned.h index c636e50f5..592b842bd 100644 --- a/src/pal/pal_bsd_aligned.h +++ b/src/pal/pal_bsd_aligned.h @@ -52,7 +52,7 @@ namespace snmalloc 0); if (p == MAP_FAILED) - PALBSD::error("Out of memory"); + return nullptr; return p; } diff --git a/src/pal/pal_posix.h b/src/pal/pal_posix.h index 8f9e2f707..ce932fdca 100644 --- a/src/pal/pal_posix.h +++ b/src/pal/pal_posix.h @@ -99,12 +99,13 @@ namespace snmalloc static const int fd = T::anonymous_memory_fd; }; + protected: /** * A RAII class to capture and restore errno */ class KeepErrno { - decltype(errno) cached_errno; + int cached_errno; public: KeepErrno() : cached_errno(errno) {} diff --git a/src/pal/pal_windows.h b/src/pal/pal_windows.h index 6fe35643f..98654d1a1 100644 --- a/src/pal/pal_windows.h +++ b/src/pal/pal_windows.h @@ -177,10 +177,6 @@ namespace snmalloc void* ret = VirtualAlloc2FromApp( nullptr, nullptr, size, flags, PAGE_READWRITE, ¶m, 1); - if (ret == nullptr) - { - error("Failed to allocate memory\n"); - } return ret; } # endif diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index c8fc74a8e..dfd92e2fa 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -41,7 +41,7 @@ void check_result(size_t size, size_t align, void* p, int err, bool null) #else const auto exact_size = align == 1; #endif - if (exact_size && (alloc_size != expected_size)) + if (exact_size && (alloc_size != expected_size) && (size != 0)) { printf( "Usable size is %zu, but required to be %zu.\n", @@ -79,7 +79,10 @@ void check_result(size_t size, size_t align, void* p, int err, bool null) } if (failed) + { printf("check_result failed! %p", p); + abort(); + } our_free(p); } @@ -149,7 +152,9 @@ int main(int argc, char** argv) { const size_t size = bits::one_at_bit(sc); printf("malloc: %zu\n", size); + errno = 0; check_result(size, 1, our_malloc(size), SUCCESS, false); + errno = 0; check_result(size + 1, 1, our_malloc(size + 1), SUCCESS, false); } @@ -180,7 +185,7 @@ int main(int argc, char** argv) test_realloc(our_malloc(size), size, SUCCESS, false); test_realloc(our_malloc(size), 0, SUCCESS, true); test_realloc(nullptr, size, SUCCESS, false); - test_realloc(our_malloc(size), (size_t)-1, ENOMEM, true); + test_realloc(our_malloc(size), ((size_t)-1) / 2, ENOMEM, true); for (sizeclass_t sc2 = 0; sc2 < NUM_SIZECLASSES; sc2++) { const size_t size2 = sizeclass_to_size(sc2); @@ -195,7 +200,7 @@ int main(int argc, char** argv) test_realloc(our_malloc(size), size, SUCCESS, false); test_realloc(our_malloc(size), 0, SUCCESS, true); test_realloc(nullptr, size, SUCCESS, false); - test_realloc(our_malloc(size), (size_t)-1, ENOMEM, true); + test_realloc(our_malloc(size), ((size_t)-1) / 2, ENOMEM, true); for (sizeclass_t sc2 = 0; sc2 < (MAX_SIZECLASS_BITS + 4); sc2++) { const size_t size2 = bits::one_at_bit(sc2); @@ -208,7 +213,7 @@ int main(int argc, char** argv) test_realloc(our_malloc(64), 4194304, SUCCESS, false); test_posix_memalign(0, 0, EINVAL, true); - test_posix_memalign((size_t)-1, 0, EINVAL, true); + test_posix_memalign(((size_t)-1) / 2, 0, EINVAL, true); test_posix_memalign(OS_PAGE_SIZE, sizeof(uintptr_t) / 2, EINVAL, true); for (size_t align = sizeof(uintptr_t); align < MAX_SIZECLASS_SIZE * 8; @@ -222,7 +227,7 @@ int main(int argc, char** argv) test_memalign(size, align, SUCCESS, false); } test_posix_memalign(0, align, SUCCESS, false); - test_posix_memalign((size_t)-1, align, ENOMEM, true); + test_posix_memalign(((size_t)-1) / 2, align, ENOMEM, true); test_posix_memalign(0, align + 1, EINVAL, true); } diff --git a/src/test/perf/external_pointer/externalpointer.cc b/src/test/perf/external_pointer/externalpointer.cc index 90aa1bacb..6a8616d77 100644 --- a/src/test/perf/external_pointer/externalpointer.cc +++ b/src/test/perf/external_pointer/externalpointer.cc @@ -53,7 +53,9 @@ namespace test void test_external_pointer(xoroshiro::p128r64& r) { auto& alloc = ThreadAlloc::get(); -#ifdef NDEBUG + // This is very slow on Windows at the moment. Until this is fixed, help + // CI terminate. +#if defined(NDEBUG) && !defined(_MSC_VER) static constexpr size_t iterations = 10000000; #else # ifdef _MSC_VER From 22e2b5b689ab9558ee71c370743f565ffde7977c Mon Sep 17 00:00:00 2001 From: Istvan Haller Date: Thu, 5 Aug 2021 18:42:17 +0100 Subject: [PATCH 031/302] Added reverse lookup to Pagemap --- src/backend/pagemap.h | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/backend/pagemap.h b/src/backend/pagemap.h index d640ae1b1..ccf262a07 100644 --- a/src/backend/pagemap.h +++ b/src/backend/pagemap.h @@ -185,6 +185,21 @@ namespace snmalloc body = new_body; } + /** + * Get the number of entries. + */ + constexpr size_t num_entries() const + { + if constexpr (has_bounds) + { + return size >> GRANULARITY_BITS; + } + else + { + return bits::one_at_bit(bits::ADDRESS_BITS - GRANULARITY_BITS); + } + } + /** * If the location has not been used before, then * `potentially_out_of_range` should be set to true. @@ -223,6 +238,21 @@ namespace snmalloc return body[p >> SHIFT]; } + /** + * Return the starting address corresponding to a given entry within the + * Pagemap. Also checks that the reference actually points to a valid entry. + */ + address_t get_address(const T& t) const + { + address_t entry_offset = address_cast(&t) - address_cast(body); + address_t entry_index = entry_offset / sizeof(T); + if (entry_offset % sizeof(T) != 0 || entry_index >= num_entries()) + { + PAL::error("Internal error: Invalid Pagemap entry for reverse lookup."); + } + return base + (entry_index << GRANULARITY_BITS); + } + void set(address_t p, T t) { #ifdef SNMALLOC_TRACING From df62abac5535660fdacdcc991255b3f28ea2a5e7 Mon Sep 17 00:00:00 2001 From: Istvan Haller Date: Fri, 6 Aug 2021 15:08:24 +0100 Subject: [PATCH 032/302] Added nodiscard --- src/backend/pagemap.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/pagemap.h b/src/backend/pagemap.h index ccf262a07..62c97fde3 100644 --- a/src/backend/pagemap.h +++ b/src/backend/pagemap.h @@ -188,7 +188,7 @@ namespace snmalloc /** * Get the number of entries. */ - constexpr size_t num_entries() const + [[nodiscard]] constexpr size_t num_entries() const { if constexpr (has_bounds) { @@ -242,7 +242,7 @@ namespace snmalloc * Return the starting address corresponding to a given entry within the * Pagemap. Also checks that the reference actually points to a valid entry. */ - address_t get_address(const T& t) const + [[nodiscard]] address_t get_address(const T& t) const { address_t entry_offset = address_cast(&t) - address_cast(body); address_t entry_index = entry_offset / sizeof(T); From e45354b3ed93876f64a8477883cefbca9932b810 Mon Sep 17 00:00:00 2001 From: Istvan Haller Date: Fri, 6 Aug 2021 16:19:35 +0100 Subject: [PATCH 033/302] Changed error handling to assert --- src/backend/pagemap.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/backend/pagemap.h b/src/backend/pagemap.h index 62c97fde3..4a923457d 100644 --- a/src/backend/pagemap.h +++ b/src/backend/pagemap.h @@ -246,10 +246,8 @@ namespace snmalloc { address_t entry_offset = address_cast(&t) - address_cast(body); address_t entry_index = entry_offset / sizeof(T); - if (entry_offset % sizeof(T) != 0 || entry_index >= num_entries()) - { - PAL::error("Internal error: Invalid Pagemap entry for reverse lookup."); - } + SNMALLOC_ASSERT( + entry_offset % sizeof(T) == 0 && entry_index < num_entries()); return base + (entry_index << GRANULARITY_BITS); } From f38a5a63d56400d2c6dedd035c2af0e5245b6c9c Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 13 Aug 2021 12:04:19 +0100 Subject: [PATCH 034/302] Make threshold for waking always a percentage The threshold for waking is used to ensure that we only use a slab when it has sufficient space that we won't hit the slow path to soon after using this slab. In the checked version, this is also used to give some entropy in layout. Changing this to always be a pertcentage in the checked case increases the effectiveness of the free list to detect OOB write. --- src/mem/sizeclasstable.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index 13c3f69ee..97f224d99 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -100,7 +100,11 @@ namespace snmalloc static_cast((slab_mask[sizeclass] + 1) / rsize); waking[sizeclass] = +#ifdef SNMALLOC_CHECK_CLIENT + static_cast(capacity[sizeclass] / 4); +#else static_cast(bits::min((capacity[sizeclass] / 4), 32)); +#endif } for (sizeclass_compress_t sizeclass = NUM_SIZECLASSES; From 7a02ae949f0b02b1ff9fdbf6eee8225c3eb58745 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Tue, 10 Aug 2021 23:58:50 +0100 Subject: [PATCH 035/302] Remove stale static_assert With snmalloc2, slabs are linked through the Metaslab structure directly rather than in-band in a free allocation, so we no longer need to store a SlabLink in even the smallest allocation classes. --- src/mem/metaslab.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index ff2da8791..41f73b10f 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -14,10 +14,6 @@ namespace snmalloc using SlabLink = CDLLNode<>; - static_assert( - sizeof(SlabLink) <= MIN_ALLOC_SIZE, - "Need to be able to pack a SlabLink into any free small alloc"); - // The Metaslab represent the status of a single slab. // This can be either a short or a standard slab. class alignas(CACHELINE_SIZE) Metaslab : public SlabLink From 4501c0ed813d23f20867d3070e30e16c52eb18b6 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Thu, 12 Aug 2021 12:01:36 +0100 Subject: [PATCH 036/302] corealloc: spurious sizeof() We mean to be allocating MIN_ALLOC_SIZE (== 2 * sizeof(void*)), not sizeof(MIN_ALLOC_SIZE) (== sizeof(size_t)). This doesn't matter in practice since, well, MIN_ALLOC_SIZE is the minimum allocation size, and so requesting either will have the same effect. Still, best to say what we mean. --- src/mem/corealloc.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 0097b1247..8c34844c7 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -146,9 +146,8 @@ namespace snmalloc { // Manufacture an allocation to prime the queue // Using an actual allocation removes a conditional from a critical path. - auto dummy = - CapPtr(small_alloc_one(sizeof(MIN_ALLOC_SIZE))) - .template as_static(); + auto dummy = CapPtr(small_alloc_one(MIN_ALLOC_SIZE)) + .template as_static(); if (dummy == nullptr) { error("Critical error: Out-of-memory during initialisation."); From f103f1c443bca1696227ea4336e7fb84df8090d0 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Mon, 19 Jul 2021 11:05:05 +0100 Subject: [PATCH 037/302] test/func/pagemap: use address_t for induction var Avoids a silent cast from uintptr_t to address_t. --- src/test/func/pagemap/pagemap.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/func/pagemap/pagemap.cc b/src/test/func/pagemap/pagemap.cc index 61a23f1dd..c5c82a930 100644 --- a/src/test/func/pagemap/pagemap.cc +++ b/src/test/func/pagemap/pagemap.cc @@ -90,7 +90,7 @@ void test_pagemap(bool bounded) // Store a pattern into page map T value = 1; - for (uintptr_t ptr = low; ptr < high; + for (address_t ptr = low; ptr < high; ptr += bits::one_at_bit(GRANULARITY_BITS + 3)) { set(bounded, ptr, value); @@ -104,7 +104,7 @@ void test_pagemap(bool bounded) // Check pattern is correctly stored std::cout << std::endl; value = 1; - for (uintptr_t ptr = low; ptr < high; + for (address_t ptr = low; ptr < high; ptr += bits::one_at_bit(GRANULARITY_BITS + 3)) { CHECK_GET(bounded, ptr, value); From 87e41559b264e545be1d8b8a43d06b9810ae2cb4 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Tue, 20 Jul 2021 11:39:47 +0100 Subject: [PATCH 038/302] test/func/memory: use pointer_offset to compute pointer offset This avoids the CHERI compiler warning that we're turning a provenance-free value to a pointer. Co-authored-by: Matthew Parkinson --- src/test/func/memory/memory.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index f4afd8fc3..cc2787c6f 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -255,7 +255,7 @@ void test_external_pointer() if ((size_t)p4 != (size_t)p1 + size - 1) { std::cout << "size: " << size << " end(p4): " << p4 << " p1: " << p1 - << " p1+size-1: " << (void*)((size_t)p1 + size - 1) + << " p1+size-1: " << pointer_offset(p1, size - 1) << std::endl; } SNMALLOC_CHECK((size_t)p4 == (size_t)p1 + size - 1); From c07f5ea7bec2b17b5489ce72fb15bb0ea59a4e43 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Tue, 20 Jul 2021 11:41:32 +0100 Subject: [PATCH 039/302] test/func/pagemap: avoid address_t to void* cast Instead, tell the iostream to write out hex. This avoids the CHERI compiler warning that we're turning a provenance-free value to a pointer. Co-authored-by: Matthew Parkinson --- src/test/func/pagemap/pagemap.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/func/pagemap/pagemap.cc b/src/test/func/pagemap/pagemap.cc index c5c82a930..d9be2838e 100644 --- a/src/test/func/pagemap/pagemap.cc +++ b/src/test/func/pagemap/pagemap.cc @@ -40,7 +40,7 @@ void check_get( if (value.v != expected.v) { - std::cout << "Location: " << (void*)address << " Read: " << value.v + std::cout << "Location: " << std::hex << address << " Read: " << value.v << " Expected: " << expected.v << " on " << file << ":" << lineno << std::endl; failure_count++; From 99f57646da42a9e9f03b186bbe3d1fb533a82e93 Mon Sep 17 00:00:00 2001 From: Istvan Haller Date: Thu, 19 Aug 2021 19:22:16 +0100 Subject: [PATCH 040/302] Re-enabled generic Pool which uses ChunkAllocator. Allocator pool renamed to AllocPool. --- src/mem/corealloc.h | 2 +- src/mem/globalalloc.h | 28 +++++++------- src/mem/localalloc.h | 9 +++-- src/mem/pool.h | 86 +++++++++++++++++++++++++++++++++++++++++-- src/mem/pooled.h | 4 +- 5 files changed, 107 insertions(+), 22 deletions(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 8c34844c7..467fa810f 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -4,7 +4,7 @@ #include "allocconfig.h" #include "localcache.h" #include "metaslab.h" -#include "pooled.h" +#include "pool.h" #include "remotecache.h" #include "sizeclasstable.h" #include "slaballocator.h" diff --git a/src/mem/globalalloc.h b/src/mem/globalalloc.h index cc4ecd53b..c44de9343 100644 --- a/src/mem/globalalloc.h +++ b/src/mem/globalalloc.h @@ -11,7 +11,7 @@ namespace snmalloc static_assert( SharedStateHandle::Options.CoreAllocIsPoolAllocated, "Global statistics are available only for pool-allocated configurations"); - auto* alloc = Pool>::template iterate< + auto* alloc = AllocPool>::template iterate< SharedStateHandle>(); while (alloc != nullptr) @@ -20,7 +20,7 @@ namespace snmalloc if (a != nullptr) stats.add(*a); stats.add(alloc->stats()); - alloc = Pool>::template iterate< + alloc = AllocPool>::template iterate< SharedStateHandle>(alloc); } } @@ -32,7 +32,7 @@ namespace snmalloc static_assert( SharedStateHandle::Options.CoreAllocIsPoolAllocated, "Global statistics are available only for pool-allocated configurations"); - auto alloc = Pool>::template iterate< + auto alloc = AllocPool>::template iterate< SharedStateHandle>(); while (alloc != nullptr) @@ -40,7 +40,7 @@ namespace snmalloc auto stats = alloc->stats(); if (stats != nullptr) stats->template print(o, dumpid, alloc->id()); - alloc = Pool>::template iterate< + alloc = AllocPool>::template iterate< SharedStateHandle>(alloc); } } @@ -64,7 +64,7 @@ namespace snmalloc // allocators that are not currently in use by any thread. // One atomic operation to extract the stack, another to restore it. // Handling the message queue for each stack is non-atomic. - auto* first = Pool>::extract(); + auto* first = AllocPool>::extract(); auto* alloc = first; decltype(alloc) last; @@ -74,10 +74,10 @@ namespace snmalloc { alloc->flush(); last = alloc; - alloc = Pool>::extract(alloc); + alloc = AllocPool>::extract(alloc); } - Pool>::restore(first, last); + AllocPool>::restore(first, last); } #endif } @@ -96,7 +96,7 @@ namespace snmalloc "Global status is available only for pool-allocated configurations"); // This is a debugging function. It checks that all memory from all // allocators has been freed. - auto* alloc = Pool>::template iterate< + auto* alloc = AllocPool>::template iterate< SharedStateHandle>(); # ifdef SNMALLOC_TRACING @@ -111,7 +111,7 @@ namespace snmalloc std::cout << "debug_check_empty: Check all allocators!" << std::endl; # endif done = true; - alloc = Pool>::template iterate< + alloc = AllocPool>::template iterate< SharedStateHandle>(); okay = true; @@ -134,7 +134,7 @@ namespace snmalloc # ifdef SNMALLOC_TRACING std::cout << "debug check empty: okay = " << okay << std::endl; # endif - alloc = Pool>::template iterate< + alloc = AllocPool>::template iterate< SharedStateHandle>(alloc); } } @@ -148,12 +148,12 @@ namespace snmalloc // Redo check so abort is on allocator with allocation left. if (!okay) { - alloc = Pool>::template iterate< + alloc = AllocPool>::template iterate< SharedStateHandle>(); while (alloc != nullptr) { alloc->debug_is_empty(nullptr); - alloc = Pool>::template iterate< + alloc = AllocPool>::template iterate< SharedStateHandle>(alloc); } } @@ -168,7 +168,7 @@ namespace snmalloc static_assert( SharedStateHandle::Options.CoreAllocIsPoolAllocated, "Global status is available only for pool-allocated configurations"); - auto alloc = Pool>::template iterate< + auto alloc = AllocPool>::template iterate< SharedStateHandle>(); while (alloc != nullptr) { @@ -180,7 +180,7 @@ namespace snmalloc } count--; } - alloc = Pool>::template iterate< + alloc = AllocPool>::template iterate< SharedStateHandle>(alloc); if (count != 0) diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 0d9e747eb..357e897fb 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -58,9 +58,12 @@ namespace snmalloc template class LocalAllocator { - using CoreAlloc = CoreAllocator; + public: + using StateHandle = SharedStateHandle; private: + using CoreAlloc = CoreAllocator; + // Free list per small size class. These are used for // allocation on the fast path. This part of the code is inspired by // mimalloc. @@ -375,7 +378,7 @@ namespace snmalloc // Initialise the global allocator structures ensure_init(); // Grab an allocator for this thread. - init(Pool::template acquire( + init(AllocPool::template acquire( &(this->local_cache))); } @@ -398,7 +401,7 @@ namespace snmalloc // Return underlying allocator to the system. if constexpr (SharedStateHandle::Options.CoreAllocOwnsLocalState) { - Pool::template release(core_alloc); + AllocPool::template release(core_alloc); } // Set up thread local allocator to look like diff --git a/src/mem/pool.h b/src/mem/pool.h index d3c25b4f9..49e3451d0 100644 --- a/src/mem/pool.h +++ b/src/mem/pool.h @@ -4,11 +4,12 @@ #include "../ds/mpmcstack.h" #include "../pal/pal_concept.h" #include "pooled.h" +#include "slaballocator.h" namespace snmalloc { /** - * Pool of a particular type of object. + * Pool of Allocators. * * This pool will never return objects to the OS. It maintains a list of all * objects ever allocated that can be iterated (not concurrency safe). Pooled @@ -20,8 +21,10 @@ namespace snmalloc template class PoolState { - template + template friend class Pool; + template + friend class AllocPool; private: std::atomic_flag lock = ATOMIC_FLAG_INIT; @@ -32,8 +35,85 @@ namespace snmalloc constexpr PoolState() = default; }; - template + template class Pool + { + PoolState state; + + public: + static Pool* make() noexcept + { + return ChunkAllocator::alloc_meta_data(nullptr); + } + + template + T* acquire(Args&&... args) + { + T* p = state.stack.pop(); + + if (p != nullptr) + { + p->set_in_use(); + return p; + } + + p = ChunkAllocator::alloc_meta_data( + nullptr, std::forward(args)...); + + FlagLock f(state.lock); + p->list_next = state.list; + state.list = p; + + p->set_in_use(); + return p; + } + + /** + * Return to the pool an object previously retrieved by `acquire` + * + * Do not return objects from `extract`. + */ + void release(T* p) + { + // The object's destructor is not run. If the object is "reallocated", it + // is returned without the constructor being run, so the object is reused + // without re-initialisation. + p->reset_in_use(); + state.stack.push(p); + } + + T* extract(T* p = nullptr) + { + // Returns a linked list of all objects in the stack, emptying the stack. + if (p == nullptr) + return state.stack.pop_all(); + + return p->next; + } + + /** + * Return to the pool a list of object previously retrieved by `extract` + * + * Do not return objects from `acquire`. + */ + void restore(T* first, T* last) + { + // Pushes a linked list of objects onto the stack. Use to put a linked + // list returned by extract back onto the stack. + state.stack.push(first, last); + } + + T* iterate(T* p = nullptr) + { + if (p == nullptr) + return state.list; + + return p->list_next; + } + }; + + template + class AllocPool { public: template diff --git a/src/mem/pooled.h b/src/mem/pooled.h index 93daad9f3..b6853571e 100644 --- a/src/mem/pooled.h +++ b/src/mem/pooled.h @@ -8,8 +8,10 @@ namespace snmalloc class Pooled { private: - template + template friend class Pool; + template + friend class AllocPool; template friend class MPMCStack; From 2d4a4caab7263de85b75718f83638febe09989c5 Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Mon, 23 Aug 2021 11:15:11 +0100 Subject: [PATCH 041/302] Fix the type checker. --- src/mem/localalloc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 0d9e747eb..39255cc9b 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -117,7 +117,7 @@ namespace snmalloc "initialisation"); // Unreachable, but needed to keep the type checker happy in deducing // the return type of this function. - return static_cast(nullptr); + return static_cast(nullptr); } else { From 4d2bf93b7a5e219ef2b394bfbd6cd08c407fcb5a Mon Sep 17 00:00:00 2001 From: Istvan Haller Date: Mon, 23 Aug 2021 16:33:48 +0100 Subject: [PATCH 042/302] Deleted the ability to implicitly copy LocalAllocator --- src/mem/localalloc.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 357e897fb..0ae59f183 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -344,6 +344,8 @@ namespace snmalloc public: constexpr LocalAllocator() = default; + LocalAllocator(const LocalAllocator&) = delete; + LocalAllocator& operator= (const LocalAllocator&) = delete; /** * Initialise the allocator. For allocators that support local From 35c94229137fc5184d64b81e3d55a4c279e14ebe Mon Sep 17 00:00:00 2001 From: Istvan Haller Date: Mon, 23 Aug 2021 19:19:58 +0100 Subject: [PATCH 043/302] Comment fixes --- src/mem/localalloc.h | 2 +- src/mem/pool.h | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 0ae59f183..19a5b9cbb 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -345,7 +345,7 @@ namespace snmalloc public: constexpr LocalAllocator() = default; LocalAllocator(const LocalAllocator&) = delete; - LocalAllocator& operator= (const LocalAllocator&) = delete; + LocalAllocator& operator=(const LocalAllocator&) = delete; /** * Initialise the allocator. For allocators that support local diff --git a/src/mem/pool.h b/src/mem/pool.h index 49e3451d0..2b0c1774d 100644 --- a/src/mem/pool.h +++ b/src/mem/pool.h @@ -9,7 +9,7 @@ namespace snmalloc { /** - * Pool of Allocators. + * Pool of a particular type of object. * * This pool will never return objects to the OS. It maintains a list of all * objects ever allocated that can be iterated (not concurrency safe). Pooled @@ -112,6 +112,11 @@ namespace snmalloc } }; + /** + * Collection of static wrappers for the allocator pool. + * The PoolState for this particular pool type is owned by the + * SharedStateHandle, so there is no object state in this class. + */ template class AllocPool { From c89a085c90347e3d64538cba9b117f031f376b13 Mon Sep 17 00:00:00 2001 From: Istvan Haller Date: Mon, 23 Aug 2021 19:54:26 +0100 Subject: [PATCH 044/302] Removed code duplication by making generoc Pool also be static --- src/mem/globalalloc.h | 28 +++++++------- src/mem/localalloc.h | 4 +- src/mem/pool.h | 90 +++++++------------------------------------ src/mem/pooled.h | 4 +- 4 files changed, 31 insertions(+), 95 deletions(-) diff --git a/src/mem/globalalloc.h b/src/mem/globalalloc.h index c44de9343..cc4ecd53b 100644 --- a/src/mem/globalalloc.h +++ b/src/mem/globalalloc.h @@ -11,7 +11,7 @@ namespace snmalloc static_assert( SharedStateHandle::Options.CoreAllocIsPoolAllocated, "Global statistics are available only for pool-allocated configurations"); - auto* alloc = AllocPool>::template iterate< + auto* alloc = Pool>::template iterate< SharedStateHandle>(); while (alloc != nullptr) @@ -20,7 +20,7 @@ namespace snmalloc if (a != nullptr) stats.add(*a); stats.add(alloc->stats()); - alloc = AllocPool>::template iterate< + alloc = Pool>::template iterate< SharedStateHandle>(alloc); } } @@ -32,7 +32,7 @@ namespace snmalloc static_assert( SharedStateHandle::Options.CoreAllocIsPoolAllocated, "Global statistics are available only for pool-allocated configurations"); - auto alloc = AllocPool>::template iterate< + auto alloc = Pool>::template iterate< SharedStateHandle>(); while (alloc != nullptr) @@ -40,7 +40,7 @@ namespace snmalloc auto stats = alloc->stats(); if (stats != nullptr) stats->template print(o, dumpid, alloc->id()); - alloc = AllocPool>::template iterate< + alloc = Pool>::template iterate< SharedStateHandle>(alloc); } } @@ -64,7 +64,7 @@ namespace snmalloc // allocators that are not currently in use by any thread. // One atomic operation to extract the stack, another to restore it. // Handling the message queue for each stack is non-atomic. - auto* first = AllocPool>::extract(); + auto* first = Pool>::extract(); auto* alloc = first; decltype(alloc) last; @@ -74,10 +74,10 @@ namespace snmalloc { alloc->flush(); last = alloc; - alloc = AllocPool>::extract(alloc); + alloc = Pool>::extract(alloc); } - AllocPool>::restore(first, last); + Pool>::restore(first, last); } #endif } @@ -96,7 +96,7 @@ namespace snmalloc "Global status is available only for pool-allocated configurations"); // This is a debugging function. It checks that all memory from all // allocators has been freed. - auto* alloc = AllocPool>::template iterate< + auto* alloc = Pool>::template iterate< SharedStateHandle>(); # ifdef SNMALLOC_TRACING @@ -111,7 +111,7 @@ namespace snmalloc std::cout << "debug_check_empty: Check all allocators!" << std::endl; # endif done = true; - alloc = AllocPool>::template iterate< + alloc = Pool>::template iterate< SharedStateHandle>(); okay = true; @@ -134,7 +134,7 @@ namespace snmalloc # ifdef SNMALLOC_TRACING std::cout << "debug check empty: okay = " << okay << std::endl; # endif - alloc = AllocPool>::template iterate< + alloc = Pool>::template iterate< SharedStateHandle>(alloc); } } @@ -148,12 +148,12 @@ namespace snmalloc // Redo check so abort is on allocator with allocation left. if (!okay) { - alloc = AllocPool>::template iterate< + alloc = Pool>::template iterate< SharedStateHandle>(); while (alloc != nullptr) { alloc->debug_is_empty(nullptr); - alloc = AllocPool>::template iterate< + alloc = Pool>::template iterate< SharedStateHandle>(alloc); } } @@ -168,7 +168,7 @@ namespace snmalloc static_assert( SharedStateHandle::Options.CoreAllocIsPoolAllocated, "Global status is available only for pool-allocated configurations"); - auto alloc = AllocPool>::template iterate< + auto alloc = Pool>::template iterate< SharedStateHandle>(); while (alloc != nullptr) { @@ -180,7 +180,7 @@ namespace snmalloc } count--; } - alloc = AllocPool>::template iterate< + alloc = Pool>::template iterate< SharedStateHandle>(alloc); if (count != 0) diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 19a5b9cbb..7b1c91255 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -380,7 +380,7 @@ namespace snmalloc // Initialise the global allocator structures ensure_init(); // Grab an allocator for this thread. - init(AllocPool::template acquire( + init(Pool::template acquire( &(this->local_cache))); } @@ -403,7 +403,7 @@ namespace snmalloc // Return underlying allocator to the system. if constexpr (SharedStateHandle::Options.CoreAllocOwnsLocalState) { - AllocPool::template release(core_alloc); + Pool::template release(core_alloc); } // Set up thread local allocator to look like diff --git a/src/mem/pool.h b/src/mem/pool.h index 2b0c1774d..990c849e3 100644 --- a/src/mem/pool.h +++ b/src/mem/pool.h @@ -21,10 +21,8 @@ namespace snmalloc template class PoolState { - template - friend class Pool; template - friend class AllocPool; + friend class Pool; private: std::atomic_flag lock = ATOMIC_FLAG_INIT; @@ -35,90 +33,30 @@ namespace snmalloc constexpr PoolState() = default; }; + /** + * Class used to instantiate non-allocator pools using a Singleton PoolState. + */ template - class Pool + class PoolStateHandle { - PoolState state; - - public: - static Pool* make() noexcept - { - return ChunkAllocator::alloc_meta_data(nullptr); - } - - template - T* acquire(Args&&... args) - { - T* p = state.stack.pop(); - - if (p != nullptr) - { - p->set_in_use(); - return p; - } - - p = ChunkAllocator::alloc_meta_data( - nullptr, std::forward(args)...); - - FlagLock f(state.lock); - p->list_next = state.list; - state.list = p; - - p->set_in_use(); - return p; - } - - /** - * Return to the pool an object previously retrieved by `acquire` - * - * Do not return objects from `extract`. - */ - void release(T* p) - { - // The object's destructor is not run. If the object is "reallocated", it - // is returned without the constructor being run, so the object is reused - // without re-initialisation. - p->reset_in_use(); - state.stack.push(p); - } + static PoolState& pool_state = + Singleton, PoolStateHandle::make_state>::get(); - T* extract(T* p = nullptr) + static PoolState* make_state() { - // Returns a linked list of all objects in the stack, emptying the stack. - if (p == nullptr) - return state.stack.pop_all(); - - return p->next; + return ChunkAllocator::alloc_meta_data, SharedStateHandle>( + nullptr); } - /** - * Return to the pool a list of object previously retrieved by `extract` - * - * Do not return objects from `acquire`. - */ - void restore(T* first, T* last) - { - // Pushes a linked list of objects onto the stack. Use to put a linked - // list returned by extract back onto the stack. - state.stack.push(first, last); - } - - T* iterate(T* p = nullptr) + public: + static PoolState& pool() { - if (p == nullptr) - return state.list; - - return p->list_next; + return pool_state; } }; - /** - * Collection of static wrappers for the allocator pool. - * The PoolState for this particular pool type is owned by the - * SharedStateHandle, so there is no object state in this class. - */ template - class AllocPool + class Pool { public: template diff --git a/src/mem/pooled.h b/src/mem/pooled.h index b6853571e..93daad9f3 100644 --- a/src/mem/pooled.h +++ b/src/mem/pooled.h @@ -8,10 +8,8 @@ namespace snmalloc class Pooled { private: - template - friend class Pool; template - friend class AllocPool; + friend class Pool; template friend class MPMCStack; From 769c61e71645b8e68c6d8a34afd0de88d4496efa Mon Sep 17 00:00:00 2001 From: Istvan Haller Date: Mon, 23 Aug 2021 20:08:27 +0100 Subject: [PATCH 045/302] Moved SharedStateHandle to Pool class instead of methods since all of them use it --- src/mem/globalalloc.h | 59 +++++++++++++++++++++++++------------------ src/mem/localalloc.h | 5 ++-- src/mem/pool.h | 10 +++----- src/mem/pooled.h | 2 +- 4 files changed, 40 insertions(+), 36 deletions(-) diff --git a/src/mem/globalalloc.h b/src/mem/globalalloc.h index cc4ecd53b..6f5aaad36 100644 --- a/src/mem/globalalloc.h +++ b/src/mem/globalalloc.h @@ -11,8 +11,8 @@ namespace snmalloc static_assert( SharedStateHandle::Options.CoreAllocIsPoolAllocated, "Global statistics are available only for pool-allocated configurations"); - auto* alloc = Pool>::template iterate< - SharedStateHandle>(); + auto* alloc = + Pool, SharedStateHandle>::iterate(); while (alloc != nullptr) { @@ -20,8 +20,9 @@ namespace snmalloc if (a != nullptr) stats.add(*a); stats.add(alloc->stats()); - alloc = Pool>::template iterate< - SharedStateHandle>(alloc); + alloc = + Pool, SharedStateHandle>::iterate( + alloc); } } @@ -32,16 +33,17 @@ namespace snmalloc static_assert( SharedStateHandle::Options.CoreAllocIsPoolAllocated, "Global statistics are available only for pool-allocated configurations"); - auto alloc = Pool>::template iterate< - SharedStateHandle>(); + auto alloc = + Pool, SharedStateHandle>::iterate(); while (alloc != nullptr) { auto stats = alloc->stats(); if (stats != nullptr) stats->template print(o, dumpid, alloc->id()); - alloc = Pool>::template iterate< - SharedStateHandle>(alloc); + alloc = + Pool, SharedStateHandle>::iterate( + alloc); } } #else @@ -64,7 +66,8 @@ namespace snmalloc // allocators that are not currently in use by any thread. // One atomic operation to extract the stack, another to restore it. // Handling the message queue for each stack is non-atomic. - auto* first = Pool>::extract(); + auto* first = + Pool, SharedStateHandle>::extract(); auto* alloc = first; decltype(alloc) last; @@ -74,10 +77,13 @@ namespace snmalloc { alloc->flush(); last = alloc; - alloc = Pool>::extract(alloc); + alloc = + Pool, SharedStateHandle>::extract( + alloc); } - Pool>::restore(first, last); + Pool, SharedStateHandle>::restore( + first, last); } #endif } @@ -96,8 +102,8 @@ namespace snmalloc "Global status is available only for pool-allocated configurations"); // This is a debugging function. It checks that all memory from all // allocators has been freed. - auto* alloc = Pool>::template iterate< - SharedStateHandle>(); + auto* alloc = + Pool, SharedStateHandle>::iterate(); # ifdef SNMALLOC_TRACING std::cout << "debug check empty: first " << alloc << std::endl; @@ -111,8 +117,8 @@ namespace snmalloc std::cout << "debug_check_empty: Check all allocators!" << std::endl; # endif done = true; - alloc = Pool>::template iterate< - SharedStateHandle>(); + alloc = + Pool, SharedStateHandle>::iterate(); okay = true; while (alloc != nullptr) @@ -134,8 +140,9 @@ namespace snmalloc # ifdef SNMALLOC_TRACING std::cout << "debug check empty: okay = " << okay << std::endl; # endif - alloc = Pool>::template iterate< - SharedStateHandle>(alloc); + alloc = + Pool, SharedStateHandle>::iterate( + alloc); } } @@ -148,13 +155,14 @@ namespace snmalloc // Redo check so abort is on allocator with allocation left. if (!okay) { - alloc = Pool>::template iterate< - SharedStateHandle>(); + alloc = + Pool, SharedStateHandle>::iterate(); while (alloc != nullptr) { alloc->debug_is_empty(nullptr); - alloc = Pool>::template iterate< - SharedStateHandle>(alloc); + alloc = + Pool, SharedStateHandle>::iterate( + alloc); } } #else @@ -168,8 +176,8 @@ namespace snmalloc static_assert( SharedStateHandle::Options.CoreAllocIsPoolAllocated, "Global status is available only for pool-allocated configurations"); - auto alloc = Pool>::template iterate< - SharedStateHandle>(); + auto alloc = + Pool, SharedStateHandle>::iterate(); while (alloc != nullptr) { if (alloc->debug_is_in_use()) @@ -180,8 +188,9 @@ namespace snmalloc } count--; } - alloc = Pool>::template iterate< - SharedStateHandle>(alloc); + alloc = + Pool, SharedStateHandle>::iterate( + alloc); if (count != 0) { diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 7b1c91255..96c1e5633 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -380,8 +380,7 @@ namespace snmalloc // Initialise the global allocator structures ensure_init(); // Grab an allocator for this thread. - init(Pool::template acquire( - &(this->local_cache))); + init(Pool::acquire(&(this->local_cache))); } // Return all state in the fast allocator and release the underlying @@ -403,7 +402,7 @@ namespace snmalloc // Return underlying allocator to the system. if constexpr (SharedStateHandle::Options.CoreAllocOwnsLocalState) { - Pool::template release(core_alloc); + Pool::release(core_alloc); } // Set up thread local allocator to look like diff --git a/src/mem/pool.h b/src/mem/pool.h index 990c849e3..184abca48 100644 --- a/src/mem/pool.h +++ b/src/mem/pool.h @@ -21,7 +21,7 @@ namespace snmalloc template class PoolState { - template + template friend class Pool; private: @@ -55,11 +55,11 @@ namespace snmalloc } }; - template + template class Pool { public: - template + template static T* acquire(Args&&... args) { PoolState& pool = SharedStateHandle::pool(); @@ -93,7 +93,6 @@ namespace snmalloc * * Do not return objects from `extract`. */ - template static void release(T* p) { // The object's destructor is not run. If the object is "reallocated", it @@ -103,7 +102,6 @@ namespace snmalloc SharedStateHandle::pool().stack.push(p); } - template static T* extract(T* p = nullptr) { // Returns a linked list of all objects in the stack, emptying the stack. @@ -118,7 +116,6 @@ namespace snmalloc * * Do not return objects from `acquire`. */ - template static void restore(T* first, T* last) { // Pushes a linked list of objects onto the stack. Use to put a linked @@ -126,7 +123,6 @@ namespace snmalloc SharedStateHandle::pool().stack.push(first, last); } - template static T* iterate(T* p = nullptr) { if (p == nullptr) diff --git a/src/mem/pooled.h b/src/mem/pooled.h index 93daad9f3..657ea2d32 100644 --- a/src/mem/pooled.h +++ b/src/mem/pooled.h @@ -8,7 +8,7 @@ namespace snmalloc class Pooled { private: - template + template friend class Pool; template friend class MPMCStack; From c01a1215c639295ff7ef5346b001da80dc0e072e Mon Sep 17 00:00:00 2001 From: Istvan Haller Date: Mon, 23 Aug 2021 21:07:51 +0100 Subject: [PATCH 046/302] Cleanup and made new variant work with Verona --- src/mem/corealloc.h | 6 ++++++ src/mem/globalalloc.h | 48 +++++++++++++------------------------------ src/mem/localalloc.h | 4 ++-- src/mem/pool.h | 34 ++++++++++++++---------------- src/mem/pooled.h | 10 +++++++-- 5 files changed, 45 insertions(+), 57 deletions(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 467fa810f..30dba5365 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -785,4 +785,10 @@ namespace snmalloc return debug_is_empty_impl(result); } }; + + template + using AllocPool = Pool< + CoreAllocator, + SharedStateHandle, + SharedStateHandle::pool>; } // namespace snmalloc diff --git a/src/mem/globalalloc.h b/src/mem/globalalloc.h index 6f5aaad36..674b7aeb5 100644 --- a/src/mem/globalalloc.h +++ b/src/mem/globalalloc.h @@ -11,8 +11,7 @@ namespace snmalloc static_assert( SharedStateHandle::Options.CoreAllocIsPoolAllocated, "Global statistics are available only for pool-allocated configurations"); - auto* alloc = - Pool, SharedStateHandle>::iterate(); + auto* alloc = AllocPool::iterate(); while (alloc != nullptr) { @@ -20,9 +19,7 @@ namespace snmalloc if (a != nullptr) stats.add(*a); stats.add(alloc->stats()); - alloc = - Pool, SharedStateHandle>::iterate( - alloc); + alloc = AllocPool::iterate(alloc); } } @@ -33,17 +30,14 @@ namespace snmalloc static_assert( SharedStateHandle::Options.CoreAllocIsPoolAllocated, "Global statistics are available only for pool-allocated configurations"); - auto alloc = - Pool, SharedStateHandle>::iterate(); + auto alloc = AllocPool::iterate(); while (alloc != nullptr) { auto stats = alloc->stats(); if (stats != nullptr) stats->template print(o, dumpid, alloc->id()); - alloc = - Pool, SharedStateHandle>::iterate( - alloc); + alloc = AllocPool::iterate(alloc); } } #else @@ -66,8 +60,7 @@ namespace snmalloc // allocators that are not currently in use by any thread. // One atomic operation to extract the stack, another to restore it. // Handling the message queue for each stack is non-atomic. - auto* first = - Pool, SharedStateHandle>::extract(); + auto* first = AllocPool::extract(); auto* alloc = first; decltype(alloc) last; @@ -77,13 +70,10 @@ namespace snmalloc { alloc->flush(); last = alloc; - alloc = - Pool, SharedStateHandle>::extract( - alloc); + alloc = AllocPool::extract(alloc); } - Pool, SharedStateHandle>::restore( - first, last); + AllocPool::restore(first, last); } #endif } @@ -102,8 +92,7 @@ namespace snmalloc "Global status is available only for pool-allocated configurations"); // This is a debugging function. It checks that all memory from all // allocators has been freed. - auto* alloc = - Pool, SharedStateHandle>::iterate(); + auto* alloc = AllocPool::iterate(); # ifdef SNMALLOC_TRACING std::cout << "debug check empty: first " << alloc << std::endl; @@ -117,8 +106,7 @@ namespace snmalloc std::cout << "debug_check_empty: Check all allocators!" << std::endl; # endif done = true; - alloc = - Pool, SharedStateHandle>::iterate(); + alloc = AllocPool::iterate(); okay = true; while (alloc != nullptr) @@ -140,9 +128,7 @@ namespace snmalloc # ifdef SNMALLOC_TRACING std::cout << "debug check empty: okay = " << okay << std::endl; # endif - alloc = - Pool, SharedStateHandle>::iterate( - alloc); + alloc = AllocPool::iterate(alloc); } } @@ -155,14 +141,11 @@ namespace snmalloc // Redo check so abort is on allocator with allocation left. if (!okay) { - alloc = - Pool, SharedStateHandle>::iterate(); + alloc = AllocPool::iterate(); while (alloc != nullptr) { alloc->debug_is_empty(nullptr); - alloc = - Pool, SharedStateHandle>::iterate( - alloc); + alloc = AllocPool::iterate(alloc); } } #else @@ -176,8 +159,7 @@ namespace snmalloc static_assert( SharedStateHandle::Options.CoreAllocIsPoolAllocated, "Global status is available only for pool-allocated configurations"); - auto alloc = - Pool, SharedStateHandle>::iterate(); + auto alloc = AllocPool::iterate(); while (alloc != nullptr) { if (alloc->debug_is_in_use()) @@ -188,9 +170,7 @@ namespace snmalloc } count--; } - alloc = - Pool, SharedStateHandle>::iterate( - alloc); + alloc = AllocPool::iterate(alloc); if (count != 0) { diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 96c1e5633..03161fa9e 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -380,7 +380,7 @@ namespace snmalloc // Initialise the global allocator structures ensure_init(); // Grab an allocator for this thread. - init(Pool::acquire(&(this->local_cache))); + init(AllocPool::acquire(&(this->local_cache))); } // Return all state in the fast allocator and release the underlying @@ -402,7 +402,7 @@ namespace snmalloc // Return underlying allocator to the system. if constexpr (SharedStateHandle::Options.CoreAllocOwnsLocalState) { - Pool::release(core_alloc); + AllocPool::release(core_alloc); } // Set up thread local allocator to look like diff --git a/src/mem/pool.h b/src/mem/pool.h index 184abca48..76950c2dc 100644 --- a/src/mem/pool.h +++ b/src/mem/pool.h @@ -21,7 +21,10 @@ namespace snmalloc template class PoolState { - template + template< + typename TT, + typename SharedStateHandle, + PoolState& get_state()> friend class Pool; private: @@ -34,35 +37,28 @@ namespace snmalloc }; /** - * Class used to instantiate non-allocator pools using a Singleton PoolState. + * Class used to instantiate a global non-allocator PoolState. */ - template - class PoolStateHandle + template + class SingletonPoolState { - static PoolState& pool_state = - Singleton, PoolStateHandle::make_state>::get(); - - static PoolState* make_state() - { - return ChunkAllocator::alloc_meta_data, SharedStateHandle>( - nullptr); - } + static inline PoolState state; public: static PoolState& pool() { - return pool_state; + return state; } }; - template + template& get_state()> class Pool { public: template static T* acquire(Args&&... args) { - PoolState& pool = SharedStateHandle::pool(); + PoolState& pool = get_state(); T* p = pool.stack.pop(); if (p != nullptr) @@ -99,14 +95,14 @@ namespace snmalloc // is returned without the constructor being run, so the object is reused // without re-initialisation. p->reset_in_use(); - SharedStateHandle::pool().stack.push(p); + get_state().stack.push(p); } static T* extract(T* p = nullptr) { // Returns a linked list of all objects in the stack, emptying the stack. if (p == nullptr) - return SharedStateHandle::pool().stack.pop_all(); + return get_state().stack.pop_all(); return p->next; } @@ -120,13 +116,13 @@ namespace snmalloc { // Pushes a linked list of objects onto the stack. Use to put a linked // list returned by extract back onto the stack. - SharedStateHandle::pool().stack.push(first, last); + get_state().stack.push(first, last); } static T* iterate(T* p = nullptr) { if (p == nullptr) - return SharedStateHandle::pool().list; + return get_state().list; return p->list_next; } diff --git a/src/mem/pooled.h b/src/mem/pooled.h index 657ea2d32..d5b988c6e 100644 --- a/src/mem/pooled.h +++ b/src/mem/pooled.h @@ -4,11 +4,17 @@ namespace snmalloc { + template + class PoolState; + template class Pooled { - private: - template + public: + template< + typename TT, + typename SharedStateHandle, + PoolState& get_state()> friend class Pool; template friend class MPMCStack; From df852dba6ace833c68df507b54b4cdedece681ca Mon Sep 17 00:00:00 2001 From: Istvan Haller Date: Tue, 24 Aug 2021 14:18:39 +0100 Subject: [PATCH 047/302] Added test and forced initialization of backend when using custom Pool. --- src/mem/pool.h | 40 ++++++++++++- src/test/func/pool/pool.cc | 117 +++++++++++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+), 3 deletions(-) create mode 100644 src/test/func/pool/pool.cc diff --git a/src/mem/pool.h b/src/mem/pool.h index 76950c2dc..34f58e97c 100644 --- a/src/mem/pool.h +++ b/src/mem/pool.h @@ -39,19 +39,53 @@ namespace snmalloc /** * Class used to instantiate a global non-allocator PoolState. */ - template + template class SingletonPoolState { static inline PoolState state; + static thread_local inline bool initialized; + + /** + * SFINAE helper. Matched only if `T` implements `ensure_init`. Calls it + * if it exists. + */ + SNMALLOC_FAST_PATH static auto call_ensure_init(SharedStateHandle*, int) + -> decltype(SharedStateHandle::ensure_init()) + { + SharedStateHandle::ensure_init(); + } + + /** + * SFINAE helper. Matched only if `T` does not implement `ensure_init`. + * Does nothing if called. + */ + SNMALLOC_FAST_PATH static auto call_ensure_init(SharedStateHandle*, long) {} + + /** + * Call `SharedStateHandle::ensure_init()` if it is implemented, do nothing + * otherwise. + */ + SNMALLOC_FAST_PATH static void ensure_init() + { + call_ensure_init(nullptr, 0); + } public: - static PoolState& pool() + SNMALLOC_FAST_PATH static PoolState& pool() { + if (unlikely(!initialized)) + { + ensure_init(); + initialized = true; + } return state; } }; - template& get_state()> + template< + typename T, + typename SharedStateHandle, + PoolState& get_state() = SingletonPoolState::pool> class Pool { public: diff --git a/src/test/func/pool/pool.cc b/src/test/func/pool/pool.cc new file mode 100644 index 000000000..550dd4e3c --- /dev/null +++ b/src/test/func/pool/pool.cc @@ -0,0 +1,117 @@ +#include +#include +#include +#include + +using namespace snmalloc; + +struct PoolAEntry : Pooled +{ + size_t field; + + PoolAEntry() : field(1){}; +}; + +using PoolA = Pool; + +struct PoolBEntry : Pooled +{ + size_t field; + + PoolBEntry() : field(0){}; + PoolBEntry(size_t f) : field(f){}; +}; + +using PoolB = Pool; + +void test_alloc() +{ + auto ptr = PoolA::acquire(); + SNMALLOC_CHECK(ptr != nullptr); + // Pool allocations should not be visible to debug_check_empty. + snmalloc::debug_check_empty(); +} + +void test_constructor() +{ + auto ptr1 = PoolA::acquire(); + SNMALLOC_CHECK(ptr1 != nullptr); + SNMALLOC_CHECK(ptr1->field == 1); + + auto ptr2 = PoolB::acquire(); + SNMALLOC_CHECK(ptr2 != nullptr); + SNMALLOC_CHECK(ptr2->field == 0); + + auto ptr3 = PoolB::acquire(1); + SNMALLOC_CHECK(ptr3 != nullptr); + SNMALLOC_CHECK(ptr3->field == 1); +} + +void test_alloc_many() +{ + constexpr size_t count = 16'000'000 / MIN_CHUNK_SIZE; + + std::unordered_set allocated; + + for (size_t i = 0; i < count; ++i) + { + auto ptr = PoolA::acquire(); + SNMALLOC_CHECK(ptr != nullptr); + allocated.insert(ptr); + } + + for (auto ptr : allocated) + { + PoolA::release(ptr); + } +} + +void test_alloc_dealloc() +{ + auto ptr = PoolA::acquire(); + SNMALLOC_CHECK(ptr != nullptr); + PoolA::release(ptr); +} + +void test_double_alloc() +{ + auto ptr1 = PoolA::acquire(); + SNMALLOC_CHECK(ptr1 != nullptr); + auto ptr2 = PoolA::acquire(); + SNMALLOC_CHECK(ptr2 != nullptr); + SNMALLOC_CHECK(ptr1 != ptr2); + PoolA::release(ptr2); + auto ptr3 = PoolA::acquire(); + SNMALLOC_CHECK(ptr2 == ptr3); +} + +void test_different_alloc() +{ + auto ptr1 = PoolA::acquire(); + SNMALLOC_CHECK(ptr1 != nullptr); + PoolA::release(ptr1); + auto ptr2 = PoolB::acquire(); + SNMALLOC_CHECK(ptr2 != nullptr); + SNMALLOC_CHECK(static_cast(ptr1) != static_cast(ptr2)); +} + +int main(int argc, char** argv) +{ + setup(); +#ifdef USE_SYSTEMATIC_TESTING + opt::Opt opt(argc, argv); + size_t seed = opt.is("--seed", 0); + Virtual::systematic_bump_ptr() += seed << 17; +#else + UNUSED(argc); + UNUSED(argv); +#endif + + test_alloc(); + test_constructor(); + test_alloc_many(); + test_alloc_dealloc(); + test_double_alloc(); + test_different_alloc(); + return 0; +} From b7fe8ea65436c106ae9c0311527c909465d3ecf6 Mon Sep 17 00:00:00 2001 From: Istvan Haller Date: Tue, 24 Aug 2021 15:32:51 +0100 Subject: [PATCH 048/302] More comments and improved test --- src/mem/corealloc.h | 3 +++ src/mem/pool.h | 27 ++++++++++++++++++++++++- src/test/func/pool/pool.cc | 41 ++++++++++++++++++++++++++++---------- 3 files changed, 60 insertions(+), 11 deletions(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 30dba5365..b67df1866 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -786,6 +786,9 @@ namespace snmalloc } }; + /** + * Use this alias to access the pool of allocators throughout snmalloc. + */ template using AllocPool = Pool< CoreAllocator, diff --git a/src/mem/pool.h b/src/mem/pool.h index 34f58e97c..aecc5501f 100644 --- a/src/mem/pool.h +++ b/src/mem/pool.h @@ -37,12 +37,20 @@ namespace snmalloc }; /** - * Class used to instantiate a global non-allocator PoolState. + * Helper class used to instantiate a global PoolState. + * + * SingletonPoolState::pool is the default provider for the PoolState within + * the Pool class. */ template class SingletonPoolState { static inline PoolState state; + /** + * Thread-local initialization marker for a contention-free way to skip + * initialization. SharedStateHandle::ensure_init allows for multiple calls, + * but it uses a reentrency-safe check which is more expensive. + */ static thread_local inline bool initialized; /** @@ -71,6 +79,10 @@ namespace snmalloc } public: + /** + * Returns a reference for the global PoolState for the given type. + * Also forces the initialization of the backend state, if needed. + */ SNMALLOC_FAST_PATH static PoolState& pool() { if (unlikely(!initialized)) @@ -82,6 +94,19 @@ namespace snmalloc } }; + /** + * Wrapper class to access a pool of a particular type of object. + * + * The third template argument is a method to retrieve the actual PoolState. + * + * For the pool of allocators, refer to the AllocPool alias defined in + * corealloc.h. + * + * For a pool of another type it is recommended to leave the + * third template argument with its default value. The SingletonPoolState + * class is used as a helper to provide a default PoolState management for + * this use case. + */ template< typename T, typename SharedStateHandle, diff --git a/src/test/func/pool/pool.cc b/src/test/func/pool/pool.cc index 550dd4e3c..8288c2992 100644 --- a/src/test/func/pool/pool.cc +++ b/src/test/func/pool/pool.cc @@ -7,7 +7,7 @@ using namespace snmalloc; struct PoolAEntry : Pooled { - size_t field; + int field; PoolAEntry() : field(1){}; }; @@ -16,7 +16,7 @@ using PoolA = Pool; struct PoolBEntry : Pooled { - size_t field; + int field; PoolBEntry() : field(0){}; PoolBEntry(size_t f) : field(f){}; @@ -30,6 +30,7 @@ void test_alloc() SNMALLOC_CHECK(ptr != nullptr); // Pool allocations should not be visible to debug_check_empty. snmalloc::debug_check_empty(); + PoolA::release(ptr); } void test_constructor() @@ -45,6 +46,10 @@ void test_constructor() auto ptr3 = PoolB::acquire(1); SNMALLOC_CHECK(ptr3 != nullptr); SNMALLOC_CHECK(ptr3->field == 1); + + PoolA::release(ptr1); + PoolB::release(ptr2); + PoolB::release(ptr3); } void test_alloc_many() @@ -66,13 +71,6 @@ void test_alloc_many() } } -void test_alloc_dealloc() -{ - auto ptr = PoolA::acquire(); - SNMALLOC_CHECK(ptr != nullptr); - PoolA::release(ptr); -} - void test_double_alloc() { auto ptr1 = PoolA::acquire(); @@ -83,6 +81,8 @@ void test_double_alloc() PoolA::release(ptr2); auto ptr3 = PoolA::acquire(); SNMALLOC_CHECK(ptr2 == ptr3); + PoolA::release(ptr1); + PoolA::release(ptr3); } void test_different_alloc() @@ -93,6 +93,27 @@ void test_different_alloc() auto ptr2 = PoolB::acquire(); SNMALLOC_CHECK(ptr2 != nullptr); SNMALLOC_CHECK(static_cast(ptr1) != static_cast(ptr2)); + PoolB::release(ptr2); +} + +void test_iterator() +{ + PoolAEntry* before_iteration_ptr = PoolA::acquire(); + + PoolAEntry* ptr = nullptr; + while ((ptr = PoolA::iterate(ptr)) != nullptr) + { + ptr->field = 2; + } + + SNMALLOC_CHECK(before_iteration_ptr->field == 2); + + PoolAEntry* after_iteration_ptr = PoolA::acquire(); + + SNMALLOC_CHECK(after_iteration_ptr->field == 2); + + PoolA::release(before_iteration_ptr); + PoolA::release(after_iteration_ptr); } int main(int argc, char** argv) @@ -110,8 +131,8 @@ int main(int argc, char** argv) test_alloc(); test_constructor(); test_alloc_many(); - test_alloc_dealloc(); test_double_alloc(); test_different_alloc(); + test_iterator(); return 0; } From 33a358e4fe0e4d75f0af9a6179fa44d8d5771311 Mon Sep 17 00:00:00 2001 From: Istvan Haller Date: Tue, 24 Aug 2021 16:18:02 +0100 Subject: [PATCH 049/302] Another attempt to fix CI --- src/test/func/pool/pool.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/func/pool/pool.cc b/src/test/func/pool/pool.cc index 8288c2992..f479f4ef6 100644 --- a/src/test/func/pool/pool.cc +++ b/src/test/func/pool/pool.cc @@ -19,7 +19,7 @@ struct PoolBEntry : Pooled int field; PoolBEntry() : field(0){}; - PoolBEntry(size_t f) : field(f){}; + PoolBEntry(int f) : field(f){}; }; using PoolB = Pool; From e8cc3af6e5f9d0011dc75c685876a3ef92d99821 Mon Sep 17 00:00:00 2001 From: Istvan Haller Date: Wed, 25 Aug 2021 13:35:13 +0100 Subject: [PATCH 050/302] Applied PR feedback --- src/ds/helpers.h | 4 ++-- src/mem/pool.h | 24 +++++++++--------------- src/mem/threadalloc.h | 6 ++---- 3 files changed, 13 insertions(+), 21 deletions(-) diff --git a/src/ds/helpers.h b/src/ds/helpers.h index e34387d9a..f8ad92c80 100644 --- a/src/ds/helpers.h +++ b/src/ds/helpers.h @@ -13,7 +13,7 @@ namespace snmalloc * initialised. This singleton class is designed to not depend on the * runtime. */ - template + template class Singleton { inline static std::atomic_flag flag; @@ -36,7 +36,7 @@ namespace snmalloc FlagLock lock(flag); if (!initialised) { - obj = init(); + init(&obj); initialised.store(true, std::memory_order_release); if (first != nullptr) *first = true; diff --git a/src/mem/pool.h b/src/mem/pool.h index aecc5501f..c57b52530 100644 --- a/src/mem/pool.h +++ b/src/mem/pool.h @@ -45,14 +45,6 @@ namespace snmalloc template class SingletonPoolState { - static inline PoolState state; - /** - * Thread-local initialization marker for a contention-free way to skip - * initialization. SharedStateHandle::ensure_init allows for multiple calls, - * but it uses a reentrency-safe check which is more expensive. - */ - static thread_local inline bool initialized; - /** * SFINAE helper. Matched only if `T` implements `ensure_init`. Calls it * if it exists. @@ -78,6 +70,13 @@ namespace snmalloc call_ensure_init(nullptr, 0); } + static void make_pool(PoolState*) noexcept + { + ensure_init(); + // Default initializer already called on PoolState, no need to use + // placement new. + } + public: /** * Returns a reference for the global PoolState for the given type. @@ -85,12 +84,7 @@ namespace snmalloc */ SNMALLOC_FAST_PATH static PoolState& pool() { - if (unlikely(!initialized)) - { - ensure_init(); - initialized = true; - } - return state; + return Singleton, &make_pool>::get(); } }; @@ -102,7 +96,7 @@ namespace snmalloc * For the pool of allocators, refer to the AllocPool alias defined in * corealloc.h. * - * For a pool of another type it is recommended to leave the + * For a pool of another type, it is recommended to leave the * third template argument with its default value. The SingletonPoolState * class is used as a helper to provide a default PoolState management for * this use case. diff --git a/src/mem/threadalloc.h b/src/mem/threadalloc.h index dc9cee6f4..3777d4422 100644 --- a/src/mem/threadalloc.h +++ b/src/mem/threadalloc.h @@ -106,11 +106,9 @@ namespace snmalloc /** * Used to give correct signature to the pthread call for the Singleton class. */ - inline pthread_key_t pthread_create() noexcept + inline void pthread_create(pthread_key_t* key) noexcept { - pthread_key_t key; - pthread_key_create(&key, &pthread_cleanup); - return key; + pthread_key_create(key, &pthread_cleanup); } /** * Performs thread local teardown for the allocator using the pthread library. From 3ceef4086191c506fd1c5bb478e8d70dc9d57803 Mon Sep 17 00:00:00 2001 From: Istvan Haller Date: Wed, 25 Aug 2021 14:38:58 +0100 Subject: [PATCH 051/302] Fixed ensure_init detection --- src/mem/pool.h | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/mem/pool.h b/src/mem/pool.h index c57b52530..f8245c5f6 100644 --- a/src/mem/pool.h +++ b/src/mem/pool.h @@ -49,17 +49,27 @@ namespace snmalloc * SFINAE helper. Matched only if `T` implements `ensure_init`. Calls it * if it exists. */ - SNMALLOC_FAST_PATH static auto call_ensure_init(SharedStateHandle*, int) - -> decltype(SharedStateHandle::ensure_init()) + template + SNMALLOC_FAST_PATH static auto call_ensure_init(SharedStateHandle_*, int) + -> decltype(SharedStateHandle_::ensure_init()) { - SharedStateHandle::ensure_init(); + static_assert( + std::is_same::value, + "SFINAE parameter, should only be used with SharedStateHandle"); + SharedStateHandle_::ensure_init(); } /** * SFINAE helper. Matched only if `T` does not implement `ensure_init`. * Does nothing if called. */ - SNMALLOC_FAST_PATH static auto call_ensure_init(SharedStateHandle*, long) {} + template + SNMALLOC_FAST_PATH static auto call_ensure_init(SharedStateHandle_*, long) + { + static_assert( + std::is_same::value, + "SFINAE parameter, should only be used with SharedStateHandle"); + } /** * Call `SharedStateHandle::ensure_init()` if it is implemented, do nothing @@ -67,7 +77,7 @@ namespace snmalloc */ SNMALLOC_FAST_PATH static void ensure_init() { - call_ensure_init(nullptr, 0); + call_ensure_init(nullptr, 0); } static void make_pool(PoolState*) noexcept From ea90f7b9c2f8a937bd040719a5585be83ba45f9a Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 25 Aug 2021 11:41:09 +0100 Subject: [PATCH 052/302] Don't include iostream. --- src/backend/globalconfig.h | 5 +++-- src/mem/slaballocator.h | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/backend/globalconfig.h b/src/backend/globalconfig.h index f084703d4..1e1cd925c 100644 --- a/src/backend/globalconfig.h +++ b/src/backend/globalconfig.h @@ -6,8 +6,9 @@ #include "../mem/slaballocator.h" #include "commonconfig.h" -#include - +#ifdef SNMALLOC_TRACING +# include +#endif namespace snmalloc { // Forward reference to thread local cleanup. diff --git a/src/mem/slaballocator.h b/src/mem/slaballocator.h index 367509be5..4341ff193 100644 --- a/src/mem/slaballocator.h +++ b/src/mem/slaballocator.h @@ -8,6 +8,8 @@ # include #endif +#include + namespace snmalloc { /** From b52e2a6e271b5e7a5865a2f934540822c7222643 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 25 Aug 2021 11:26:44 +0100 Subject: [PATCH 053/302] Expose pthread feature flag The code was able to use pthread destructors rather than C++ thread local destructors. This removes the dependence on a C++ .so on linux. However, this is not stable on other platforms such as Apple. Where the C++ thread local state can be cleared before the pthread destructor runs. --- .github/workflows/main.yml | 11 +++++++++-- CMakeLists.txt | 5 +++++ src/mem/threadalloc.h | 3 ++- .../thread_alloc_external/thread_alloc_external.cc | 4 ++++ 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index feabadeb7..c04e28d8b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -38,16 +38,23 @@ jobs: build-type: Debug build-only: yes # Add the self-host build - - os: ubuntu-latest + - os: "ubuntu-latest" + build-type: Debug + self-host: true + # Extra build to check using pthread library for destructing local state. + - os: "ubuntu-latest" + variant: "Ubuntu (with pthread destructors)." + dependencies: "sudo apt install ninja-build" build-type: Debug self-host: true + extra-cmake-flags: "-DSNMALLOC_USE_PTHREAD_DESTRUCTORS=On -DCMAKE_CXX_COMPILER=clang++-10 -DCMAKE_C_COMPILER=clang-10" # Add an extra element to the matrix that does a build with clang 12 # but doesn't run tests. - os: "freebsd-13.0" variant: Clang 12 (Build only) extra-cmake-flags: "-DCMAKE_CXX_COMPILER=clang++12" build-only: yes - - os: ubuntu-latest + - os: "ubuntu-latest" variant: Clang 10 libstdc++ (Build only) dependencies: "sudo apt install ninja-build" extra-cmake-flags: "-DCMAKE_CXX_COMPILER=clang++-10 -DCMAKE_C_COMPILER=clang-10 -DCMAKE_CXX_FLAGS=-stdlib=libstdc++" diff --git a/CMakeLists.txt b/CMakeLists.txt index c313118b7..977b31c1a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,6 +19,7 @@ option(SNMALLOC_QEMU_WORKAROUND "Disable using madvise(DONT_NEED) to zero memory option(SNMALLOC_OPTIMISE_FOR_CURRENT_MACHINE "Compile for current machine architecture" Off) set(SNMALLOC_STATIC_LIBRARY_PREFIX "sn_" CACHE STRING "Static library function prefix") option(SNMALLOC_USE_CXX17 "Build as C++17 for legacy support." OFF) +option(SNMALLOC_USE_PTHREAD_DESTRUCTORS "Build using pthread destructors. Reduces the system dependencies, but may interact badly with C++ on some platforms." OFF) # malloc.h will error if you include it on FreeBSD, so this test must not # unconditionally include it. @@ -150,6 +151,10 @@ if(SNMALLOC_PLATFORM_HAS_GETENTROPY) target_compile_definitions(snmalloc_lib INTERFACE -DSNMALLOC_PLATFORM_HAS_GETENTROPY) endif() +if(SNMALLOC_USE_PTHREAD_DESTRUCTORS) + target_compile_definitions(snmalloc_lib INTERFACE -DSNMALLOC_USE_PTHREAD_DESTRUCTORS) +endif() + if(CONST_QUALIFIED_MALLOC_USABLE_SIZE) target_compile_definitions(snmalloc_lib INTERFACE -DMALLOC_USABLE_SIZE_QUALIFIER=const) endif() diff --git a/src/mem/threadalloc.h b/src/mem/threadalloc.h index 3777d4422..3c75c42ae 100644 --- a/src/mem/threadalloc.h +++ b/src/mem/threadalloc.h @@ -19,6 +19,7 @@ # if defined(SNMALLOC_THREAD_TEARDOWN_DEFINED) # error At most one out of method of thread teardown can be specified. # else +# include # define SNMALLOC_THREAD_TEARDOWN_DEFINED # endif #endif @@ -95,7 +96,7 @@ namespace snmalloc } }; -# ifdef SNMALLOC_USE_PTHREAD_DESTRUCTOR +# ifdef SNMALLOC_USE_PTHREAD_DESTRUCTORS /** * Used to give correct signature to teardown required by pthread_key. */ diff --git a/src/test/func/thread_alloc_external/thread_alloc_external.cc b/src/test/func/thread_alloc_external/thread_alloc_external.cc index effb62fa0..832d400bd 100644 --- a/src/test/func/thread_alloc_external/thread_alloc_external.cc +++ b/src/test/func/thread_alloc_external/thread_alloc_external.cc @@ -1,3 +1,7 @@ +#ifdef SNMALLOC_USE_PTHREAD_DESTRUCTORS +# undef SNMALLOC_USE_PTHREAD_DESTRUCTORS +#endif + #include #include From 935f3ccd291a54afe31ffa3699772a79fc7bb962 Mon Sep 17 00:00:00 2001 From: Istvan Haller Date: Thu, 26 Aug 2021 12:18:53 +0100 Subject: [PATCH 054/302] Improved support for MSVC with C++17 --- CMakeLists.txt | 4 ++++ src/ds/defines.h | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c313118b7..1f3a1e2a3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -107,6 +107,10 @@ endif() add_library(snmalloc_lib INTERFACE) target_include_directories(snmalloc_lib INTERFACE src/) +if(SNMALLOC_USE_CXX17) + target_compile_definitions(snmalloc_lib INTERFACE -DSNMALLOC_USE_CXX17) +endif() + if(NOT MSVC) find_package(Threads REQUIRED COMPONENTS snmalloc_lib) target_link_libraries(snmalloc_lib INTERFACE ${CMAKE_THREAD_LIBS_INIT}) diff --git a/src/ds/defines.h b/src/ds/defines.h index b00198247..d8c495556 100644 --- a/src/ds/defines.h +++ b/src/ds/defines.h @@ -7,7 +7,7 @@ # define unlikely(x) !!(x) # define SNMALLOC_SLOW_PATH NOINLINE # define SNMALLOC_FAST_PATH ALWAYSINLINE -# if _MSC_VER >= 1927 +# if _MSC_VER >= 1927 && !defined(SNMALLOC_USE_CXX17) # define SNMALLOC_FAST_PATH_LAMBDA [[msvc::forceinline]] # else # define SNMALLOC_FAST_PATH_LAMBDA From 0f70494d55af9af3cc44367a49d0d8e665bcd271 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 26 Aug 2021 10:04:44 +0100 Subject: [PATCH 055/302] Enable passthrough to an underlying allocator This passes though to an underlying allocator rather than using snmalloc. This is required for using ASAN in Verona. Verona takes a close coupling with snmalloc, but to use with ASAN would require a more work, so we pass to the system allocator in this case. --- CMakeLists.txt | 2 +- src/mem/localalloc.h | 29 ++++++++++++++++++++++------- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ce9cbc780..c8b0d0da9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -286,7 +286,7 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) # NetBSD, OpenBSD and DragonFlyBSD do not support malloc*size calls. set(FLAVOURS fast;check) else() - set(FLAVOURS fast;check) #malloc - TODO-need to add pass through back + set(FLAVOURS fast;check;malloc) endif() foreach(FLAVOUR ${FLAVOURS}) unset(SRC) diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index f4a576e1f..ae2590514 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -14,6 +14,10 @@ #include "remotecache.h" #include "sizeclasstable.h" +#ifdef SNMALLOC_PASS_THROUGH +# include "external_alloc.h" +#endif + #ifdef SNMALLOC_TRACING # include #endif @@ -458,7 +462,9 @@ namespace snmalloc SNMALLOC_FAST_PATH void dealloc(void* p) { - // TODO Pass through code! +#ifdef SNMALLOC_PASS_THROUGH + external_alloc::free(p); +#else // TODO: // Care is needed so that dealloc(nullptr) works before init // The backend allocator must ensure that a minimal page map exists @@ -485,10 +491,10 @@ namespace snmalloc entry.get_remote()->trunc_id(), CapPtr(p), key_global); -#ifdef SNMALLOC_TRACING +# ifdef SNMALLOC_TRACING std::cout << "Remote dealloc fast" << p << " size " << alloc_size(p) << std::endl; -#endif +# endif return; } @@ -509,10 +515,10 @@ namespace snmalloc pointer_align_down(p, size) == p, "Not start of an allocation."); size_t slab_sizeclass = large_size_to_chunk_sizeclass(size); -#ifdef SNMALLOC_TRACING +# ifdef SNMALLOC_TRACING std::cout << "Large deallocation: " << size << " chunk sizeclass: " << slab_sizeclass << std::endl; -#endif +# endif ChunkRecord* slab_record = reinterpret_cast(entry.get_metaslab()); slab_record->chunk = CapPtr(p); @@ -524,10 +530,11 @@ namespace snmalloc return; } -#ifdef SNMALLOC_TRACING +# ifdef SNMALLOC_TRACING std::cout << "nullptr deallocation" << std::endl; -#endif +# endif return; +#endif } SNMALLOC_FAST_PATH void dealloc(void* p, size_t s) @@ -558,6 +565,9 @@ namespace snmalloc SNMALLOC_FAST_PATH size_t alloc_size(const void* p_raw) { +#ifdef SNMALLOC_PASS_THROUGH + return external_alloc::malloc_usable_size(const_cast(p_raw)); +#else // Note that this should return 0 for nullptr. // Other than nullptr, we know the system will be initialised as it must // be called with something we have already allocated. @@ -574,6 +584,7 @@ namespace snmalloc return bits::one_at_bit(entry.get_sizeclass()); return 0; +#endif } /** @@ -586,6 +597,7 @@ namespace snmalloc template void* external_pointer(void* p_raw) { +#ifndef SNMALLOC_PASS_THROUGH // TODO bring back the CHERI bits. Wes to review if required. if (likely(is_initialised())) { @@ -627,6 +639,9 @@ namespace snmalloc { // Allocator not initialised, so definitely not our allocation } +#else + UNUSED(p_raw); +#endif if constexpr ((location == End) || (location == OnePastEnd)) // We don't know the End, so return MAX_PTR From 44416ed70e5179b3c2c58c9e3b358992a26fcd60 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 25 Aug 2021 14:49:06 +0100 Subject: [PATCH 056/302] Factor sizeclass meta-data for cache locality This commit splits the sizeclass meta-data to generate better cache locality for various lookups for checking for size and start of sizeclasses. Also, contains some tidying including removing sizeclasses covering large range. This is left over from an alternative design for large classes that is no longer in use. --- src/mem/localalloc.h | 4 +- src/mem/sizeclasstable.h | 215 ++++++++++++++------------- src/test/func/memory/memory.cc | 2 +- src/test/func/sizeclass/sizeclass.cc | 3 +- 4 files changed, 116 insertions(+), 108 deletions(-) diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index ae2590514..8e0978a9a 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -189,8 +189,8 @@ namespace snmalloc // set up meta data so sizeclass is correct, and hence alloc size, and // external pointer. #ifdef SNMALLOC_TRACING - std::cout << "size " << size << " sizeclass " << size_to_sizeclass(size) - << std::endl; + std::cout << "size " << size << " pow2 size " + << bits::next_pow2_bits(size) << std::endl; #endif // Note that meta data is not currently used for large allocs. diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index 97f224d99..3e08c44a8 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -30,6 +30,7 @@ namespace snmalloc static inline size_t large_sizeclass_to_size(uint8_t large_class) { + // TODO. Remove UNUSED(large_class); abort(); // return bits::one_at_bit(large_class + SUPERSLAB_BITS); @@ -53,98 +54,77 @@ namespace snmalloc return ((alignment - 1) | (size - 1)) + 1; } - constexpr static SNMALLOC_PURE size_t sizeclass_lookup_index(const size_t s) - { - // We subtract and shift to reduce the size of the table, i.e. we don't have - // to store a value for every size. - return (s - 1) >> MIN_ALLOC_BITS; - } - - constexpr static size_t NUM_SIZECLASSES_EXTENDED = - size_to_sizeclass_const(bits::one_at_bit(bits::ADDRESS_BITS - 1)); - - constexpr static size_t sizeclass_lookup_size = - sizeclass_lookup_index(MAX_SIZECLASS_SIZE); - - struct SizeClassTable + /** + * This structure contains the fields required for fast paths for sizeclasses. + */ + struct sizeclass_data_fast { - sizeclass_compress_t sizeclass_lookup[sizeclass_lookup_size] = {{}}; - ModArray size; - - ModArray capacity; - ModArray waking; + size_t size; // We store the mask as it is used more on the fast path, and the size of // the slab. - ModArray slab_mask; - + size_t slab_mask; // Table of constants for reciprocal division for each sizeclass. - ModArray div_mult; + size_t div_mult; // Table of constants for reciprocal modulus for each sizeclass. - ModArray mod_mult; + size_t mod_mult; + }; + + /** + * This structure contains the remaining fields required for slow paths for + * sizeclasses. + */ + struct sizeclass_data_slow + { + uint16_t capacity; + uint16_t waking; + }; + + struct SizeClassTable + { + ModArray fast; + ModArray slow; - constexpr SizeClassTable() - : size(), capacity(), waking(), slab_mask(), div_mult(), mod_mult() + constexpr SizeClassTable() : fast(), slow() { for (sizeclass_compress_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++) { size_t rsize = bits::from_exp_mant(sizeclass); - size[sizeclass] = rsize; + fast[sizeclass].size = rsize; size_t slab_bits = bits::max( bits::next_pow2_bits_const(MIN_OBJECT_COUNT * rsize), MIN_CHUNK_BITS); - slab_mask[sizeclass] = bits::one_at_bit(slab_bits) - 1; + fast[sizeclass].slab_mask = bits::one_at_bit(slab_bits) - 1; - capacity[sizeclass] = - static_cast((slab_mask[sizeclass] + 1) / rsize); + slow[sizeclass].capacity = + static_cast((fast[sizeclass].slab_mask + 1) / rsize); - waking[sizeclass] = + slow[sizeclass].waking = #ifdef SNMALLOC_CHECK_CLIENT - static_cast(capacity[sizeclass] / 4); + static_cast(slow[sizeclass].capacity / 4); #else - static_cast(bits::min((capacity[sizeclass] / 4), 32)); + static_cast(bits::min((slow[sizeclass].capacity / 4), 32)); #endif } - for (sizeclass_compress_t sizeclass = NUM_SIZECLASSES; - sizeclass < NUM_SIZECLASSES_EXTENDED; - sizeclass++) - { - size[sizeclass] = bits::prev_pow2_const( - bits::from_exp_mant(sizeclass)); - } - for (sizeclass_compress_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++) { - div_mult[sizeclass] = // TODO is MAX_SIZECLASS_BITS right? + fast[sizeclass].div_mult = // TODO is MAX_SIZECLASS_BITS right? (bits::one_at_bit(bits::BITS - 24) / - (size[sizeclass] / MIN_ALLOC_SIZE)); - if (!bits::is_pow2(size[sizeclass])) - div_mult[sizeclass]++; - - mod_mult[sizeclass] = - (bits::one_at_bit(bits::BITS - 1) / size[sizeclass]); - if (!bits::is_pow2(size[sizeclass])) - mod_mult[sizeclass]++; + (fast[sizeclass].size / MIN_ALLOC_SIZE)); + if (!bits::is_pow2(fast[sizeclass].size)) + fast[sizeclass].div_mult++; + + fast[sizeclass].mod_mult = + (bits::one_at_bit(bits::BITS - 1) / fast[sizeclass].size); + if (!bits::is_pow2(fast[sizeclass].size)) + fast[sizeclass].mod_mult++; // Shift multiplier, so that the result of division completely // overflows, and thus the top SUPERSLAB_BITS will be zero if the mod is // zero. - mod_mult[sizeclass] *= 2; - } - - size_t curr = 1; - for (sizeclass_compress_t sizeclass = 0; sizeclass <= NUM_SIZECLASSES; - sizeclass++) - { - for (; curr <= size[sizeclass]; curr += 1 << MIN_ALLOC_BITS) - { - auto i = sizeclass_lookup_index(curr); - if (i == sizeclass_lookup_size) - break; - sizeclass_lookup[i] = sizeclass; - } + fast[sizeclass].mod_mult *= 2; } } }; @@ -153,12 +133,12 @@ namespace snmalloc constexpr static inline size_t sizeclass_to_size(sizeclass_t sizeclass) { - return sizeclass_metadata.size[sizeclass]; + return sizeclass_metadata.fast[sizeclass].size; } inline static size_t sizeclass_to_slab_size(sizeclass_t sizeclass) { - return sizeclass_metadata.slab_mask[sizeclass] + 1; + return sizeclass_metadata.fast[sizeclass].slab_mask + 1; } /** @@ -170,12 +150,7 @@ namespace snmalloc */ inline uint16_t threshold_for_waking_slab(sizeclass_t sizeclass) { - // #ifdef SNMALLOC_CHECK_CLIENT - return sizeclass_metadata.waking[sizeclass]; - // #else - // UNUSED(sizeclass); - // return 1; - // #endif + return sizeclass_metadata.slow[sizeclass].waking; } inline static size_t sizeclass_to_slab_sizeclass(sizeclass_t sizeclass) @@ -193,26 +168,7 @@ namespace snmalloc inline constexpr static uint16_t sizeclass_to_slab_object_count(sizeclass_t sizeclass) { - return sizeclass_metadata.capacity[sizeclass]; - } - - static inline sizeclass_t size_to_sizeclass(size_t size) - { - auto index = sizeclass_lookup_index(size); - if (index < sizeclass_lookup_size) - { - return sizeclass_metadata.sizeclass_lookup[index]; - } - - // Don't use sizeclasses that are not a multiple of the alignment. - // For example, 24 byte allocations can be - // problematic for some data due to alignment issues. - - // TODO hack to power of 2 for large sizes - size = bits::next_pow2(size); - - return static_cast( - bits::to_exp_mant(size)); + return sizeclass_metadata.slow[sizeclass].capacity; } inline static size_t round_by_sizeclass(sizeclass_t sc, size_t offset) @@ -236,7 +192,8 @@ namespace snmalloc // SUPERSLAB_BITS <= 24, "The following code assumes max of 24 bits"); // TODO 24 hack - return (((offset >> MIN_ALLOC_BITS) * sizeclass_metadata.div_mult[sc]) >> + return (((offset >> MIN_ALLOC_BITS) * + sizeclass_metadata.fast[sc].div_mult) >> (bits::BITS - 24)) * rsize; } @@ -265,7 +222,7 @@ namespace snmalloc static constexpr size_t MASK = ~(bits::one_at_bit(bits::BITS - 1 - 24) - 1); - return ((offset * sizeclass_metadata.mod_mult[sc]) & MASK) == 0; + return ((offset * sizeclass_metadata.fast[sc].mod_mult) & MASK) == 0; } else // Use 32-bit division as considerably faster than 64-bit, and @@ -273,6 +230,68 @@ namespace snmalloc return static_cast(offset % sizeclass_to_size(sc)) == 0; } + inline static size_t large_size_to_chunk_size(size_t size) + { + return bits::next_pow2(size); + } + + inline static size_t large_size_to_chunk_sizeclass(size_t size) + { + return bits::next_pow2_bits(size) - MIN_CHUNK_BITS; + } + + constexpr static SNMALLOC_PURE size_t sizeclass_lookup_index(const size_t s) + { + // We subtract and shift to reduce the size of the table, i.e. we don't have + // to store a value for every size. + return (s - 1) >> MIN_ALLOC_BITS; + } + + static inline sizeclass_t size_to_sizeclass(size_t size) + { + constexpr static size_t sizeclass_lookup_size = + sizeclass_lookup_index(MAX_SIZECLASS_SIZE); + + /** + * This struct is used to statically initialise a table for looking up + * the correct sizeclass. + */ + struct SizeClassLookup + { + sizeclass_compress_t table[sizeclass_lookup_size] = {{}}; + + constexpr SizeClassLookup() + { + size_t curr = 1; + for (sizeclass_compress_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; + sizeclass++) + { + for (; curr <= sizeclass_metadata.fast[sizeclass].size; + curr += 1 << MIN_ALLOC_BITS) + { + auto i = sizeclass_lookup_index(curr); + if (i == sizeclass_lookup_size) + break; + table[i] = sizeclass; + } + } + } + }; + + static constexpr SizeClassLookup sizeclass_lookup = SizeClassLookup(); + + auto index = sizeclass_lookup_index(size); + if (index < sizeclass_lookup_size) + { + return sizeclass_lookup.table[index]; + } + + // Check this is not called on large sizes. + SNMALLOC_ASSERT(size == 0); + // Map size == 0 to the first sizeclass. + return 0; + } + inline SNMALLOC_FAST_PATH static size_t round_size(size_t size) { if (size > sizeclass_to_size(NUM_SIZECLASSES - 1)) @@ -295,14 +314,4 @@ namespace snmalloc return 1; return bits::one_at_bit(bits::ctz(rsize)); } - - inline static size_t large_size_to_chunk_size(size_t size) - { - return bits::next_pow2(size); - } - - inline static size_t large_size_to_chunk_sizeclass(size_t size) - { - return bits::next_pow2_bits(size) - MIN_CHUNK_BITS; - } } // namespace snmalloc diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index cc2787c6f..cf65096ac 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -415,7 +415,7 @@ void test_static_sized_allocs() { // For each small, medium, and large class, do each kind dealloc. This is // mostly to ensure that all of these forms compile. - for (size_t sc = 0; sc < NUM_SIZECLASSES_EXTENDED; sc++) + for (size_t sc = 0; sc < NUM_SIZECLASSES; sc++) { // test_static_sized_alloc(); // test_static_sized_alloc(); diff --git a/src/test/func/sizeclass/sizeclass.cc b/src/test/func/sizeclass/sizeclass.cc index 3d4709e45..8b2faf345 100644 --- a/src/test/func/sizeclass/sizeclass.cc +++ b/src/test/func/sizeclass/sizeclass.cc @@ -78,9 +78,8 @@ int main(int, char**) std::cout << "sizeclass |-> [size_low, size_high] " << std::endl; size_t slab_size = 0; - for (snmalloc::sizeclass_t sz = 0; sz < snmalloc::NUM_SIZECLASSES + 20; sz++) + for (snmalloc::sizeclass_t sz = 0; sz < snmalloc::NUM_SIZECLASSES; sz++) { - // Separate printing for small and medium sizeclasses if ( sz < snmalloc::NUM_SIZECLASSES && slab_size != snmalloc::sizeclass_to_slab_size(sz)) From 27c4a6a55e628ea31fdeadf597128730418014a0 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 25 Aug 2021 16:26:12 +0100 Subject: [PATCH 057/302] Make pagemap check for init on some gets. When we are accessing potentially out of range, then we might be accessing before the pagemap has been initialised. Move the check into the pagemap for better codegen. --- src/backend/pagemap.h | 21 +++++++++++++-- src/mem/localalloc.h | 61 +++++++++++++++++++------------------------ 2 files changed, 46 insertions(+), 36 deletions(-) diff --git a/src/backend/pagemap.h b/src/backend/pagemap.h index 4a923457d..eae470306 100644 --- a/src/backend/pagemap.h +++ b/src/backend/pagemap.h @@ -40,6 +40,12 @@ namespace snmalloc */ T* body{const_cast(&default_value)}; + /** + * The representation of the pagemap, but nullptr if it has not been + * initialised. Used to combine init checking and lookup. + */ + T* body_opt{nullptr}; + /** * If `has_bounds` is set, then these should contain the * bounds of the heap that is being managed by this pagemap. @@ -94,6 +100,7 @@ namespace snmalloc static_assert( has_bounds_ == has_bounds, "Don't set SFINAE template parameter!"); body = address; + body_opt = address; } /** @@ -124,7 +131,7 @@ namespace snmalloc // Put pagemap at start of range. // TODO CHERI capability bound here! body = reinterpret_cast(b); - + body_opt = body; // Advance by size of pagemap. // Note that base needs to be aligned to GRANULARITY for the rest of the // code to work @@ -183,6 +190,7 @@ namespace snmalloc new_body[0] = body[0]; body = new_body; + body_opt = new_body; } /** @@ -209,6 +217,12 @@ namespace snmalloc template const T& get(address_t p) { + if constexpr (potentially_out_of_range) + { + if (unlikely(body_opt == nullptr)) + return default_value; + } + if constexpr (has_bounds) { if (p - base > size) @@ -235,7 +249,10 @@ namespace snmalloc register_range(p, 1); } - return body[p >> SHIFT]; + if constexpr (potentially_out_of_range) + return body_opt[p >> SHIFT]; + else + return body[p >> SHIFT]; } /** diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 8e0978a9a..ba952229c 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -599,45 +599,38 @@ namespace snmalloc { #ifndef SNMALLOC_PASS_THROUGH // TODO bring back the CHERI bits. Wes to review if required. - if (likely(is_initialised())) + MetaEntry entry = + SharedStateHandle::template get_meta_data(address_cast(p_raw)); + auto sizeclass = entry.get_sizeclass(); + if (likely(entry.get_remote() != SharedStateHandle::fake_large_remote)) { - MetaEntry entry = - SharedStateHandle::template get_meta_data(address_cast(p_raw)); - auto sizeclass = entry.get_sizeclass(); - if (likely(entry.get_remote() != SharedStateHandle::fake_large_remote)) - { - auto rsize = sizeclass_to_size(sizeclass); - auto offset = - address_cast(p_raw) & (sizeclass_to_slab_size(sizeclass) - 1); - auto start_offset = round_by_sizeclass(sizeclass, offset); - if constexpr (location == Start) - { - UNUSED(rsize); - return pointer_offset(p_raw, start_offset - offset); - } - else if constexpr (location == End) - return pointer_offset(p_raw, rsize + start_offset - offset - 1); - else - return pointer_offset(p_raw, rsize + start_offset - offset); - } - - // Sizeclass zero of a large allocation is used for not managed by us. - if (likely(sizeclass != 0)) + auto rsize = sizeclass_to_size(sizeclass); + auto offset = + address_cast(p_raw) & (sizeclass_to_slab_size(sizeclass) - 1); + auto start_offset = round_by_sizeclass(sizeclass, offset); + if constexpr (location == Start) { - // This is a large allocation, find start by masking. - auto rsize = bits::one_at_bit(sizeclass); - auto start = pointer_align_down(p_raw, rsize); - if constexpr (location == Start) - return start; - else if constexpr (location == End) - return pointer_offset(start, rsize); - else - return pointer_offset(start, rsize - 1); + UNUSED(rsize); + return pointer_offset(p_raw, start_offset - offset); } + else if constexpr (location == End) + return pointer_offset(p_raw, rsize + start_offset - offset - 1); + else + return pointer_offset(p_raw, rsize + start_offset - offset); } - else + + // Sizeclass zero of a large allocation is used for not managed by us. + if (likely(sizeclass != 0)) { - // Allocator not initialised, so definitely not our allocation + // This is a large allocation, find start by masking. + auto rsize = bits::one_at_bit(sizeclass); + auto start = pointer_align_down(p_raw, rsize); + if constexpr (location == Start) + return start; + else if constexpr (location == End) + return pointer_offset(start, rsize); + else + return pointer_offset(start, rsize - 1); } #else UNUSED(p_raw); From 7eb876995082193d7dd40169d5136e91a9a5f50d Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 25 Aug 2021 16:57:56 +0100 Subject: [PATCH 058/302] Fix codegen for dealloc The PR #359 regressed codegen for deallocation. This fixes it. --- src/mem/localalloc.h | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index ba952229c..df0ce29dd 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -522,11 +522,19 @@ namespace snmalloc ChunkRecord* slab_record = reinterpret_cast(entry.get_metaslab()); slab_record->chunk = CapPtr(p); - check_init([&](CoreAlloc* core_alloc) { - ChunkAllocator::dealloc( - core_alloc->get_backend_local_state(), slab_record, slab_sizeclass); - return nullptr; - }); + check_init( + []( + CoreAlloc* core_alloc, + ChunkRecord* slab_record, + size_t slab_sizeclass) { + ChunkAllocator::dealloc( + core_alloc->get_backend_local_state(), + slab_record, + slab_sizeclass); + return nullptr; + }, + slab_record, + slab_sizeclass); return; } From 70c3e00df77e6a930a16ac76694ce289e2e8a36b Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Fri, 20 Aug 2021 21:19:18 +0100 Subject: [PATCH 059/302] AddressSpace: use Backend to access Pagemap And do so by type, rather than by value. While here, introduce a C++20 concept for this Backend-offered proxy and adjust the template parameters appropriately. This will be useful for the process sandbox code, which needs to mediate stores to the pagemap, but can provide a read-only view. --- src/backend/address_space.h | 30 ++++++------ src/backend/address_space_core.h | 67 +++++++++++++++------------ src/backend/backend.h | 79 +++++++++++++++++--------------- src/backend/backend_concept.h | 57 +++++++++++++++++++++++ src/mem/corealloc.h | 9 ++-- src/mem/localalloc.h | 11 +++-- src/mem/remotecache.h | 5 +- src/mem/slaballocator.h | 3 +- 8 files changed, 168 insertions(+), 93 deletions(-) create mode 100644 src/backend/backend_concept.h diff --git a/src/backend/address_space.h b/src/backend/address_space.h index fbb8be944..cbdc0fd42 100644 --- a/src/backend/address_space.h +++ b/src/backend/address_space.h @@ -42,8 +42,8 @@ namespace snmalloc * part of satisfying the request will be registered with the provided * arena_map for use in subsequent amplification. */ - template - CapPtr reserve(size_t size, Pagemap& pagemap) + template + CapPtr reserve(size_t size) { #ifdef SNMALLOC_TRACING std::cout << "ASM reserve request:" << size << std::endl; @@ -63,7 +63,7 @@ namespace snmalloc { auto base = CapPtr( PAL::template reserve_aligned(size)); - pagemap.register_range(address_cast(base), size); + Pagemap::register_range(address_cast(base), size); return base; } } @@ -71,7 +71,7 @@ namespace snmalloc CapPtr res; { FlagLock lock(spin_lock); - res = core.template reserve(size, pagemap); + res = core.template reserve(size); if (res == nullptr) { // Allocation failed ask OS for more memory @@ -133,12 +133,12 @@ namespace snmalloc return nullptr; } - pagemap.register_range(address_cast(block), block_size); + Pagemap::register_range(address_cast(block), block_size); - core.template add_range(block, block_size, pagemap); + core.template add_range(block, block_size); // still holding lock so guaranteed to succeed. - res = core.template reserve(size, pagemap); + res = core.template reserve(size); } } @@ -156,8 +156,8 @@ namespace snmalloc * This is useful for allowing the space required for alignment to be * used, by smaller objects. */ - template - CapPtr reserve_with_left_over(size_t size, Pagemap& pagemap) + template + CapPtr reserve_with_left_over(size_t size) { SNMALLOC_ASSERT(size >= sizeof(void*)); @@ -165,15 +165,15 @@ namespace snmalloc size_t rsize = bits::next_pow2(size); - auto res = reserve(rsize, pagemap); + auto res = reserve(rsize); if (res != nullptr) { if (rsize > size) { FlagLock lock(spin_lock); - core.template add_range( - pointer_offset(res, size), rsize - size, pagemap); + core.template add_range( + pointer_offset(res, size), rsize - size); } if constexpr (committed) @@ -193,11 +193,11 @@ namespace snmalloc * Add a range of memory to the address space. * Divides blocks into power of two sizes with natural alignment */ - template - void add_range(CapPtr base, size_t length, Pagemap& pagemap) + template + void add_range(CapPtr base, size_t length) { FlagLock lock(spin_lock); - core.add_range(base, length, pagemap); + core.add_range(base, length); } }; } // namespace snmalloc diff --git a/src/backend/address_space_core.h b/src/backend/address_space_core.h index d24b89f6f..68712b0b2 100644 --- a/src/backend/address_space_core.h +++ b/src/backend/address_space_core.h @@ -4,6 +4,7 @@ #include "../mem/allocconfig.h" #include "../mem/metaslab.h" #include "../pal/pal.h" +#include "backend_concept.h" #include #ifdef SNMALLOC_TRACING @@ -67,12 +68,11 @@ namespace snmalloc * to store the next pointer for the list of unused address space of a * particular size. */ - template + template void set_next( size_t align_bits, CapPtr base, - CapPtr next, - Pagemap& pagemap) + CapPtr next) { if (align_bits >= MIN_CHUNK_BITS) { @@ -84,7 +84,7 @@ namespace snmalloc // external_pointer, for example) will not attempt to follow this // "Metaslab" pointer. MetaEntry t(reinterpret_cast(next.unsafe_ptr()), nullptr, 0); - pagemap.set(address_cast(base), t); + Pagemap::set_meta_data(address_cast(base), 1, t); return; } @@ -100,13 +100,14 @@ namespace snmalloc * to store the next pointer for the list of unused address space of a * particular size. */ - template - CapPtr get_next( - size_t align_bits, CapPtr base, Pagemap& pagemap) + template + CapPtr + get_next(size_t align_bits, CapPtr base) { if (align_bits >= MIN_CHUNK_BITS) { - const MetaEntry& t = pagemap.template get(address_cast(base)); + const MetaEntry& t = + Pagemap::template get_meta_data(address_cast(base)); return CapPtr( reinterpret_cast(t.get_metaslab())); } @@ -117,14 +118,15 @@ namespace snmalloc /** * Adds a block to `ranges`. */ - template - void add_block( - size_t align_bits, CapPtr base, Pagemap& pagemap) + template< + SNMALLOC_CONCEPT(ConceptPAL) PAL, + SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> + void add_block(size_t align_bits, CapPtr base) { check_block(base, align_bits); SNMALLOC_ASSERT(align_bits < 64); - set_next(align_bits, base, ranges[align_bits], pagemap); + set_next(align_bits, base, ranges[align_bits]); ranges[align_bits] = base.as_static(); } @@ -132,8 +134,10 @@ namespace snmalloc * Find a block of the correct size. May split larger blocks * to satisfy this request. */ - template - CapPtr remove_block(size_t align_bits, Pagemap& pagemap) + template< + SNMALLOC_CONCEPT(ConceptPAL) PAL, + SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> + CapPtr remove_block(size_t align_bits) { CapPtr first = ranges[align_bits]; if (first == nullptr) @@ -146,7 +150,7 @@ namespace snmalloc // Look for larger block and split up recursively CapPtr bigger = - remove_block(align_bits + 1, pagemap); + remove_block(align_bits + 1); if (bigger != nullptr) { // This block is going to be broken up into sub CHUNK_SIZE blocks @@ -160,10 +164,9 @@ namespace snmalloc size_t left_over_size = bits::one_at_bit(align_bits); auto left_over = pointer_offset(bigger, left_over_size); - add_block( + add_block( align_bits, - Aal::capptr_bound(left_over, left_over_size), - pagemap); + Aal::capptr_bound(left_over, left_over_size)); check_block(left_over.as_static(), align_bits); } check_block(bigger.as_static(), align_bits + 1); @@ -171,7 +174,7 @@ namespace snmalloc } check_block(first, align_bits); - ranges[align_bits] = get_next(align_bits, first, pagemap); + ranges[align_bits] = get_next(align_bits, first); return first.as_void(); } @@ -180,8 +183,10 @@ namespace snmalloc * Add a range of memory to the address space. * Divides blocks into power of two sizes with natural alignment */ - template - void add_range(CapPtr base, size_t length, Pagemap& pagemap) + template< + SNMALLOC_CONCEPT(ConceptPAL) PAL, + SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> + void add_range(CapPtr base, size_t length) { // For start and end that are not chunk sized, we need to // commit the pages to track the allocations. @@ -206,7 +211,7 @@ namespace snmalloc auto b = base.as_static(); check_block(b, align_bits); - add_block(align_bits, b, pagemap); + add_block(align_bits, b); base = pointer_offset(base, align); length -= align; @@ -238,8 +243,10 @@ namespace snmalloc * part of satisfying the request will be registered with the provided * arena_map for use in subsequent amplification. */ - template - CapPtr reserve(size_t size, Pagemap& pagemap) + template< + SNMALLOC_CONCEPT(ConceptPAL) PAL, + SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> + CapPtr reserve(size_t size) { #ifdef SNMALLOC_TRACING std::cout << "ASM Core reserve request:" << size << std::endl; @@ -248,7 +255,7 @@ namespace snmalloc SNMALLOC_ASSERT(bits::is_pow2(size)); SNMALLOC_ASSERT(size >= sizeof(void*)); - return remove_block(bits::next_pow2_bits(size), pagemap); + return remove_block(bits::next_pow2_bits(size)); } /** @@ -258,8 +265,10 @@ namespace snmalloc * This is useful for allowing the space required for alignment to be * used, by smaller objects. */ - template - CapPtr reserve_with_left_over(size_t size, Pagemap& pagemap) + template< + SNMALLOC_CONCEPT(ConceptPAL) PAL, + SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> + CapPtr reserve_with_left_over(size_t size) { SNMALLOC_ASSERT(size >= sizeof(void*)); @@ -267,13 +276,13 @@ namespace snmalloc size_t rsize = bits::next_pow2(size); - auto res = reserve(rsize, pagemap); + auto res = reserve(rsize); if (res != nullptr) { if (rsize > size) { - add_range(pointer_offset(res, size), rsize - size, pagemap); + add_range(pointer_offset(res, size), rsize - size); } } return res; diff --git a/src/backend/backend.h b/src/backend/backend.h index b9b62b9f9..18379106c 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -62,6 +62,38 @@ namespace snmalloc static inline FlatPagemap pagemap; + struct Pagemap + { + /** + * Get the metadata associated with a chunk. + * + * Set template parameter to true if it not an error + * to access a location that is not backed by a chunk. + */ + template + SNMALLOC_FAST_PATH static const MetaEntry& get_meta_data(address_t p) + { + return pagemap.template get(p); + } + + /** + * Set the metadata associated with a chunk. + */ + SNMALLOC_FAST_PATH + static void set_meta_data(address_t p, size_t size, MetaEntry t) + { + for (address_t a = p; a < p + size; a += MIN_CHUNK_SIZE) + { + pagemap.set(a, t); + } + } + + static void register_range(address_t p, size_t sz) + { + pagemap.register_range(p, sz); + } + }; + public: template static std::enable_if_t init() @@ -77,8 +109,8 @@ namespace snmalloc static_assert(fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); auto [heap_base, heap_length] = pagemap.init(base, length); - address_space.add_range( - CapPtr(heap_base), heap_length, pagemap); + address_space.template add_range( + CapPtr(heap_base), heap_length); } private: @@ -136,14 +168,14 @@ namespace snmalloc auto& local = local_state->local_address_space; #endif - p = local.template reserve_with_left_over(size, pagemap); + p = local.template reserve_with_left_over(size); if (p != nullptr) { return p; } auto refill_size = LOCAL_CACHE_BLOCK; - auto refill = global.template reserve(refill_size, pagemap); + auto refill = global.template reserve(refill_size); if (refill == nullptr) return nullptr; @@ -155,10 +187,10 @@ namespace snmalloc } #endif PAL::template notify_using(refill.unsafe_ptr(), refill_size); - local.template add_range(refill, refill_size, pagemap); + local.template add_range(refill, refill_size); // This should succeed - return local.template reserve_with_left_over(size, pagemap); + return local.template reserve_with_left_over(size); } #ifdef SNMALLOC_CHECK_CLIENT @@ -169,7 +201,7 @@ namespace snmalloc size_t rsize = bits::max(OS_PAGE_SIZE, bits::next_pow2(size)); size_t size_request = rsize * 64; - p = global.template reserve(size_request, pagemap); + p = global.template reserve(size_request); if (p == nullptr) return nullptr; @@ -185,7 +217,7 @@ namespace snmalloc SNMALLOC_ASSERT(!is_meta); #endif - p = global.template reserve_with_left_over(size, pagemap); + p = global.template reserve_with_left_over(size); return p; } @@ -249,37 +281,8 @@ namespace snmalloc } MetaEntry t(meta, remote, sizeclass); - - for (address_t a = address_cast(p); - a < address_cast(pointer_offset(p, size)); - a += MIN_CHUNK_SIZE) - { - pagemap.set(a, t); - } + Pagemap::set_meta_data(address_cast(p), size, t); return {p, meta}; } - - /** - * Get the metadata associated with a chunk. - * - * Set template parameter to true if it not an error - * to access a location that is not backed by a chunk. - */ - template - static const MetaEntry& get_meta_data(address_t p) - { - return pagemap.template get(p); - } - - /** - * Set the metadata associated with a chunk. - */ - static void set_meta_data(address_t p, size_t size, MetaEntry t) - { - for (address_t a = p; a < p + size; a += MIN_CHUNK_SIZE) - { - pagemap.set(a, t); - } - } }; } // namespace snmalloc diff --git a/src/backend/backend_concept.h b/src/backend/backend_concept.h new file mode 100644 index 000000000..cca098b83 --- /dev/null +++ b/src/backend/backend_concept.h @@ -0,0 +1,57 @@ +#pragma once + +#ifdef __cpp_concepts +# include +# include "../ds/concept.h" + +namespace snmalloc +{ + class MetaEntry; + + /** + * The core of the static pagemap accessor interface: {get,set}_metadata. + * + * get_metadata takes a bool-ean template parameter indicating whether it may + * be accessing memory that is not known to be committed. + */ + template + concept ConceptBackendMeta = + requires(address_t addr, size_t sz, MetaEntry t) + { + { Meta::set_meta_data(addr, sz, t) } -> ConceptSame; + + { Meta::template get_meta_data(addr) } + -> ConceptSame; + + { Meta::template get_meta_data(addr) } + -> ConceptSame; + }; + + /** + * The pagemap can also be told to commit backing storage for a range of + * addresses. This is broken out to a separate concept so that we can + * annotate which functions expect to do this vs. which merely use the core + * interface above. In practice, use ConceptBackendMetaRange (without the + * underscore) below, which combines this and the core concept, above. + */ + template + concept ConceptBackendMeta_Range = + requires(address_t addr, size_t sz) + { + { Meta::register_range(addr, sz) } -> ConceptSame; + }; + + /** + * The full pagemap accessor interface, with all of {get,set}_metadata and + * register_range. Use this to annotate callers that need the full interface + * and use ConceptBackendMeta for callers that merely need {get,set}_metadata, + * but note that the difference is just for humans and not compilers (since + * concept checking is lower bounding and does not constrain the templatized + * code to use only those affordances given by the concept). + */ + template + concept ConceptBackendMetaRange = + ConceptBackendMeta && ConceptBackendMeta_Range; +} // namespace snmalloc + +#endif diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index b67df1866..588ed5465 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -377,7 +377,7 @@ namespace snmalloc { auto p = message_queue().peek(); auto& entry = - SharedStateHandle::get_meta_data(snmalloc::address_cast(p)); + SharedStateHandle::Pagemap::get_meta_data(snmalloc::address_cast(p)); auto r = message_queue().dequeue(key_global); @@ -538,7 +538,8 @@ namespace snmalloc SNMALLOC_FAST_PATH void dealloc_local_object(void* p) { - auto entry = SharedStateHandle::get_meta_data(snmalloc::address_cast(p)); + auto entry = + SharedStateHandle::Pagemap::get_meta_data(snmalloc::address_cast(p)); if (likely(dealloc_local_object_fast(entry, p, entropy))) return; @@ -664,8 +665,8 @@ namespace snmalloc { bool need_post = true; // Always going to post, so ignore. auto n = p->atomic_read_next(key_global); - auto& entry = - SharedStateHandle::get_meta_data(snmalloc::address_cast(p)); + auto& entry = SharedStateHandle::Pagemap::get_meta_data( + snmalloc::address_cast(p)); handle_dealloc_remote(entry, p, need_post); p = n; } diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index df0ce29dd..3e0b2d277 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -260,7 +260,8 @@ namespace snmalloc std::cout << "Remote dealloc post" << p << " size " << alloc_size(p) << std::endl; #endif - MetaEntry entry = SharedStateHandle::get_meta_data(address_cast(p)); + MetaEntry entry = + SharedStateHandle::Pagemap::get_meta_data(address_cast(p)); local_cache.remote_dealloc_cache.template dealloc( entry.get_remote()->trunc_id(), CapPtr(p), key_global); post_remote_cache(); @@ -472,7 +473,7 @@ namespace snmalloc // in thread local state. const MetaEntry& entry = - SharedStateHandle::get_meta_data(address_cast(p)); + SharedStateHandle::Pagemap::get_meta_data(address_cast(p)); if (likely(local_cache.remote_allocator == entry.get_remote())) { if (likely(CoreAlloc::dealloc_local_object_fast( @@ -582,7 +583,8 @@ namespace snmalloc // To handle this case we require the uninitialised pagemap contain an // entry for the first chunk of memory, that states it represents a large // object, so we can pull the check for null off the fast path. - MetaEntry entry = SharedStateHandle::get_meta_data(address_cast(p_raw)); + MetaEntry entry = + SharedStateHandle::Pagemap::get_meta_data(address_cast(p_raw)); if (likely(entry.get_remote() != SharedStateHandle::fake_large_remote)) return sizeclass_to_size(entry.get_sizeclass()); @@ -608,7 +610,8 @@ namespace snmalloc #ifndef SNMALLOC_PASS_THROUGH // TODO bring back the CHERI bits. Wes to review if required. MetaEntry entry = - SharedStateHandle::template get_meta_data(address_cast(p_raw)); + SharedStateHandle::Pagemap::template get_meta_data( + address_cast(p_raw)); auto sizeclass = entry.get_sizeclass(); if (likely(entry.get_remote() != SharedStateHandle::fake_large_remote)) { diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index a062b9a6f..a397feb49 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -95,7 +95,7 @@ namespace snmalloc { auto [first, last] = list[i].extract_segment(key); MetaEntry entry = - SharedStateHandle::get_meta_data(address_cast(first)); + SharedStateHandle::Pagemap::get_meta_data(address_cast(first)); entry.get_remote()->enqueue(first, last, key); sent_something = true; } @@ -117,7 +117,8 @@ namespace snmalloc // Use the next N bits to spread out remote deallocs in our own // slot. auto r = resend.take(key); - MetaEntry entry = SharedStateHandle::get_meta_data(address_cast(r)); + MetaEntry entry = + SharedStateHandle::Pagemap::get_meta_data(address_cast(r)); auto i = entry.get_remote()->trunc_id(); size_t slot = get_slot(i, post_round); list[slot].add(r, key); diff --git a/src/mem/slaballocator.h b/src/mem/slaballocator.h index 4341ff193..f9886b1ef 100644 --- a/src/mem/slaballocator.h +++ b/src/mem/slaballocator.h @@ -107,7 +107,8 @@ namespace snmalloc << std::endl; #endif MetaEntry entry{meta, remote, sizeclass}; - SharedStateHandle::set_meta_data(address_cast(slab), slab_size, entry); + SharedStateHandle::Pagemap::set_meta_data( + address_cast(slab), slab_size, entry); return {slab, meta}; } From f913f8b820c6b0105560525ab847731c38086867 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Tue, 24 Aug 2021 14:07:57 +0100 Subject: [PATCH 060/302] Rename [gs]et_meta_data to [gs]et_metaentry. Co-authored-by: David Chisnall --- src/backend/address_space_core.h | 4 ++-- src/backend/backend.h | 6 +++--- src/backend/backend_concept.h | 6 +++--- src/mem/corealloc.h | 6 +++--- src/mem/localalloc.h | 8 ++++---- src/mem/remotecache.h | 4 ++-- src/mem/slaballocator.h | 2 +- 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/backend/address_space_core.h b/src/backend/address_space_core.h index 68712b0b2..189771c92 100644 --- a/src/backend/address_space_core.h +++ b/src/backend/address_space_core.h @@ -84,7 +84,7 @@ namespace snmalloc // external_pointer, for example) will not attempt to follow this // "Metaslab" pointer. MetaEntry t(reinterpret_cast(next.unsafe_ptr()), nullptr, 0); - Pagemap::set_meta_data(address_cast(base), 1, t); + Pagemap::set_metaentry(address_cast(base), 1, t); return; } @@ -107,7 +107,7 @@ namespace snmalloc if (align_bits >= MIN_CHUNK_BITS) { const MetaEntry& t = - Pagemap::template get_meta_data(address_cast(base)); + Pagemap::template get_metaentry(address_cast(base)); return CapPtr( reinterpret_cast(t.get_metaslab())); } diff --git a/src/backend/backend.h b/src/backend/backend.h index 18379106c..15afa97d7 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -71,7 +71,7 @@ namespace snmalloc * to access a location that is not backed by a chunk. */ template - SNMALLOC_FAST_PATH static const MetaEntry& get_meta_data(address_t p) + SNMALLOC_FAST_PATH static const MetaEntry& get_metaentry(address_t p) { return pagemap.template get(p); } @@ -80,7 +80,7 @@ namespace snmalloc * Set the metadata associated with a chunk. */ SNMALLOC_FAST_PATH - static void set_meta_data(address_t p, size_t size, MetaEntry t) + static void set_metaentry(address_t p, size_t size, MetaEntry t) { for (address_t a = p; a < p + size; a += MIN_CHUNK_SIZE) { @@ -281,7 +281,7 @@ namespace snmalloc } MetaEntry t(meta, remote, sizeclass); - Pagemap::set_meta_data(address_cast(p), size, t); + Pagemap::set_metaentry(address_cast(p), size, t); return {p, meta}; } }; diff --git a/src/backend/backend_concept.h b/src/backend/backend_concept.h index cca098b83..2ee194218 100644 --- a/src/backend/backend_concept.h +++ b/src/backend/backend_concept.h @@ -18,12 +18,12 @@ namespace snmalloc concept ConceptBackendMeta = requires(address_t addr, size_t sz, MetaEntry t) { - { Meta::set_meta_data(addr, sz, t) } -> ConceptSame; + { Meta::set_metaentry(addr, sz, t) } -> ConceptSame; - { Meta::template get_meta_data(addr) } + { Meta::template get_metaentry(addr) } -> ConceptSame; - { Meta::template get_meta_data(addr) } + { Meta::template get_metaentry(addr) } -> ConceptSame; }; diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 588ed5465..8747941fe 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -377,7 +377,7 @@ namespace snmalloc { auto p = message_queue().peek(); auto& entry = - SharedStateHandle::Pagemap::get_meta_data(snmalloc::address_cast(p)); + SharedStateHandle::Pagemap::get_metaentry(snmalloc::address_cast(p)); auto r = message_queue().dequeue(key_global); @@ -539,7 +539,7 @@ namespace snmalloc SNMALLOC_FAST_PATH void dealloc_local_object(void* p) { auto entry = - SharedStateHandle::Pagemap::get_meta_data(snmalloc::address_cast(p)); + SharedStateHandle::Pagemap::get_metaentry(snmalloc::address_cast(p)); if (likely(dealloc_local_object_fast(entry, p, entropy))) return; @@ -665,7 +665,7 @@ namespace snmalloc { bool need_post = true; // Always going to post, so ignore. auto n = p->atomic_read_next(key_global); - auto& entry = SharedStateHandle::Pagemap::get_meta_data( + auto& entry = SharedStateHandle::Pagemap::get_metaentry( snmalloc::address_cast(p)); handle_dealloc_remote(entry, p, need_post); p = n; diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 3e0b2d277..f263ab7d2 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -261,7 +261,7 @@ namespace snmalloc << std::endl; #endif MetaEntry entry = - SharedStateHandle::Pagemap::get_meta_data(address_cast(p)); + SharedStateHandle::Pagemap::get_metaentry(address_cast(p)); local_cache.remote_dealloc_cache.template dealloc( entry.get_remote()->trunc_id(), CapPtr(p), key_global); post_remote_cache(); @@ -473,7 +473,7 @@ namespace snmalloc // in thread local state. const MetaEntry& entry = - SharedStateHandle::Pagemap::get_meta_data(address_cast(p)); + SharedStateHandle::Pagemap::get_metaentry(address_cast(p)); if (likely(local_cache.remote_allocator == entry.get_remote())) { if (likely(CoreAlloc::dealloc_local_object_fast( @@ -584,7 +584,7 @@ namespace snmalloc // entry for the first chunk of memory, that states it represents a large // object, so we can pull the check for null off the fast path. MetaEntry entry = - SharedStateHandle::Pagemap::get_meta_data(address_cast(p_raw)); + SharedStateHandle::Pagemap::get_metaentry(address_cast(p_raw)); if (likely(entry.get_remote() != SharedStateHandle::fake_large_remote)) return sizeclass_to_size(entry.get_sizeclass()); @@ -610,7 +610,7 @@ namespace snmalloc #ifndef SNMALLOC_PASS_THROUGH // TODO bring back the CHERI bits. Wes to review if required. MetaEntry entry = - SharedStateHandle::Pagemap::template get_meta_data( + SharedStateHandle::Pagemap::template get_metaentry( address_cast(p_raw)); auto sizeclass = entry.get_sizeclass(); if (likely(entry.get_remote() != SharedStateHandle::fake_large_remote)) diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index a397feb49..cee32743e 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -95,7 +95,7 @@ namespace snmalloc { auto [first, last] = list[i].extract_segment(key); MetaEntry entry = - SharedStateHandle::Pagemap::get_meta_data(address_cast(first)); + SharedStateHandle::Pagemap::get_metaentry(address_cast(first)); entry.get_remote()->enqueue(first, last, key); sent_something = true; } @@ -118,7 +118,7 @@ namespace snmalloc // slot. auto r = resend.take(key); MetaEntry entry = - SharedStateHandle::Pagemap::get_meta_data(address_cast(r)); + SharedStateHandle::Pagemap::get_metaentry(address_cast(r)); auto i = entry.get_remote()->trunc_id(); size_t slot = get_slot(i, post_round); list[slot].add(r, key); diff --git a/src/mem/slaballocator.h b/src/mem/slaballocator.h index f9886b1ef..9fb2119d8 100644 --- a/src/mem/slaballocator.h +++ b/src/mem/slaballocator.h @@ -107,7 +107,7 @@ namespace snmalloc << std::endl; #endif MetaEntry entry{meta, remote, sizeclass}; - SharedStateHandle::Pagemap::set_meta_data( + SharedStateHandle::Pagemap::set_metaentry( address_cast(slab), slab_size, entry); return {slab, meta}; } From 3c14a7ddf168bf240b185ccc02f0c2132d137c7a Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Mon, 23 Aug 2021 21:24:16 +0100 Subject: [PATCH 061/302] NFC: Concepts and fixed-pointing interact poorly Fortunately, C++ taketh away and C++ giveth, both, so here we are: a way to detect if we're in the middle of definining a type that uses itself as a template parameter in a way that flows into a concept check and, if so, short-circuit out of the need to actually do any checks. Wonders never cease. --- src/ds/concept.h | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/ds/concept.h b/src/ds/concept.h index 4e489edea..b49545cf4 100644 --- a/src/ds/concept.h +++ b/src/ds/concept.h @@ -38,5 +38,34 @@ namespace snmalloc template concept ConceptSame = std::is_same::value; # endif + + /** + * Some of the types in snmalloc are circular in their definition and use + * templating as a lazy language to carefully tie knots and only pull on the + * whole mess once it's assembled. Unfortunately, concepts amount to eagerly + * demanding the result of the computation. If concepts come into play during + * the circular definition, they may see an incomplete type and so fail (with + * "incomplete type ... used in type trait expression" or similar). However, + * it turns out that SFINAE gives us a way to detect whether a template + * parameter refers to an incomplete type, and short circuit evaluation means + * we can bail on concept checking if we find ourselves in this situation. + * + * See https://devblogs.microsoft.com/oldnewthing/20190710-00/?p=102678 + * + * Unfortunately, C++20 concepts are not first-order things and, in + * particular, cannot themselves be template parameters. So while we would + * love to write a generic Lazy combinator, + * + * template concept C, typename T> + * concept Lazy = !is_type_complete_v || C(); + * + * this will instead have to be inlined at every definition (and referred to + * explicitly at call sites) until C++23 or later. + */ + template + constexpr bool is_type_complete_v = false; + template + constexpr bool is_type_complete_v> = true; + } // namespace snmalloc #endif From 2e1658fc53bbcfc58200629aa36f0c1020fe1562 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Mon, 23 Aug 2021 21:24:42 +0100 Subject: [PATCH 062/302] NFC: Make config objects expose their PoolState types Just introduce the alias publicly so we can grab it when checking concepts --- src/backend/fixedglobalconfig.h | 7 +++++-- src/backend/globalconfig.h | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/backend/fixedglobalconfig.h b/src/backend/fixedglobalconfig.h index 2e4473e1c..dcb6e9f44 100644 --- a/src/backend/fixedglobalconfig.h +++ b/src/backend/fixedglobalconfig.h @@ -14,11 +14,14 @@ namespace snmalloc template class FixedGlobals final : public BackendAllocator { + public: + using GlobalPoolState = PoolState>; + private: using Backend = BackendAllocator; inline static ChunkAllocatorState slab_allocator_state; - inline static PoolState> alloc_pool; + inline static GlobalPoolState alloc_pool; public: static ChunkAllocatorState& @@ -27,7 +30,7 @@ namespace snmalloc return slab_allocator_state; } - static PoolState>& pool() + static GlobalPoolState& pool() { return alloc_pool; } diff --git a/src/backend/globalconfig.h b/src/backend/globalconfig.h index 1e1cd925c..15c24bfa4 100644 --- a/src/backend/globalconfig.h +++ b/src/backend/globalconfig.h @@ -31,13 +31,16 @@ namespace snmalloc */ class Globals final : public BackendAllocator { + public: + using GlobalPoolState = PoolState>; + private: using Backend = BackendAllocator; SNMALLOC_REQUIRE_CONSTINIT inline static ChunkAllocatorState slab_allocator_state; SNMALLOC_REQUIRE_CONSTINIT - inline static PoolState> alloc_pool; + inline static GlobalPoolState alloc_pool; SNMALLOC_REQUIRE_CONSTINIT inline static std::atomic initialised{false}; @@ -52,7 +55,7 @@ namespace snmalloc return slab_allocator_state; } - static PoolState>& pool() + static GlobalPoolState& pool() { return alloc_pool; } From bb6e706590540da260e578a804f0e703b5ce2957 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Tue, 24 Aug 2021 11:52:06 +0100 Subject: [PATCH 063/302] NFC: Add Concept for equality modulo references --- src/ds/concept.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/ds/concept.h b/src/ds/concept.h index b49545cf4..2e4824426 100644 --- a/src/ds/concept.h +++ b/src/ds/concept.h @@ -39,6 +39,13 @@ namespace snmalloc concept ConceptSame = std::is_same::value; # endif + /** + * Equivalence mod std::remove_reference + */ + template + concept ConceptSameModRef = + ConceptSame, std::remove_reference_t>; + /** * Some of the types in snmalloc are circular in their definition and use * templating as a lazy language to carefully tie knots and only pull on the From e530f5629ad583e58ccb4dfd87d8882058fdecc7 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Mon, 23 Aug 2021 21:25:48 +0100 Subject: [PATCH 064/302] NFC: Add a concept for backend Global objects --- src/backend/backend_concept.h | 39 +++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/backend/backend_concept.h b/src/backend/backend_concept.h index 2ee194218..0717c49f4 100644 --- a/src/backend/backend_concept.h +++ b/src/backend/backend_concept.h @@ -3,6 +3,7 @@ #ifdef __cpp_concepts # include # include "../ds/concept.h" +# include "../pal/pal_concept.h" namespace snmalloc { @@ -52,6 +53,44 @@ namespace snmalloc template concept ConceptBackendMetaRange = ConceptBackendMeta && ConceptBackendMeta_Range; + + class CommonConfig; + struct Flags; + + /** + * Backend global objects of type T must obey a number of constraints. They + * must... + * + * * inherit from CommonConfig (see commonconfig.h) + * * specify which PAL is in use via T::Pal + * * have static pagemap accessors via T::Pagemap + * * define a T::LocalState type + * * define T::Options of type snmalloc::Flags + * * expose the global allocator pool via T::pool() + * + */ + template + concept ConceptBackendGlobals = + std::is_base_of::value && + ConceptPAL && + ConceptBackendMetaRange && + requires() + { + typename Globals::LocalState; + + { Globals::Options } -> ConceptSameModRef; + + typename Globals::GlobalPoolState; + { Globals::pool() } -> ConceptSame; + }; + + /** + * The lazy version of the above; please see ds/concept.h and use sparingly. + */ + template + concept ConceptBackendGlobalsLazy = + !is_type_complete_v || ConceptBackendGlobals; + } // namespace snmalloc #endif From 2be44d2e6fd5a90b0a79672f44c450c5cc483604 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Mon, 23 Aug 2021 21:26:56 +0100 Subject: [PATCH 065/302] Use backend global concept on template args Wire the concept into the rest of the tree, being careful to avoid demanding the result of fixed-pointing computation while tying the knot. --- src/mem/corealloc.h | 4 ++-- src/mem/globalalloc.h | 12 ++++++------ src/mem/localalloc.h | 2 +- src/mem/pool.h | 8 +++++--- src/mem/pooled.h | 4 ++-- src/mem/slaballocator.h | 9 ++++++--- 6 files changed, 22 insertions(+), 17 deletions(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 8747941fe..af3b19f47 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -43,13 +43,13 @@ namespace snmalloc * provided externally, then it must be set explicitly with * `init_message_queue`. */ - template + template class CoreAllocator : public std::conditional_t< SharedStateHandle::Options.CoreAllocIsPoolAllocated, Pooled>, NotPoolAllocated> { - template + template friend class LocalAllocator; /** diff --git a/src/mem/globalalloc.h b/src/mem/globalalloc.h index 674b7aeb5..2a0b9caf9 100644 --- a/src/mem/globalalloc.h +++ b/src/mem/globalalloc.h @@ -5,7 +5,7 @@ namespace snmalloc { - template + template inline static void aggregate_stats(Stats& stats) { static_assert( @@ -24,7 +24,7 @@ namespace snmalloc } #ifdef USE_SNMALLOC_STATS - template + template inline static void print_all_stats(std::ostream& o, uint64_t dumpid = 0) { static_assert( @@ -41,7 +41,7 @@ namespace snmalloc } } #else - template + template inline static void print_all_stats(void*& o, uint64_t dumpid = 0) { UNUSED(o); @@ -49,7 +49,7 @@ namespace snmalloc } #endif - template + template inline static void cleanup_unused() { #ifndef SNMALLOC_PASS_THROUGH @@ -83,7 +83,7 @@ namespace snmalloc allocators are empty. If you don't pass a pointer to a bool, then will raise an error all the allocators are not empty. */ - template + template inline static void debug_check_empty(bool* result = nullptr) { #ifndef SNMALLOC_PASS_THROUGH @@ -153,7 +153,7 @@ namespace snmalloc #endif } - template + template inline static void debug_in_use(size_t count) { static_assert( diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index f263ab7d2..02c210c5e 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -59,7 +59,7 @@ namespace snmalloc * core allocator must be provided externally by invoking the `init` method * on this class *before* any allocation-related methods are called. */ - template + template class LocalAllocator { public: diff --git a/src/mem/pool.h b/src/mem/pool.h index f8245c5f6..057c9ba16 100644 --- a/src/mem/pool.h +++ b/src/mem/pool.h @@ -23,7 +23,7 @@ namespace snmalloc { template< typename TT, - typename SharedStateHandle, + SNMALLOC_CONCEPT(ConceptBackendGlobals) SharedStateHandle, PoolState& get_state()> friend class Pool; @@ -42,7 +42,9 @@ namespace snmalloc * SingletonPoolState::pool is the default provider for the PoolState within * the Pool class. */ - template + template< + typename T, + SNMALLOC_CONCEPT(ConceptBackendGlobals) SharedStateHandle> class SingletonPoolState { /** @@ -113,7 +115,7 @@ namespace snmalloc */ template< typename T, - typename SharedStateHandle, + SNMALLOC_CONCEPT(ConceptBackendGlobals) SharedStateHandle, PoolState& get_state() = SingletonPoolState::pool> class Pool { diff --git a/src/mem/pooled.h b/src/mem/pooled.h index d5b988c6e..7b499fd81 100644 --- a/src/mem/pooled.h +++ b/src/mem/pooled.h @@ -13,7 +13,7 @@ namespace snmalloc public: template< typename TT, - typename SharedStateHandle, + SNMALLOC_CONCEPT(ConceptBackendGlobals) SharedStateHandle, PoolState& get_state()> friend class Pool; template @@ -45,4 +45,4 @@ namespace snmalloc return result; } }; -} // namespace snmalloc \ No newline at end of file +} // namespace snmalloc diff --git a/src/mem/slaballocator.h b/src/mem/slaballocator.h index 9fb2119d8..31f6e9471 100644 --- a/src/mem/slaballocator.h +++ b/src/mem/slaballocator.h @@ -75,7 +75,7 @@ namespace snmalloc class ChunkAllocator { public: - template + template static std::pair, Metaslab*> alloc_chunk( typename SharedStateHandle::LocalState& local_state, sizeclass_t sizeclass, @@ -130,7 +130,7 @@ namespace snmalloc return {slab, meta}; } - template + template SNMALLOC_SLOW_PATH static void dealloc( typename SharedStateHandle::LocalState& local_state, ChunkRecord* p, @@ -153,7 +153,10 @@ namespace snmalloc * Backend allocator may use guard pages and separate area of * address space to protect this from corruption. */ - template + template< + typename U, + SNMALLOC_CONCEPT(ConceptBackendGlobals) SharedStateHandle, + typename... Args> static U* alloc_meta_data( typename SharedStateHandle::LocalState* local_state, Args&&... args) { From 3710e351fe74f66f08367021d2b7e89e51143ea6 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Tue, 24 Aug 2021 14:55:00 +0100 Subject: [PATCH 066/302] Move BackendAllocator::pagemap closer to use While here give it a slightly more appropriate name to better distinguish the backing store from the interface class that gets passed around. --- src/backend/backend.h | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/backend/backend.h b/src/backend/backend.h index 15afa97d7..49bbcc4ee 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -58,12 +58,15 @@ namespace snmalloc SNMALLOC_REQUIRE_CONSTINIT static inline AddressSpaceManager address_space; - SNMALLOC_REQUIRE_CONSTINIT - static inline FlatPagemap - pagemap; - - struct Pagemap + class Pagemap { + friend class BackendAllocator; + + SNMALLOC_REQUIRE_CONSTINIT + static inline FlatPagemap + concretePagemap; + + public: /** * Get the metadata associated with a chunk. * @@ -73,7 +76,7 @@ namespace snmalloc template SNMALLOC_FAST_PATH static const MetaEntry& get_metaentry(address_t p) { - return pagemap.template get(p); + return concretePagemap.template get(p); } /** @@ -84,13 +87,13 @@ namespace snmalloc { for (address_t a = p; a < p + size; a += MIN_CHUNK_SIZE) { - pagemap.set(a, t); + concretePagemap.set(a, t); } } static void register_range(address_t p, size_t sz) { - pagemap.register_range(p, sz); + concretePagemap.register_range(p, sz); } }; @@ -100,7 +103,7 @@ namespace snmalloc { static_assert(fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); - pagemap.init(); + Pagemap::concretePagemap.init(); } template @@ -108,7 +111,8 @@ namespace snmalloc { static_assert(fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); - auto [heap_base, heap_length] = pagemap.init(base, length); + auto [heap_base, heap_length] = + Pagemap::concretePagemap.init(base, length); address_space.template add_range( CapPtr(heap_base), heap_length); } From 3af9d3509942ae33d83dd97067a6f44469f1cadd Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Tue, 24 Aug 2021 16:34:30 +0100 Subject: [PATCH 067/302] Plumb LocalState ptrs through to Pagemap accessors David points out that we might not have a static way to get at the pagemap, so it is potentially useful to pass pointers to state objects down from the Allocators. --- src/backend/address_space.h | 27 +++++++----- src/backend/address_space_core.h | 49 ++++++++++++++-------- src/backend/backend.h | 41 ++++++++++++------ src/backend/backend_concept.h | 21 ++++++---- src/backend/fixedglobalconfig.h | 5 ++- src/mem/corealloc.h | 34 +++++++++++---- src/mem/localalloc.h | 14 +++---- src/mem/localcache.h | 5 ++- src/mem/remotecache.h | 13 +++--- src/mem/slaballocator.h | 2 +- src/test/func/fixed_region/fixed_region.cc | 2 +- src/test/func/two_alloc_types/alloc1.cc | 3 +- 12 files changed, 141 insertions(+), 75 deletions(-) diff --git a/src/backend/address_space.h b/src/backend/address_space.h index cbdc0fd42..ada529590 100644 --- a/src/backend/address_space.h +++ b/src/backend/address_space.h @@ -43,7 +43,8 @@ namespace snmalloc * arena_map for use in subsequent amplification. */ template - CapPtr reserve(size_t size) + CapPtr + reserve(typename Pagemap::LocalState* local_state, size_t size) { #ifdef SNMALLOC_TRACING std::cout << "ASM reserve request:" << size << std::endl; @@ -63,7 +64,7 @@ namespace snmalloc { auto base = CapPtr( PAL::template reserve_aligned(size)); - Pagemap::register_range(address_cast(base), size); + Pagemap::register_range(local_state, address_cast(base), size); return base; } } @@ -71,7 +72,7 @@ namespace snmalloc CapPtr res; { FlagLock lock(spin_lock); - res = core.template reserve(size); + res = core.template reserve(local_state, size); if (res == nullptr) { // Allocation failed ask OS for more memory @@ -133,12 +134,12 @@ namespace snmalloc return nullptr; } - Pagemap::register_range(address_cast(block), block_size); + Pagemap::register_range(local_state, address_cast(block), block_size); - core.template add_range(block, block_size); + core.template add_range(local_state, block, block_size); // still holding lock so guaranteed to succeed. - res = core.template reserve(size); + res = core.template reserve(local_state, size); } } @@ -157,7 +158,8 @@ namespace snmalloc * used, by smaller objects. */ template - CapPtr reserve_with_left_over(size_t size) + CapPtr reserve_with_left_over( + typename Pagemap::LocalState* local_state, size_t size) { SNMALLOC_ASSERT(size >= sizeof(void*)); @@ -165,7 +167,7 @@ namespace snmalloc size_t rsize = bits::next_pow2(size); - auto res = reserve(rsize); + auto res = reserve(local_state, rsize); if (res != nullptr) { @@ -173,7 +175,7 @@ namespace snmalloc { FlagLock lock(spin_lock); core.template add_range( - pointer_offset(res, size), rsize - size); + local_state, pointer_offset(res, size), rsize - size); } if constexpr (committed) @@ -194,10 +196,13 @@ namespace snmalloc * Divides blocks into power of two sizes with natural alignment */ template - void add_range(CapPtr base, size_t length) + void add_range( + typename Pagemap::LocalState* local_state, + CapPtr base, + size_t length) { FlagLock lock(spin_lock); - core.add_range(base, length); + core.add_range(local_state, base, length); } }; } // namespace snmalloc diff --git a/src/backend/address_space_core.h b/src/backend/address_space_core.h index 189771c92..0aa5872aa 100644 --- a/src/backend/address_space_core.h +++ b/src/backend/address_space_core.h @@ -70,6 +70,7 @@ namespace snmalloc */ template void set_next( + typename Pagemap::LocalState* local_state, size_t align_bits, CapPtr base, CapPtr next) @@ -84,7 +85,7 @@ namespace snmalloc // external_pointer, for example) will not attempt to follow this // "Metaslab" pointer. MetaEntry t(reinterpret_cast(next.unsafe_ptr()), nullptr, 0); - Pagemap::set_metaentry(address_cast(base), 1, t); + Pagemap::set_metaentry(local_state, address_cast(base), 1, t); return; } @@ -101,13 +102,15 @@ namespace snmalloc * particular size. */ template - CapPtr - get_next(size_t align_bits, CapPtr base) + CapPtr get_next( + typename Pagemap::LocalState* local_state, + size_t align_bits, + CapPtr base) { if (align_bits >= MIN_CHUNK_BITS) { - const MetaEntry& t = - Pagemap::template get_metaentry(address_cast(base)); + const MetaEntry& t = Pagemap::template get_metaentry( + local_state, address_cast(base)); return CapPtr( reinterpret_cast(t.get_metaslab())); } @@ -121,12 +124,15 @@ namespace snmalloc template< SNMALLOC_CONCEPT(ConceptPAL) PAL, SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> - void add_block(size_t align_bits, CapPtr base) + void add_block( + typename Pagemap::LocalState* local_state, + size_t align_bits, + CapPtr base) { check_block(base, align_bits); SNMALLOC_ASSERT(align_bits < 64); - set_next(align_bits, base, ranges[align_bits]); + set_next(local_state, align_bits, base, ranges[align_bits]); ranges[align_bits] = base.as_static(); } @@ -137,7 +143,8 @@ namespace snmalloc template< SNMALLOC_CONCEPT(ConceptPAL) PAL, SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> - CapPtr remove_block(size_t align_bits) + CapPtr + remove_block(typename Pagemap::LocalState* local_state, size_t align_bits) { CapPtr first = ranges[align_bits]; if (first == nullptr) @@ -150,7 +157,7 @@ namespace snmalloc // Look for larger block and split up recursively CapPtr bigger = - remove_block(align_bits + 1); + remove_block(local_state, align_bits + 1); if (bigger != nullptr) { // This block is going to be broken up into sub CHUNK_SIZE blocks @@ -165,6 +172,7 @@ namespace snmalloc auto left_over = pointer_offset(bigger, left_over_size); add_block( + local_state, align_bits, Aal::capptr_bound(left_over, left_over_size)); check_block(left_over.as_static(), align_bits); @@ -174,7 +182,7 @@ namespace snmalloc } check_block(first, align_bits); - ranges[align_bits] = get_next(align_bits, first); + ranges[align_bits] = get_next(local_state, align_bits, first); return first.as_void(); } @@ -186,7 +194,10 @@ namespace snmalloc template< SNMALLOC_CONCEPT(ConceptPAL) PAL, SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> - void add_range(CapPtr base, size_t length) + void add_range( + typename Pagemap::LocalState* local_state, + CapPtr base, + size_t length) { // For start and end that are not chunk sized, we need to // commit the pages to track the allocations. @@ -211,7 +222,7 @@ namespace snmalloc auto b = base.as_static(); check_block(b, align_bits); - add_block(align_bits, b); + add_block(local_state, align_bits, b); base = pointer_offset(base, align); length -= align; @@ -246,7 +257,8 @@ namespace snmalloc template< SNMALLOC_CONCEPT(ConceptPAL) PAL, SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> - CapPtr reserve(size_t size) + CapPtr + reserve(typename Pagemap::LocalState* local_state, size_t size) { #ifdef SNMALLOC_TRACING std::cout << "ASM Core reserve request:" << size << std::endl; @@ -255,7 +267,8 @@ namespace snmalloc SNMALLOC_ASSERT(bits::is_pow2(size)); SNMALLOC_ASSERT(size >= sizeof(void*)); - return remove_block(bits::next_pow2_bits(size)); + return remove_block( + local_state, bits::next_pow2_bits(size)); } /** @@ -268,7 +281,8 @@ namespace snmalloc template< SNMALLOC_CONCEPT(ConceptPAL) PAL, SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> - CapPtr reserve_with_left_over(size_t size) + CapPtr reserve_with_left_over( + typename Pagemap::LocalState* local_state, size_t size) { SNMALLOC_ASSERT(size >= sizeof(void*)); @@ -276,13 +290,14 @@ namespace snmalloc size_t rsize = bits::next_pow2(size); - auto res = reserve(rsize); + auto res = reserve(local_state, rsize); if (res != nullptr) { if (rsize > size) { - add_range(pointer_offset(res, size), rsize - size); + add_range( + local_state, pointer_offset(res, size), rsize - size); } } return res; diff --git a/src/backend/backend.h b/src/backend/backend.h index 49bbcc4ee..9a3964584 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -67,6 +67,12 @@ namespace snmalloc concretePagemap; public: + /** + * Provide a type alias for LocalState so that we can refer to it without + * needing the whole BackendAllocator type at hand. + */ + using LocalState = BackendAllocator::LocalState; + /** * Get the metadata associated with a chunk. * @@ -74,8 +80,10 @@ namespace snmalloc * to access a location that is not backed by a chunk. */ template - SNMALLOC_FAST_PATH static const MetaEntry& get_metaentry(address_t p) + SNMALLOC_FAST_PATH static const MetaEntry& + get_metaentry(LocalState* ls, address_t p) { + UNUSED(ls); return concretePagemap.template get(p); } @@ -83,16 +91,19 @@ namespace snmalloc * Set the metadata associated with a chunk. */ SNMALLOC_FAST_PATH - static void set_metaentry(address_t p, size_t size, MetaEntry t) + static void + set_metaentry(LocalState* ls, address_t p, size_t size, MetaEntry t) { + UNUSED(ls); for (address_t a = p; a < p + size; a += MIN_CHUNK_SIZE) { concretePagemap.set(a, t); } } - static void register_range(address_t p, size_t sz) + static void register_range(LocalState* ls, address_t p, size_t sz) { + UNUSED(ls); concretePagemap.register_range(p, sz); } }; @@ -107,14 +118,15 @@ namespace snmalloc } template - static std::enable_if_t init(void* base, size_t length) + static std::enable_if_t + init(LocalState* local_state, void* base, size_t length) { static_assert(fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); auto [heap_base, heap_length] = Pagemap::concretePagemap.init(base, length); address_space.template add_range( - CapPtr(heap_base), heap_length); + local_state, CapPtr(heap_base), heap_length); } private: @@ -172,14 +184,16 @@ namespace snmalloc auto& local = local_state->local_address_space; #endif - p = local.template reserve_with_left_over(size); + p = local.template reserve_with_left_over( + local_state, size); if (p != nullptr) { return p; } auto refill_size = LOCAL_CACHE_BLOCK; - auto refill = global.template reserve(refill_size); + auto refill = + global.template reserve(local_state, refill_size); if (refill == nullptr) return nullptr; @@ -191,10 +205,12 @@ namespace snmalloc } #endif PAL::template notify_using(refill.unsafe_ptr(), refill_size); - local.template add_range(refill, refill_size); + local.template add_range( + local_state, refill, refill_size); // This should succeed - return local.template reserve_with_left_over(size); + return local.template reserve_with_left_over( + local_state, size); } #ifdef SNMALLOC_CHECK_CLIENT @@ -205,7 +221,7 @@ namespace snmalloc size_t rsize = bits::max(OS_PAGE_SIZE, bits::next_pow2(size)); size_t size_request = rsize * 64; - p = global.template reserve(size_request); + p = global.template reserve(local_state, size_request); if (p == nullptr) return nullptr; @@ -221,7 +237,8 @@ namespace snmalloc SNMALLOC_ASSERT(!is_meta); #endif - p = global.template reserve_with_left_over(size); + p = global.template reserve_with_left_over( + local_state, size); return p; } @@ -285,7 +302,7 @@ namespace snmalloc } MetaEntry t(meta, remote, sizeclass); - Pagemap::set_metaentry(address_cast(p), size, t); + Pagemap::set_metaentry(local_state, address_cast(p), size, t); return {p, meta}; } }; diff --git a/src/backend/backend_concept.h b/src/backend/backend_concept.h index 0717c49f4..96c731900 100644 --- a/src/backend/backend_concept.h +++ b/src/backend/backend_concept.h @@ -17,14 +17,18 @@ namespace snmalloc */ template concept ConceptBackendMeta = - requires(address_t addr, size_t sz, MetaEntry t) + requires( + typename Meta::LocalState* ls, + address_t addr, + size_t sz, + MetaEntry t) { - { Meta::set_metaentry(addr, sz, t) } -> ConceptSame; + { Meta::set_metaentry(ls, addr, sz, t) } -> ConceptSame; - { Meta::template get_metaentry(addr) } + { Meta::template get_metaentry(ls, addr) } -> ConceptSame; - { Meta::template get_metaentry(addr) } + { Meta::template get_metaentry(ls, addr) } -> ConceptSame; }; @@ -37,9 +41,9 @@ namespace snmalloc */ template concept ConceptBackendMeta_Range = - requires(address_t addr, size_t sz) + requires(typename Meta::LocalState* ls, address_t addr, size_t sz) { - { Meta::register_range(addr, sz) } -> ConceptSame; + { Meta::register_range(ls, addr, sz) } -> ConceptSame; }; /** @@ -64,7 +68,7 @@ namespace snmalloc * * inherit from CommonConfig (see commonconfig.h) * * specify which PAL is in use via T::Pal * * have static pagemap accessors via T::Pagemap - * * define a T::LocalState type + * * define a T::LocalState type (and alias it as T::Pagemap::LocalState) * * define T::Options of type snmalloc::Flags * * expose the global allocator pool via T::pool() * @@ -74,6 +78,9 @@ namespace snmalloc std::is_base_of::value && ConceptPAL && ConceptBackendMetaRange && + ConceptSame< + typename Globals::LocalState, + typename Globals::Pagemap::LocalState> && requires() { typename Globals::LocalState; diff --git a/src/backend/fixedglobalconfig.h b/src/backend/fixedglobalconfig.h index dcb6e9f44..78647237f 100644 --- a/src/backend/fixedglobalconfig.h +++ b/src/backend/fixedglobalconfig.h @@ -46,9 +46,10 @@ namespace snmalloc snmalloc::register_clean_up(); } - static void init(void* base, size_t length) + static void + init(typename Backend::LocalState* local_state, void* base, size_t length) { - Backend::init(base, length); + Backend::init(local_state, base, length); } }; } diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index af3b19f47..c70812771 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -114,6 +114,21 @@ namespace snmalloc } } + /** + * Return a pointer to the backend state. + */ + LocalState* backend_state_ptr() + { + if constexpr (SharedStateHandle::Options.CoreAllocOwnsLocalState) + { + return &backend_state; + } + else + { + return backend_state; + } + } + /** * Return this allocator's "truncated" ID, an integer useful as a hash * value of this allocator. @@ -376,8 +391,8 @@ namespace snmalloc for (size_t i = 0; i < REMOTE_BATCH; i++) { auto p = message_queue().peek(); - auto& entry = - SharedStateHandle::Pagemap::get_metaentry(snmalloc::address_cast(p)); + auto& entry = SharedStateHandle::Pagemap::get_metaentry( + backend_state_ptr(), snmalloc::address_cast(p)); auto r = message_queue().dequeue(key_global); @@ -516,9 +531,10 @@ namespace snmalloc SNMALLOC_FAST_PATH bool post() { // stats().remote_post(); // TODO queue not in line! - bool sent_something = attached_cache->remote_dealloc_cache - .post( - public_state()->trunc_id(), key_global); + bool sent_something = + attached_cache->remote_dealloc_cache + .post( + backend_state_ptr(), public_state()->trunc_id(), key_global); return sent_something; } @@ -538,8 +554,8 @@ namespace snmalloc SNMALLOC_FAST_PATH void dealloc_local_object(void* p) { - auto entry = - SharedStateHandle::Pagemap::get_metaentry(snmalloc::address_cast(p)); + auto entry = SharedStateHandle::Pagemap::get_metaentry( + backend_state_ptr(), snmalloc::address_cast(p)); if (likely(dealloc_local_object_fast(entry, p, entropy))) return; @@ -666,7 +682,7 @@ namespace snmalloc bool need_post = true; // Always going to post, so ignore. auto n = p->atomic_read_next(key_global); auto& entry = SharedStateHandle::Pagemap::get_metaentry( - snmalloc::address_cast(p)); + backend_state_ptr(), snmalloc::address_cast(p)); handle_dealloc_remote(entry, p, need_post); p = n; } @@ -681,7 +697,7 @@ namespace snmalloc auto posted = attached_cache->flush( - [&](auto p) { dealloc_local_object(p); }); + backend_state_ptr(), [&](auto p) { dealloc_local_object(p); }); // We may now have unused slabs, return to the global allocator. for (sizeclass_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++) diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 02c210c5e..36f4544b9 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -260,8 +260,8 @@ namespace snmalloc std::cout << "Remote dealloc post" << p << " size " << alloc_size(p) << std::endl; #endif - MetaEntry entry = - SharedStateHandle::Pagemap::get_metaentry(address_cast(p)); + MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry( + core_alloc->backend_state_ptr(), address_cast(p)); local_cache.remote_dealloc_cache.template dealloc( entry.get_remote()->trunc_id(), CapPtr(p), key_global); post_remote_cache(); @@ -472,8 +472,8 @@ namespace snmalloc // before init, that maps null to a remote_deallocator that will never be // in thread local state. - const MetaEntry& entry = - SharedStateHandle::Pagemap::get_metaentry(address_cast(p)); + const MetaEntry& entry = SharedStateHandle::Pagemap::get_metaentry( + core_alloc->backend_state_ptr(), address_cast(p)); if (likely(local_cache.remote_allocator == entry.get_remote())) { if (likely(CoreAlloc::dealloc_local_object_fast( @@ -583,8 +583,8 @@ namespace snmalloc // To handle this case we require the uninitialised pagemap contain an // entry for the first chunk of memory, that states it represents a large // object, so we can pull the check for null off the fast path. - MetaEntry entry = - SharedStateHandle::Pagemap::get_metaentry(address_cast(p_raw)); + MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry( + core_alloc->backend_state_ptr(), address_cast(p_raw)); if (likely(entry.get_remote() != SharedStateHandle::fake_large_remote)) return sizeclass_to_size(entry.get_sizeclass()); @@ -611,7 +611,7 @@ namespace snmalloc // TODO bring back the CHERI bits. Wes to review if required. MetaEntry entry = SharedStateHandle::Pagemap::template get_metaentry( - address_cast(p_raw)); + core_alloc->backend_state_ptr(), address_cast(p_raw)); auto sizeclass = entry.get_sizeclass(); if (likely(entry.get_remote() != SharedStateHandle::fake_large_remote)) { diff --git a/src/mem/localcache.h b/src/mem/localcache.h index 7e7687f05..74a35fbab 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -74,7 +74,8 @@ namespace snmalloc size_t allocator_size, typename SharedStateHandle, typename DeallocFun> - bool flush(DeallocFun dealloc) + bool flush( + typename SharedStateHandle::LocalState* local_state, DeallocFun dealloc) { auto& key = entropy.get_free_list_key(); @@ -92,7 +93,7 @@ namespace snmalloc } return remote_dealloc_cache.post( - remote_allocator->trunc_id(), key_global); + local_state, remote_allocator->trunc_id(), key_global); } template diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index cee32743e..4b88d23bf 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -76,7 +76,10 @@ namespace snmalloc } template - bool post(RemoteAllocator::alloc_id_t id, const FreeListKey& key) + bool post( + typename SharedStateHandle::LocalState* local_state, + RemoteAllocator::alloc_id_t id, + const FreeListKey& key) { SNMALLOC_ASSERT(initialised); size_t post_round = 0; @@ -94,8 +97,8 @@ namespace snmalloc if (!list[i].empty()) { auto [first, last] = list[i].extract_segment(key); - MetaEntry entry = - SharedStateHandle::Pagemap::get_metaentry(address_cast(first)); + MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry( + local_state, address_cast(first)); entry.get_remote()->enqueue(first, last, key); sent_something = true; } @@ -117,8 +120,8 @@ namespace snmalloc // Use the next N bits to spread out remote deallocs in our own // slot. auto r = resend.take(key); - MetaEntry entry = - SharedStateHandle::Pagemap::get_metaentry(address_cast(r)); + MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry( + local_state, address_cast(r)); auto i = entry.get_remote()->trunc_id(); size_t slot = get_slot(i, post_round); list[slot].add(r, key); diff --git a/src/mem/slaballocator.h b/src/mem/slaballocator.h index 31f6e9471..f714c07ac 100644 --- a/src/mem/slaballocator.h +++ b/src/mem/slaballocator.h @@ -108,7 +108,7 @@ namespace snmalloc #endif MetaEntry entry{meta, remote, sizeclass}; SharedStateHandle::Pagemap::set_metaentry( - address_cast(slab), slab_size, entry); + &local_state, address_cast(slab), slab_size, entry); return {slab, meta}; } diff --git a/src/test/func/fixed_region/fixed_region.cc b/src/test/func/fixed_region/fixed_region.cc index db75ccf96..527377221 100644 --- a/src/test/func/fixed_region/fixed_region.cc +++ b/src/test/func/fixed_region/fixed_region.cc @@ -29,7 +29,7 @@ int main() std::cout << "Allocated region " << oe_base << " - " << pointer_offset(oe_base, size) << std::endl; - CustomGlobals::init(oe_base, size); + CustomGlobals::init(nullptr, oe_base, size); FixedAlloc a; size_t object_size = 128; diff --git a/src/test/func/two_alloc_types/alloc1.cc b/src/test/func/two_alloc_types/alloc1.cc index feb47bd09..8a21c8004 100644 --- a/src/test/func/two_alloc_types/alloc1.cc +++ b/src/test/func/two_alloc_types/alloc1.cc @@ -19,5 +19,6 @@ namespace snmalloc extern "C" void oe_allocator_init(void* base, void* end) { - snmalloc::CustomGlobals::init(base, address_cast(end) - address_cast(base)); + snmalloc::CustomGlobals::init( + nullptr, base, address_cast(end) - address_cast(base)); } From aedb666cdda5107cc18a58ffb32e673667fc48b5 Mon Sep 17 00:00:00 2001 From: Istvan Haller <31476121+ihaller@users.noreply.github.com> Date: Tue, 31 Aug 2021 21:59:43 +0100 Subject: [PATCH 068/302] Added constness to argv in Opt (#383) --- src/test/opt.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/opt.h b/src/test/opt.h index 1d48e7abb..356b10c04 100644 --- a/src/test/opt.h +++ b/src/test/opt.h @@ -11,10 +11,10 @@ namespace opt { private: int argc; - char** argv; + const char* const* argv; public: - Opt(int argc, char** argv) : argc(argc), argv(argv) {} + Opt(int argc, const char* const* argv) : argc(argc), argv(argv) {} bool has(const char* opt) { From c70c23ad74adfd1413bbd19889006e29f1109e44 Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Fri, 3 Sep 2021 11:31:05 +0100 Subject: [PATCH 069/302] CMake cleanup. (#384) Modernise and tidy the CMake a bit: - Use generator expressions for a lot of conditionals so that things are more reliable with multi-config generators (and less verbose). - Remove C as a needed language. None of the code was C but we were using C to test if headers worked. This was fragile because a build with `CMAKE_CXX_COMPILER` set might have checked things compiled with the system C compiler and then failed when the specified C++ compiler used different headers. - Rename the `BACKTRACE_HEADER` macro to `SNMALLOC_BACKTRACE_HEADER`. This is exposed into code that consumes snmalloc and so should be 'namespaced' (to the degree that's possible with C macros). - Clean up the options and use dependent options to hide options that are not always relevant. - Use functions instead of macros for better variable scoping. - Factor out some duplicated bits into functions. - Update to the latest way of telling CMake to use C++17 or C++20. - Migrate everything that's setting global properties to setting only per-target properties. - Link with -nostdlib++ if it's available. If it isn't, fall back to enabling the C language and linking with the C compiler. - Make the per-test log messages verbose outputs. These kept scrolling important messages off the top of the screen for me. - Make building as a header-only library a public option. - Add install targets that install all of the headers and provide a config option. This works with the header-only configuration for integration with things like vcpkg. - Fix a missing `#endif` in the `malloc_useable_size` check. This was failing co compile on all platforms because of the missing `#endif`. - Bump the minimum version to 3.14 so that we have access to target_link_options. This is necessary to use generator expressions for linker flags. - Make the linker error if the shim libraries depend on symbols that are not defined in the explicitly-provided libraries. - Make the old-Ubuntu CI jobs use C++17 explicitly (previously CMake was silently ignoring the fact that the compiler didn't support C++20) - Fix errors found by the more aggressive linking mode. With these changes, it's now possible to install snmalloc and then, in another project, do something like this: ```cmake find_package(snmalloc CONFIG REQUIRED) target_link_libraries(t1 snmalloc::snmalloc) target_link_libraries(t2 snmalloc::snmallocshim-static) ``` In this example, `t1` gets all of the compile flags necessary to include snmalloc headers for its build configuration. `t2` is additionally linked to the snmalloc static shim library. --- .github/workflows/main.yml | 9 +- CMakeLists.txt | 342 ++++++++++++++++++++++--------------- src/mem/threadalloc.h | 11 +- src/override/new.cc | 6 +- src/pal/pal_posix.h | 4 +- 5 files changed, 230 insertions(+), 142 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c04e28d8b..157ea9e10 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,12 +18,13 @@ jobs: # Build each combination of OS and release/debug variants os: [ "ubuntu-latest", "ubuntu-18.04", "macos-11", "macos-10.15", "freebsd-12.2", "freebsd-13.0" ] build-type: [ Release, Debug ] - extra-cmake-flags: [ " " ] + extra-cmake-flags: [ "" ] # Modify the complete matrix include: # Provide the dependency installation for each platform - os: "ubuntu-18.04" dependencies: "sudo apt install ninja-build" + cmake-flags: "-DSNMALLOC_USE_CXX17=ON" - os: "ubuntu-latest" dependencies: "sudo apt install ninja-build" - os: "macos-11" @@ -68,7 +69,7 @@ jobs: - name: Install build dependencies run: ${{ matrix.dependencies }} - name: Configure CMake - run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{matrix.build-type}} -G Ninja ${{ matrix.extra-cmake-flags }} + run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{matrix.build-type}} -G Ninja ${{ matrix.cmake-flags }} ${{ matrix.extra-cmake-flags }} # Build with a nice ninja status line - name: Build working-directory: ${{github.workspace}}/build @@ -201,7 +202,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Configure CMake - run: cmake -B ${{github.workspace}}/build + run: cmake -B ${{github.workspace}}/build -DSNMALLOC_USE_CXX17=ON - name: Install clang-tidy run: sudo apt install clang-tidy-9 # Run the clang-format check and error if it generates a diff @@ -213,7 +214,7 @@ jobs: git diff --exit-code - name: Run clang-tidy run: | - clang-tidy-9 src/override/malloc.cc -header-filter="`pwd`/*" -warnings-as-errors='*' -export-fixes=tidy.fail -- -std=c++17 -mcx16 -DSNMALLOC_PLATFORM_HAS_GETENTROPY=0 + clang-tidy-9 src/override/malloc.cc -header-filter="`pwd`/*" -warnings-as-errors='*' -export-fixes=tidy.fail -- -std=c++17 -mcx16 -DSNMALLOC_PLATFORM_HAS_GETENTROPY=0 if [ -f tidy.fail ] ; then cat tidy.fail exit 1 diff --git a/CMakeLists.txt b/CMakeLists.txt index c8b0d0da9..79950cf66 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ -cmake_minimum_required(VERSION 3.8) -project(snmalloc C CXX) +cmake_minimum_required(VERSION 3.14) +project(snmalloc CXX) if (NOT CMAKE_BUILD_TYPE) message(STATUS "No build type selected, default to: Release") @@ -7,25 +7,70 @@ if (NOT CMAKE_BUILD_TYPE) endif() include(CheckCXXCompilerFlag) -include(CheckCSourceCompiles) +include(CheckCXXSourceCompiles) +include(CMakeDependentOption) +option(SNMALLOC_HEADER_ONLY_LIBRARY "Use snmalloc has a header-only library" OFF) +# Options that apply globally option(USE_SNMALLOC_STATS "Track allocation stats" OFF) option(SNMALLOC_CI_BUILD "Disable features not sensible for CI" OFF) -option(EXPOSE_EXTERNAL_PAGEMAP "Expose the global pagemap" OFF) -option(EXPOSE_EXTERNAL_RESERVE "Expose an interface to reserve memory using the default memory provider" OFF) -option(SNMALLOC_RUST_SUPPORT "Build static library for rust" OFF) -option(SNMALLOC_STATIC_LIBRARY "Build static libraries" ON) option(SNMALLOC_QEMU_WORKAROUND "Disable using madvise(DONT_NEED) to zero memory on Linux" Off) -option(SNMALLOC_OPTIMISE_FOR_CURRENT_MACHINE "Compile for current machine architecture" Off) -set(SNMALLOC_STATIC_LIBRARY_PREFIX "sn_" CACHE STRING "Static library function prefix") option(SNMALLOC_USE_CXX17 "Build as C++17 for legacy support." OFF) -option(SNMALLOC_USE_PTHREAD_DESTRUCTORS "Build using pthread destructors. Reduces the system dependencies, but may interact badly with C++ on some platforms." OFF) +# Options that apply only if we're not building the header-only library +cmake_dependent_option(SNMALLOC_RUST_SUPPORT "Build static library for rust" OFF "NOT SNMALLOC_HEADER_ONLY_LIBRARY" OFF) +cmake_dependent_option(SNMALLOC_STATIC_LIBRARY "Build static libraries" ON "NOT SNMALLOC_HEADER_ONLY_LIBRARY" OFF) +cmake_dependent_option(SNMALLOC_OPTIMISE_FOR_CURRENT_MACHINE "Compile for current machine architecture" Off "NOT SNMALLOC_HEADER_ONLY_LIBRARY" OFF) +if (NOT SNMALLOC_HEADER_ONLY_LIBRARY) + # Pick a sensible default for the thread cleanup mechanism + if (${CMAKE_SYSTEM_NAME} STREQUAL FreeBSD) + set(SNMALLOC_CLEANUP_DEFAULT THREAD_CLEANUP) + elseif (UNIX AND NOT APPLE) + set(SNMALLOC_CLEANUP_DEFAULT PTHREAD_DESTRUCTORS) + else () + set(SNMALLOC_CLEANUP_DEFAULT CXX11_DESTRUCTORS) + endif() + # Specify the thread cleanup mechanism to use. + set(SNMALLOC_CLEANUP ${SNMALLOC_CLEANUP_DEFAULT} CACHE STRING "The mechanism that snmalloc will use for thread destructors. Valid options are: CXX11_DESTRUCTORS (use C++11 destructors, may depend on the C++ runtime library), PTHREAD_DESTRUCTORS (use pthreads, may interact badly with C++ on some platforms, such as macOS) THREAD_CLEANUP (depend on an explicit call to _malloc_thread_cleanup on thread exit, supported by FreeBSD's threading implementation and possibly elsewhere)") + set_property(CACHE SNMALLOC_CLEANUP PROPERTY STRINGS THREAD_CLEANUP PTHREAD_DESTRUCTORS CXX11_DESTRUCTORS) + + set(SNMALLOC_STATIC_LIBRARY_PREFIX "sn_" CACHE STRING "Static library function prefix") +else () + unset(SNMALLOC_STATIC_LIBRARY_PREFIX CACHE) + unset(SNMALLOC_CLEANUP CACHE) +endif () + +if (NOT SNMALLOC_CLEANUP STREQUAL CXX11_DESTRUCTORS) + set(CMAKE_CXX_IMPLICIT_LINK_LIBRARIES "") +endif() + +# If CheckLinkerFlag doesn't exist then provide a dummy implementation that +# always fails. The fallback can be removed when we move to CMake 3.18 as the +# baseline. +include(CheckLinkerFlag OPTIONAL RESULT_VARIABLE CHECK_LINKER_FLAG) +if (NOT CHECK_LINKER_FLAG) + function(check_linker_flag) + endfunction() +endif () + +if (NOT MSVC AND NOT (SNMALLOC_CLEANUP STREQUAL CXX11_DESTRUCTORS)) + # If the target compiler doesn't support -nostdlib++ then we must enable C at + # the global scope for the fallbacks to work. + check_linker_flag(CXX "-nostdlib++" SNMALLOC_LINKER_SUPPORT_NOSTDLIBXX) + if (NOT SNMALLOC_LINKER_SUPPORT_NOSTDLIBXX AND NOT SNMALLOC_HEADER_ONLY_LIBRARY) + enable_language(C) + endif() +endif() + +# Define a generator expression for things that will be enabled in either CI +# builds or debug mode. +set(ci_or_debug "$,$>") # malloc.h will error if you include it on FreeBSD, so this test must not # unconditionally include it. -CHECK_C_SOURCE_COMPILES(" +CHECK_CXX_SOURCE_COMPILES(" #if __has_include() #include +#endif #if __has_include() #include #else @@ -35,9 +80,9 @@ size_t malloc_usable_size(const void* ptr) { return 0; } int main() { return 0; } " CONST_QUALIFIED_MALLOC_USABLE_SIZE) -# older libcs might not have getentropy, e.g. it appeared in gliobc 2.25 +# Some libcs might not have getentropy, e.g. it appeared in glibc 2.25 # so we need to fallback if we cannot compile this -CHECK_C_SOURCE_COMPILES(" +CHECK_CXX_SOURCE_COMPILES(" #if __has_include() # include #endif @@ -51,8 +96,11 @@ int main() { } " SNMALLOC_PLATFORM_HAS_GETENTROPY) -# Provide as macro so other projects can reuse -macro(warnings_high) +# Provide as function so other projects can reuse +# FIXME: This modifies some variables that may or may not be the ones that +# provide flags and so is broken by design. It should be removed once Verona +# no longer uses it. +function(warnings_high) if(MSVC) # Force to always compile with W4 if(CMAKE_CXX_FLAGS MATCHES "/W[0-4]") @@ -68,9 +116,9 @@ macro(warnings_high) endif () add_compile_options(-Wall -Wextra -Werror -Wundef) endif() -endmacro() +endfunction() -macro(clangformat_targets) +function(clangformat_targets) # The clang-format tool is installed under a variety of different names. Try # to find a sensible one. Only look for versions 9 explicitly - we don't # know whether our clang-format file will work with newer versions of the @@ -95,118 +143,96 @@ macro(clangformat_targets) -i ${ALL_SOURCE_FILES}) endif() -endmacro() +endfunction() + +# The main target for snmalloc. This is the exported target for the +# header-only configuration and is used as a dependency for all of the builds +# that compile anything. +add_library(snmalloc INTERFACE) -# Have to set this globally, as can't be set on an interface target. if(SNMALLOC_USE_CXX17) - set(CMAKE_CXX_STANDARD 17) + target_compile_definitions(snmalloc INTERFACE -DSNMALLOC_USE_CXX17) + target_compile_features(snmalloc INTERFACE cxx_std_17) else() - set(CMAKE_CXX_STANDARD 20) + target_compile_features(snmalloc INTERFACE cxx_std_20) endif() -# The main target for snmalloc -add_library(snmalloc_lib INTERFACE) -target_include_directories(snmalloc_lib INTERFACE src/) - -if(SNMALLOC_USE_CXX17) - target_compile_definitions(snmalloc_lib INTERFACE -DSNMALLOC_USE_CXX17) -endif() +# Add header paths. +target_include_directories(snmalloc + INTERFACE + $ + $) if(NOT MSVC) - find_package(Threads REQUIRED COMPONENTS snmalloc_lib) - target_link_libraries(snmalloc_lib INTERFACE ${CMAKE_THREAD_LIBS_INIT}) - if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") - target_link_libraries(snmalloc_lib INTERFACE atomic) - endif() + find_package(Threads REQUIRED COMPONENTS snmalloc) + target_link_libraries(snmalloc INTERFACE + ${CMAKE_THREAD_LIBS_INIT} $<$:atomic>) endif() if (WIN32) set(WIN8COMPAT FALSE CACHE BOOL "Avoid Windows 10 APIs") - if (WIN8COMPAT) - target_compile_definitions(snmalloc_lib INTERFACE -DWINVER=0x0603) - message(STATUS "snmalloc: Avoiding Windows 10 APIs") - else() - message(STATUS "snmalloc: Using Windows 10 APIs") - # VirtualAlloc2 is exposed by mincore.lib, not Kernel32.lib (as the - # documentation says) - target_link_libraries(snmalloc_lib INTERFACE mincore) - endif() + target_compile_definitions(snmalloc INTERFACE $<$:WINVER=0x0603>) + # VirtualAlloc2 is exposed by mincore.lib, not Kernel32.lib (as the + # documentation says) + target_link_libraries(snmalloc INTERFACE $<$>:mincore>) + message(STATUS "snmalloc: Avoiding Windows 10 APIs is ${WIN8COMPAT}") endif() -# detect support for cmpxchg16b; werror is needed to make sure mcx16 must be used by targets +# Detect support for cmpxchg16b; Werror is needed to make sure mcx16 must be used by targets check_cxx_compiler_flag("-Werror -Wextra -Wall -mcx16" SNMALLOC_COMPILER_SUPPORT_MCX16) if(SNMALLOC_COMPILER_SUPPORT_MCX16) - target_compile_options(snmalloc_lib INTERFACE $<$:-mcx16>) + target_compile_options(snmalloc INTERFACE $<$:-mcx16>) endif() -if(USE_SNMALLOC_STATS) - target_compile_definitions(snmalloc_lib INTERFACE -DUSE_SNMALLOC_STATS) +# Helper function that conditionally defines a macro for the build target if +# the CMake variable of the same name is set. +function(add_as_define FLAG) + target_compile_definitions(snmalloc INTERFACE $<$:${FLAG}>) +endfunction() + +add_as_define(USE_SNMALLOC_STATS) +add_as_define(SNMALLOC_QEMU_WORKAROUND) +add_as_define(SNMALLOC_CI_BUILD) +add_as_define(SNMALLOC_PLATFORM_HAS_GETENTROPY) +target_compile_definitions(snmalloc INTERFACE $<$:MALLOC_USABLE_SIZE_QUALIFIER=const>) + +# In debug and CI builds, link the backtrace library so that we can get stack +# traces on errors. +find_package(Backtrace) +if(${Backtrace_FOUND}) + target_compile_definitions(snmalloc INTERFACE + $<${ci_or_debug}:SNMALLOC_BACKTRACE_HEADER="${Backtrace_HEADER}">) + target_link_libraries(snmalloc INTERFACE + $<${ci_or_debug}:${Backtrace_LIBRARIES}>) + target_include_directories(snmalloc INTERFACE + $<${ci_or_debug}:${Backtrace_INCLUDE_DIRS}>) endif() -if(SNMALLOC_QEMU_WORKAROUND) - target_compile_definitions(snmalloc_lib INTERFACE -DSNMALLOC_QEMU_WORKAROUND) -endif() - -if(SNMALLOC_CI_BUILD) - target_compile_definitions(snmalloc_lib INTERFACE -DSNMALLOC_CI_BUILD) -endif() - -if(SNMALLOC_PLATFORM_HAS_GETENTROPY) - target_compile_definitions(snmalloc_lib INTERFACE -DSNMALLOC_PLATFORM_HAS_GETENTROPY) -endif() - -if(SNMALLOC_USE_PTHREAD_DESTRUCTORS) - target_compile_definitions(snmalloc_lib INTERFACE -DSNMALLOC_USE_PTHREAD_DESTRUCTORS) -endif() - -if(CONST_QUALIFIED_MALLOC_USABLE_SIZE) - target_compile_definitions(snmalloc_lib INTERFACE -DMALLOC_USABLE_SIZE_QUALIFIER=const) +if(MSVC) + target_compile_definitions(snmalloc INTERFACE -D_HAS_EXCEPTIONS=0) +else() + # All symbols are always dynamic on haiku and -rdynamic is redundant (and unsupported). + if (NOT CMAKE_SYSTEM_NAME STREQUAL "Haiku") + # Get better stack traces in CI and debug builds. + target_link_options(snmalloc INTERFACE $<${ci_or_debug}:-rdynamic>) + endif() endif() +check_linker_flag(CXX "-Wl,--no-undefined" SNMALLOC_LINKER_SUPPORT_NO_ALLOW_SHLIB_UNDEF) -# To build with just the header library target define SNMALLOC_ONLY_HEADER_LIBRARY -# in containing Cmake file. -if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) +function(add_warning_flags name) + target_compile_options(${name} PRIVATE + $<$:/Zi /W4 /WX /wd4127 /wd4324 /wd4201 $<${ci_or_debug}:/DEBUG>> + $<$,$>>:-fno-exceptions -fno-rtti -Wall -Wextra -Werror -Wundef> + $<$:-Wsign-conversion -Wconversion>) + target_link_options(${name} PRIVATE $<$:-Wl,--no-undefined>) +endfunction() - warnings_high() - if(MSVC) - set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Zi") - set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /DEBUG") - target_compile_definitions(snmalloc_lib INTERFACE -D_HAS_EXCEPTIONS=0) - add_compile_options(/std:c++latest) - else() - add_compile_options(-fno-exceptions -fno-rtti -fomit-frame-pointer -ffunction-sections) - # Static TLS model is unsupported on Haiku. - # All symbols are always dynamic on haiku and -rdynamic is redundant (and unsupported). - if (NOT CMAKE_SYSTEM_NAME MATCHES "Haiku") - add_compile_options(-ftls-model=initial-exec) - if(SNMALLOC_CI_BUILD OR (${CMAKE_BUILD_TYPE} MATCHES "Debug")) - # Get better stack traces in CI and Debug. - target_link_libraries(snmalloc_lib INTERFACE "-rdynamic") - add_compile_options(-g) - endif() - endif() +# To build with just the header library target define SNMALLOC_HEADER_ONLY_LIBRARY +if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) - if(SNMALLOC_OPTIMISE_FOR_CURRENT_MACHINE) - check_cxx_compiler_flag(-march=native SUPPORT_MARCH_NATIVE) - if (SUPPORT_MARCH_NATIVE) - add_compile_options(-march=native) - else() - message(WARNING "Compiler does not support `-march=native` required by SNMALLOC_OPTIMISE_FOR_CURRENT_MACHINE") - endif() - endif() - - find_package(Backtrace) - if(${Backtrace_FOUND}) - target_compile_definitions(snmalloc_lib INTERFACE -DBACKTRACE_HEADER="${Backtrace_HEADER}") - target_link_libraries(snmalloc_lib INTERFACE ${Backtrace_LIBRARIES}) - target_include_directories(snmalloc_lib INTERFACE ${Backtrace_INCLUDE_DIRS}) - endif() - - endif() - - macro(subdirlist result curdir) + function(subdirlist result curdir) file(GLOB children CONFIGURE_DEPENDS LIST_DIRECTORIES true RELATIVE ${curdir} ${curdir}/* ) set(dirlist "") foreach(child ${children}) @@ -214,43 +240,63 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) list(APPEND dirlist ${child}) endif() endforeach() - set(${result} ${dirlist}) - endmacro() + set(${result} ${dirlist} PARENT_SCOPE) + endfunction() - if(CMAKE_VERSION VERSION_LESS 3.14) - set(CMAKE_REQUIRED_LIBRARIES -fuse-ld=lld) - else() - set(CMAKE_REQUIRED_LINK_OPTIONS -fuse-ld=lld) - endif() + set(CMAKE_REQUIRED_LINK_OPTIONS -fuse-ld=lld) check_cxx_source_compiles("int main() { return 1; }" LLD_WORKS) + if (LLD_WORKS) + message(STATUS "Using LLD to link snmalloc shims") + endif() - macro(add_shim name type) + function(add_shim name type) add_library(${name} ${type} ${ARGN}) - target_link_libraries(${name} snmalloc_lib) - if(NOT MSVC) - target_compile_definitions(${name} PRIVATE "SNMALLOC_EXPORT=__attribute__((visibility(\"default\")))") - endif() + target_link_libraries(${name} snmalloc) set_target_properties(${name} PROPERTIES CXX_VISIBILITY_PRESET hidden) + target_compile_definitions(${name} PRIVATE "SNMALLOC_USE_${SNMALLOC_CLEANUP}") - # Ensure that we do not link against C++ stdlib when compiling shims. + add_warning_flags(${name}) if(NOT MSVC) - set_target_properties(${name} PROPERTIES LINKER_LANGUAGE C) - if (LLD_WORKS) - message("Using LLD.") + target_compile_definitions(${name} PRIVATE "SNMALLOC_EXPORT=__attribute__((visibility(\"default\")))") + target_compile_options(${name} PRIVATE + -fomit-frame-pointer -ffunction-sections) + # Static TLS model is unsupported on Haiku. + if (NOT CMAKE_SYSTEM_NAME STREQUAL "Haiku") + target_compile_options(${name} PRIVATE -ftls-model=initial-exec) + target_compile_options(${name} PRIVATE $<$:-g>) + endif() - # Only include the dependencies that actually are touched. - # Required so C++ library is not included with direct pthread access. - target_link_libraries(${name} -Wl,--as-needed) + if(SNMALLOC_OPTIMISE_FOR_CURRENT_MACHINE) + check_cxx_compiler_flag(-march=native SUPPORT_MARCH_NATIVE) + if (SUPPORT_MARCH_NATIVE) + target_compile_options(${name} -march=native) + else() + message(WARNING "Compiler does not support `-march=native` required by SNMALLOC_OPTIMISE_FOR_CURRENT_MACHINE") + endif() + endif() - # Remove all the duplicate new/malloc and free/delete definitions - target_link_libraries(${name} -Wl,--icf=all -fuse-ld=lld) + # Ensure that we do not link against C++ stdlib when compiling shims. + # If the compiler supports excluding the C++ stdlib implementation, use + # it. Otherwise, fall back to linking the library as if it were C, which + # has roughly the same effect. + if (NOT ${SNMALLOC_CLEANUP} STREQUAL CXX11_DESTRUCTORS) + check_linker_flag(CXX "-nostdlib++" SNMALLOC_LINKER_SUPPORT_NOSTDLIBXX) + if (SNMALLOC_LINKER_SUPPORT_NOSTDLIBXX) + target_link_options(${name} PRIVATE -nostdlib++) + else() + set_target_properties(${name} PROPERTIES LINKER_LANGUAGE C) + endif() endif() + # Remove all the duplicate new/malloc and free/delete definitions + target_link_options(${name} PRIVATE $<$:-Wl,--icf=all -fuse-ld=lld>) endif() - endmacro() + install(TARGETS ${name} EXPORT snmallocConfig) + + endfunction() if (SNMALLOC_STATIC_LIBRARY) - add_shim(snmallocshim-static STATIC src/override/malloc.cc) + add_shim(snmallocshim-static STATIC src/override/new.cc) target_compile_definitions(snmallocshim-static PRIVATE SNMALLOC_STATIC_LIBRARY_PREFIX=${SNMALLOC_STATIC_LIBRARY_PREFIX}) endif () @@ -273,7 +319,13 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) set(TESTDIR ${CMAKE_CURRENT_SOURCE_DIR}/src/test) subdirlist(TEST_CATEGORIES ${TESTDIR}) list(REVERSE TEST_CATEGORIES) + if (${SNMALLOC_CLEANUP} STREQUAL THREAD_CLEANUP) + set(TEST_CLEANUP PTHREAD_DESTRUCTORS) + else () + set(TEST_CLEANUP ${SNMALLOC_CLEANUP}) + endif() foreach(TEST_CATEGORY ${TEST_CATEGORIES}) + message(STATUS "Adding ${TEST_CATEGORY} tests") subdirlist(TESTS ${TESTDIR}/${TEST_CATEGORY}) foreach(TEST ${TESTS}) if (WIN32 @@ -294,6 +346,7 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) set(TESTNAME "${TEST_CATEGORY}-${TEST}-${FLAVOUR}") add_executable(${TESTNAME} ${SRC}) + add_warning_flags(${TESTNAME}) if (${FLAVOUR} STREQUAL "malloc") target_compile_definitions(${TESTNAME} PRIVATE SNMALLOC_PASS_THROUGH) @@ -301,31 +354,32 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) if (${FLAVOUR} STREQUAL "check") target_compile_definitions(${TESTNAME} PRIVATE SNMALLOC_CHECK_CLIENT) endif() - target_link_libraries(${TESTNAME} snmalloc_lib) + target_link_libraries(${TESTNAME} snmalloc) + target_compile_definitions(${TESTNAME} PRIVATE "SNMALLOC_USE_${TEST_CLEANUP}") if (${TEST} MATCHES "release-.*") - message(STATUS "Adding test: ${TESTNAME} only for release configs") + message(VERBOSE "Adding test: ${TESTNAME} only for release configs") add_test(NAME ${TESTNAME} COMMAND ${TESTNAME} CONFIGURATIONS "Release") else() - message(STATUS "Adding test: ${TESTNAME}") + message(VERBOSE "Adding test: ${TESTNAME}") add_test(${TESTNAME} ${TESTNAME}) endif() if (${TEST_CATEGORY} MATCHES "perf") - message(STATUS "Single threaded test: ${TESTNAME}") + message(VERBOSE "Single threaded test: ${TESTNAME}") set_tests_properties(${TESTNAME} PROPERTIES PROCESSORS 4) endif() if(WIN32) # On Windows these tests use a lot of memory as it doesn't support # lazy commit. if (${TEST} MATCHES "two_alloc_types") - message(STATUS "Single threaded test: ${TESTNAME}") + message(VERBOSE "Single threaded test: ${TESTNAME}") set_tests_properties(${TESTNAME} PROPERTIES PROCESSORS 4) endif() if (${TEST} MATCHES "fixed_region") - message(STATUS "Single threaded test: ${TESTNAME}") + message(VERBOSE "Single threaded test: ${TESTNAME}") set_tests_properties(${TESTNAME} PROPERTIES PROCESSORS 4) endif() if (${TEST} MATCHES "memory") - message(STATUS "Single threaded test: ${TESTNAME}") + message(VERBOSE "Single threaded test: ${TESTNAME}") set_tests_properties(${TESTNAME} PROPERTIES PROCESSORS 4) endif() endif() @@ -338,3 +392,23 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) clangformat_targets() endif() + +install(TARGETS snmalloc EXPORT snmallocConfig) + +install(TARGETS EXPORT snmallocConfig DESTINATION ${CMAKE_INSTALL_LIBDIR} + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/snmalloc) + +install(DIRECTORY src/aal DESTINATION include/snmalloc) +install(DIRECTORY src/ds DESTINATION include/snmalloc) +install(DIRECTORY src/override DESTINATION include/snmalloc) +install(DIRECTORY src/backend DESTINATION include/snmalloc) +install(DIRECTORY src/mem DESTINATION include/snmalloc) +install(DIRECTORY src/pal DESTINATION include/snmalloc) +install(FILES src/snmalloc.h;src/snmalloc_core.h;src/snmalloc_front.h DESTINATION include/snmalloc) + +install(EXPORT snmallocConfig + FILE snmalloc-config.cmake + NAMESPACE snmalloc:: + DESTINATION "share/snmalloc" +) + diff --git a/src/mem/threadalloc.h b/src/mem/threadalloc.h index 3c75c42ae..bf5f822a1 100644 --- a/src/mem/threadalloc.h +++ b/src/mem/threadalloc.h @@ -153,8 +153,17 @@ namespace snmalloc * Entry point that allows libc to call into the allocator for per-thread * cleanup. */ -void _malloc_thread_cleanup() +inline void _malloc_thread_cleanup() { snmalloc::ThreadAlloc::get().teardown(); } + +namespace snmalloc +{ + /** + * No-op version of register_clean_up. This is called unconditionally by + * globalconfig but is not necessary when using a libc hook. + */ + inline void register_clean_up() {} +} #endif diff --git a/src/override/new.cc b/src/override/new.cc index 96b1c94d2..c0acc953a 100644 --- a/src/override/new.cc +++ b/src/override/new.cc @@ -1,7 +1,11 @@ #include "malloc.cc" #ifdef _WIN32 -# define EXCEPTSPEC +# ifdef __clang__ +# define EXCEPTSPEC noexcept +# else +# define EXCEPTSPEC +# endif #else # ifdef _GLIBCXX_USE_NOEXCEPT # define EXCEPTSPEC _GLIBCXX_USE_NOEXCEPT diff --git a/src/pal/pal_posix.h b/src/pal/pal_posix.h index ce932fdca..b087c9298 100644 --- a/src/pal/pal_posix.h +++ b/src/pal/pal_posix.h @@ -4,8 +4,8 @@ # include #endif #include "../ds/address.h" -#if defined(BACKTRACE_HEADER) -# include BACKTRACE_HEADER +#if defined(SNMALLOC_BACKTRACE_HEADER) +# include SNMALLOC_BACKTRACE_HEADER #endif #include #include From dac1ba6a6caded330a0212e803de000800414bc7 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Sun, 29 Aug 2021 18:57:42 +0100 Subject: [PATCH 070/302] Metaslab: SlabLink field rather than parent class Introduce Metaslab::from_link(SlabLink*) to encapsulate the "container of" dance. Note that Metaslab was not a standard layout type prior to this change (since both SlabLink and Metaslab defined non-static data members), and so the reinterpret_cast<>s replaced here with ::from_link() were UB, but everyone lays out classes as one expects so it was fine in practice. Most of the uses of ::from_link() are already guarded by checks that the link pointer is not nullptr, but in src/mem/corealloc.h:/debug_is_empty_impl we shift to testing the link pointer explicitly before converting to the metaslab. Despite that Metaslab is now standard layout, we still don't fall back to the inter-convertibility of a standard layout class and its first[*] data member since we're going to want to put a common initial sequence across Metaslab and ChunkRecord and the SlabLink isn't likely to be in it. --- src/mem/corealloc.h | 13 +++++++------ src/mem/metaslab.h | 19 +++++++++++++++++-- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index c70812771..87c6aa78a 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -304,7 +304,7 @@ namespace snmalloc while (curr != nullptr) { auto nxt = curr->get_next(); - auto meta = reinterpret_cast(curr); + auto meta = Metaslab::from_link(curr); if (meta->needed() == 0) { prev->pop(); @@ -348,7 +348,7 @@ namespace snmalloc // Wake slab up. meta->set_not_sleeping(sizeclass); - alloc_classes[sizeclass].insert(meta); + alloc_classes[sizeclass].insert(&meta->link); alloc_classes[sizeclass].length++; #ifdef SNMALLOC_TRACING @@ -593,7 +593,7 @@ namespace snmalloc auto& sl = alloc_classes[sizeclass]; if (likely(!(sl.is_empty()))) { - auto meta = reinterpret_cast(sl.pop()); + auto meta = Metaslab::from_link(sl.pop()); // Drop length of sl, and empty count if it was empty. alloc_classes[sizeclass].length--; if (meta->needed() == 0) @@ -736,17 +736,18 @@ namespace snmalloc auto test = [&result](auto& queue) { if (!queue.is_empty()) { - auto curr = reinterpret_cast(queue.get_next()); + auto curr = queue.get_next(); while (curr != nullptr) { - if (curr->needed() != 0) + auto currmeta = Metaslab::from_link(curr); + if (currmeta->needed() != 0) { if (result != nullptr) *result = false; else error("debug_is_empty: found non-empty allocator"); } - curr = reinterpret_cast(curr->get_next()); + curr = currmeta->link.get_next(); } } }; diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 41f73b10f..56499ec49 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -16,10 +16,19 @@ namespace snmalloc // The Metaslab represent the status of a single slab. // This can be either a short or a standard slab. - class alignas(CACHELINE_SIZE) Metaslab : public SlabLink + class alignas(CACHELINE_SIZE) Metaslab { public: - constexpr Metaslab() : SlabLink(true) {} + // TODO: Annotate with CHERI subobject unbound for pointer arithmetic + SlabLink link; + + constexpr Metaslab() : link(true) {} + + /** + * Metaslab::link points at another link field. To get the actual Metaslab, + * use this encapsulation of the container-of logic. + */ + static Metaslab* from_link(SlabLink* ptr); /** * Data-structure for building the free list for this slab. @@ -160,6 +169,12 @@ namespace snmalloc } }; + Metaslab* Metaslab::from_link(SlabLink* lptr) + { + return pointer_offset_signed( + lptr, -static_cast(offsetof(Metaslab, link))); + } + struct RemoteAllocator; /** From fd185282788f66bbc621a24c1b2e0c5f53eac77f Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Tue, 7 Sep 2021 08:44:47 +0100 Subject: [PATCH 071/302] Add missing `inline` from header. (#388) --- .github/workflows/main.yml | 8 ++++++-- src/mem/metaslab.h | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 157ea9e10..bed0e701f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,9 +28,13 @@ jobs: - os: "ubuntu-latest" dependencies: "sudo apt install ninja-build" - os: "macos-11" - dependencies: "brew update && brew install ninja" + # The homebrew packages are broken at the moment and error out + # after trying to install Python as a dependency of ninja because + # 2to3 exists. As a quick hack, delete it first. This should be + # removed once the homebrew install is fixed. + dependencies: "rm -f /usr/local/bin/2to3 ; brew update && brew install ninja" - os: "macos-10.15" - dependencies: "brew update && brew install ninja" + dependencies: "rm -f /usr/local/bin/2to3 ; brew update && brew install ninja" # Skip the tests for the FreeBSD release builds - os: "freebsd-13.0" build-type: Release diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 56499ec49..6212faa8f 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -169,7 +169,7 @@ namespace snmalloc } }; - Metaslab* Metaslab::from_link(SlabLink* lptr) + inline Metaslab* Metaslab::from_link(SlabLink* lptr) { return pointer_offset_signed( lptr, -static_cast(offsetof(Metaslab, link))); From 6c5626fe5f07b89eb6afc47d0a3abce517c8fbe1 Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Mon, 6 Sep 2021 16:28:03 +0100 Subject: [PATCH 072/302] Install test headers. Verona uses these. --- CMakeLists.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 79950cf66..8d0d613b9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -404,6 +404,14 @@ install(DIRECTORY src/override DESTINATION include/snmalloc) install(DIRECTORY src/backend DESTINATION include/snmalloc) install(DIRECTORY src/mem DESTINATION include/snmalloc) install(DIRECTORY src/pal DESTINATION include/snmalloc) +install(FILES + src/test/measuretime.h + src/test/opt.h + src/test/setup.h + src/test/usage.h + src/test/xoroshiro.h + DESTINATION include/snmalloc/test + ) install(FILES src/snmalloc.h;src/snmalloc_core.h;src/snmalloc_front.h DESTINATION include/snmalloc) install(EXPORT snmallocConfig From d524ef5cac2ca0efbdb2772f529e6e36142e1476 Mon Sep 17 00:00:00 2001 From: Istvan Haller <31476121+ihaller@users.noreply.github.com> Date: Tue, 14 Sep 2021 11:49:32 +0100 Subject: [PATCH 073/302] Extracted the core elements of the BackendAllocator into a common helper (#390) * Extracted backend common functionality --- src/backend/backend.h | 348 ++++++++++++++++++++++++------------------ 1 file changed, 200 insertions(+), 148 deletions(-) diff --git a/src/backend/backend.h b/src/backend/backend.h index 9a3964584..8f36b7e2f 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -9,14 +9,15 @@ namespace snmalloc { /** - * This class implements the standard backend for handling allocations. - * It abstracts page table management and address space management. + * This helper class implements the core functionality to allocate from an + * address space and pagemap. Any backend implementation can use this class to + * help with basic address space managment. */ template< SNMALLOC_CONCEPT(ConceptPAL) PAL, - bool fixed_range, - typename PageMapEntry = MetaEntry> - class BackendAllocator : public CommonConfig + typename LocalState, + SNMALLOC_CONCEPT(ConceptBackendMetaRange) Pagemap> + class AddressSpaceAllocatorCommon { // Size of local address space requests. Currently aimed at 2MiB large // pages but should make this configurable (i.e. for OE, so we don't need as @@ -32,138 +33,73 @@ namespace snmalloc #endif public: - using Pal = PAL; - /** - * Local state for the backend allocator. + * Provide a block of meta-data with size and align. * - * This class contains thread local structures to make the implementation - * of the backend allocator more efficient. + * Backend allocator may use guard pages and separate area of + * address space to protect this from corruption. */ - class LocalState + static CapPtr alloc_meta_data( + AddressSpaceManager& global, LocalState* local_state, size_t size) { - friend BackendAllocator; - - AddressSpaceManagerCore local_address_space; - -#ifdef SNMALLOC_CHECK_CLIENT - /** - * Secondary local address space, so we can apply some randomisation - * and guard pages to protect the meta-data. - */ - AddressSpaceManagerCore local_meta_address_space; -#endif - }; - - SNMALLOC_REQUIRE_CONSTINIT - static inline AddressSpaceManager address_space; + return reserve(global, local_state, size); + } - class Pagemap + /** + * Returns a chunk of memory with alignment and size of `size`, and a + * metaslab block. + * + * It additionally set the meta-data for this chunk of memory to + * be + * (remote, sizeclass, metaslab) + * where metaslab, is the second element of the pair return. + */ + static std::pair, Metaslab*> alloc_chunk( + AddressSpaceManager& global, + LocalState* local_state, + size_t size, + RemoteAllocator* remote, + sizeclass_t sizeclass) { - friend class BackendAllocator; + SNMALLOC_ASSERT(bits::is_pow2(size)); + SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE); - SNMALLOC_REQUIRE_CONSTINIT - static inline FlatPagemap - concretePagemap; + auto meta = reinterpret_cast( + reserve(global, local_state, sizeof(Metaslab)).unsafe_ptr()); - public: - /** - * Provide a type alias for LocalState so that we can refer to it without - * needing the whole BackendAllocator type at hand. - */ - using LocalState = BackendAllocator::LocalState; + if (meta == nullptr) + return {nullptr, nullptr}; - /** - * Get the metadata associated with a chunk. - * - * Set template parameter to true if it not an error - * to access a location that is not backed by a chunk. - */ - template - SNMALLOC_FAST_PATH static const MetaEntry& - get_metaentry(LocalState* ls, address_t p) - { - UNUSED(ls); - return concretePagemap.template get(p); - } + CapPtr p = reserve(global, local_state, size); - /** - * Set the metadata associated with a chunk. - */ - SNMALLOC_FAST_PATH - static void - set_metaentry(LocalState* ls, address_t p, size_t size, MetaEntry t) - { - UNUSED(ls); - for (address_t a = p; a < p + size; a += MIN_CHUNK_SIZE) - { - concretePagemap.set(a, t); - } - } - - static void register_range(LocalState* ls, address_t p, size_t sz) +#ifdef SNMALLOC_TRACING + std::cout << "Alloc chunk: " << p.unsafe_ptr() << " (" << size << ")" + << std::endl; +#endif + if (p == nullptr) { - UNUSED(ls); - concretePagemap.register_range(p, sz); + // TODO: This is leaking `meta`. Currently there is no facility for + // meta-data reuse, so will leave until we develop more expressive + // meta-data management. +#ifdef SNMALLOC_TRACING + std::cout << "Out of memory" << std::endl; +#endif + return {p, nullptr}; } - }; - - public: - template - static std::enable_if_t init() - { - static_assert(fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); - - Pagemap::concretePagemap.init(); - } - - template - static std::enable_if_t - init(LocalState* local_state, void* base, size_t length) - { - static_assert(fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); - auto [heap_base, heap_length] = - Pagemap::concretePagemap.init(base, length); - address_space.template add_range( - local_state, CapPtr(heap_base), heap_length); + MetaEntry t(meta, remote, sizeclass); + Pagemap::set_metaentry(local_state, address_cast(p), size, t); + return {p, meta}; } private: -#ifdef SNMALLOC_CHECK_CLIENT - /** - * Returns a sub-range of [return, return+sub_size] that is contained in - * the range [base, base+full_size]. The first and last slot are not used - * so that the edges can be used for guard pages. - */ - static CapPtr - sub_range(CapPtr base, size_t full_size, size_t sub_size) - { - SNMALLOC_ASSERT(bits::is_pow2(full_size)); - SNMALLOC_ASSERT(bits::is_pow2(sub_size)); - SNMALLOC_ASSERT(full_size % sub_size == 0); - SNMALLOC_ASSERT(full_size / sub_size >= 4); - - size_t offset_mask = full_size - sub_size; - - // Don't use first or last block in the larger reservation - // Loop required to get uniform distribution. - size_t offset; - do - { - offset = get_entropy64() & offset_mask; - } while ((offset == 0) || (offset == offset_mask)); - - return pointer_offset(base, offset); - } -#endif - /** * Internal method for acquiring state from the local and global address * space managers. */ template - static CapPtr reserve(LocalState* local_state, size_t size) + static CapPtr reserve( + AddressSpaceManager& global, LocalState* local_state, size_t size) { #ifdef SNMALLOC_CHECK_CLIENT constexpr auto MAX_CACHED_SIZE = @@ -172,8 +108,6 @@ namespace snmalloc constexpr auto MAX_CACHED_SIZE = LOCAL_CACHE_BLOCK; #endif - auto& global = address_space; - CapPtr p; if ((local_state != nullptr) && (size <= MAX_CACHED_SIZE)) { @@ -242,7 +176,151 @@ namespace snmalloc return p; } +#ifdef SNMALLOC_CHECK_CLIENT + /** + * Returns a sub-range of [return, return+sub_size] that is contained in + * the range [base, base+full_size]. The first and last slot are not used + * so that the edges can be used for guard pages. + */ + static CapPtr + sub_range(CapPtr base, size_t full_size, size_t sub_size) + { + SNMALLOC_ASSERT(bits::is_pow2(full_size)); + SNMALLOC_ASSERT(bits::is_pow2(sub_size)); + SNMALLOC_ASSERT(full_size % sub_size == 0); + SNMALLOC_ASSERT(full_size / sub_size >= 4); + + size_t offset_mask = full_size - sub_size; + + // Don't use first or last block in the larger reservation + // Loop required to get uniform distribution. + size_t offset; + do + { + offset = get_entropy64() & offset_mask; + } while ((offset == 0) || (offset == offset_mask)); + + return pointer_offset(base, offset); + } +#endif + }; + + /** + * This class implements the standard backend for handling allocations. + * It abstracts page table management and address space management. + */ + template< + SNMALLOC_CONCEPT(ConceptPAL) PAL, + bool fixed_range, + typename PageMapEntry = MetaEntry> + class BackendAllocator : public CommonConfig + { + public: + using Pal = PAL; + + /** + * Local state for the backend allocator. + * + * This class contains thread local structures to make the implementation + * of the backend allocator more efficient. + */ + class LocalState + { + template< + SNMALLOC_CONCEPT(ConceptPAL) PAL2, + typename LocalState, + SNMALLOC_CONCEPT(ConceptBackendMetaRange) Pagemap> + friend class AddressSpaceAllocatorCommon; + + AddressSpaceManagerCore local_address_space; + +#ifdef SNMALLOC_CHECK_CLIENT + /** + * Secondary local address space, so we can apply some randomisation + * and guard pages to protect the meta-data. + */ + AddressSpaceManagerCore local_meta_address_space; +#endif + }; + + SNMALLOC_REQUIRE_CONSTINIT + static inline AddressSpaceManager address_space; + + class Pagemap + { + friend class BackendAllocator; + + SNMALLOC_REQUIRE_CONSTINIT + static inline FlatPagemap + concretePagemap; + + public: + /** + * Provide a type alias for LocalState so that we can refer to it without + * needing the whole BackendAllocator type at hand. + */ + using LocalState = BackendAllocator::LocalState; + + /** + * Get the metadata associated with a chunk. + * + * Set template parameter to true if it not an error + * to access a location that is not backed by a chunk. + */ + template + SNMALLOC_FAST_PATH static const MetaEntry& + get_metaentry(LocalState* ls, address_t p) + { + UNUSED(ls); + return concretePagemap.template get(p); + } + + /** + * Set the metadata associated with a chunk. + */ + SNMALLOC_FAST_PATH + static void + set_metaentry(LocalState* ls, address_t p, size_t size, MetaEntry t) + { + UNUSED(ls); + for (address_t a = p; a < p + size; a += MIN_CHUNK_SIZE) + { + concretePagemap.set(a, t); + } + } + + static void register_range(LocalState* ls, address_t p, size_t sz) + { + UNUSED(ls); + concretePagemap.register_range(p, sz); + } + }; + + private: + using AddressSpaceAllocator = + AddressSpaceAllocatorCommon; + public: + template + static std::enable_if_t init() + { + static_assert(fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); + + Pagemap::concretePagemap.init(); + } + + template + static std::enable_if_t + init(LocalState* local_state, void* base, size_t length) + { + static_assert(fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); + + auto [heap_base, heap_length] = + Pagemap::concretePagemap.init(base, length); + address_space.template add_range( + local_state, CapPtr(heap_base), heap_length); + } + /** * Provide a block of meta-data with size and align. * @@ -257,7 +335,8 @@ namespace snmalloc static CapPtr alloc_meta_data(LocalState* local_state, size_t size) { - return reserve(local_state, size); + return AddressSpaceAllocator::alloc_meta_data( + address_space, local_state, size); } /** @@ -275,35 +354,8 @@ namespace snmalloc RemoteAllocator* remote, sizeclass_t sizeclass) { - SNMALLOC_ASSERT(bits::is_pow2(size)); - SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE); - - auto meta = reinterpret_cast( - reserve(local_state, sizeof(Metaslab)).unsafe_ptr()); - - if (meta == nullptr) - return {nullptr, nullptr}; - - CapPtr p = reserve(local_state, size); - -#ifdef SNMALLOC_TRACING - std::cout << "Alloc chunk: " << p.unsafe_ptr() << " (" << size << ")" - << std::endl; -#endif - if (p == nullptr) - { - // TODO: This is leaking `meta`. Currently there is no facility for - // meta-data reuse, so will leave until we develop more expressive - // meta-data management. -#ifdef SNMALLOC_TRACING - std::cout << "Out of memory" << std::endl; -#endif - return {p, nullptr}; - } - - MetaEntry t(meta, remote, sizeclass); - Pagemap::set_metaentry(local_state, address_cast(p), size, t); - return {p, meta}; + return AddressSpaceAllocator::alloc_chunk( + address_space, local_state, size, remote, sizeclass); } }; } // namespace snmalloc From 51e75bca89cfd85de430c99c5ef9110fdc992656 Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Tue, 10 Aug 2021 11:36:53 +0100 Subject: [PATCH 074/302] Add memcpy with bounds checks. The memcpy implementation is not completely stupid but is almost certainly not as good as a carefully tuned and optimised one. Building snmalloc with FreeBSD's libc memcpy + jemalloc and with this, each 10 times, does not show a statistically significant performance difference at 95% confidence. The snmalloc version has very slightly lower median and worst-case times. This is in no way a sensible benchmark, but it serves as a smoke test for significant performance regressions. The CI self-host job now uses the checked memcpy. This also fixes an off-by-one error in the external bounds. This is triggered by ninja, so we will see breakage in CI if it is reintroduced. In debug builds, we provide a verbose error containing the address of the allocation, the base and bounds of the allocation, and a backtrace. The backtrace was broken by the CI cleanup moving the BACKTRACE_HEADER macro into the SNMALLOC_ namespace. This is also fixed. The test involves hijacking `abort`, which doesn't work everywhere. It also requires `backtrace` to work in configurations where stack traces are enabled. This is disabled in QEMU because `backtrace` appears to crash reliably in QEMU user mode. For now, in the -checks build configurations, we are hitting a slow path in the pagemap on accesses so that the pages that are `PROT_NONE` don't cause crashes. These need to be made read-only, but this requires a PAL change. --- .github/workflows/main.yml | 16 +- CMakeLists.txt | 17 +- src/backend/commonconfig.h | 22 +++ src/backend/pagemap.h | 9 +- src/ds/address.h | 4 +- src/ds/defines.h | 2 + src/mem/localalloc.h | 27 +--- src/override/malloc.cc | 27 +--- src/override/memcpy.cc | 232 ++++++++++++++++++++++++++++ src/override/override.h | 28 ++++ src/pal/pal_posix.h | 2 +- src/test/func/memcpy/func-memcpy.cc | 155 +++++++++++++++++++ 12 files changed, 482 insertions(+), 59 deletions(-) create mode 100644 src/override/memcpy.cc create mode 100644 src/override/override.h create mode 100644 src/test/func/memcpy/func-memcpy.cc diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bed0e701f..8383e8d91 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,6 +18,13 @@ jobs: # Build each combination of OS and release/debug variants os: [ "ubuntu-latest", "ubuntu-18.04", "macos-11", "macos-10.15", "freebsd-12.2", "freebsd-13.0" ] build-type: [ Release, Debug ] + # Extra cmake flags. GitHub Actions matrix overloads `include` to mean + # 'add extra things to a job' and 'add jobs'. You can add extra things + # to a job by specifying things that exist in a job created from the + # matrix definition and adding things. You can specify extra jobs by + # specifying properties that don't match existing jobs. We use + # `cmake-flags` to add cmake flags to all jobs matching a pattern and + # `extra-cmake-flags` to specify a new job with custom CMake flags. extra-cmake-flags: [ "" ] # Modify the complete matrix include: @@ -36,16 +43,23 @@ jobs: - os: "macos-10.15" dependencies: "rm -f /usr/local/bin/2to3 ; brew update && brew install ninja" # Skip the tests for the FreeBSD release builds + # Also build-test the checked memcpy implementation while doing these. + # It is run-tested on Linux and should be the same everywhere. - os: "freebsd-13.0" build-type: Release build-only: yes + cmake-flags: "-DSNMALLOC_MEMCPY_BOUNDS=ON -DSNMALLOC_CHECK_LOADS=ON" - os: "freebsd-12.2" build-type: Debug build-only: yes - # Add the self-host build + cmake-flags: "-DSNMALLOC_MEMCPY_BOUNDS=ON -DSNMALLOC_CHECK_LOADS=ON" + # Add the self-host build, using the bounds-checked memcpy in + # maximally paranoid mode (checking loads and stores) - os: "ubuntu-latest" build-type: Debug self-host: true + extra-cmake-flags: "-DSNMALLOC_MEMCPY_BOUNDS=ON -DSNMALLOC_CHECK_LOADS=ON" + dependencies: "sudo apt install ninja-build" # Extra build to check using pthread library for destructing local state. - os: "ubuntu-latest" variant: "Ubuntu (with pthread destructors)." diff --git a/CMakeLists.txt b/CMakeLists.txt index 8d0d613b9..534331e7c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,6 +19,8 @@ option(SNMALLOC_USE_CXX17 "Build as C++17 for legacy support." OFF) # Options that apply only if we're not building the header-only library cmake_dependent_option(SNMALLOC_RUST_SUPPORT "Build static library for rust" OFF "NOT SNMALLOC_HEADER_ONLY_LIBRARY" OFF) cmake_dependent_option(SNMALLOC_STATIC_LIBRARY "Build static libraries" ON "NOT SNMALLOC_HEADER_ONLY_LIBRARY" OFF) +cmake_dependent_option(SNMALLOC_MEMCPY_BOUNDS "Perform bounds checks on memcpy with heap objects" OFF "NOT SNMALLOC_HEADER_ONLY_LIBRARY" OFF) +cmake_dependent_option(SNMALLOC_CHECK_LOADS "Perform bounds checks on the source argument to memcpy with heap objects" OFF "SNMALLOC_MEMCPY_BOUNDS" OFF) cmake_dependent_option(SNMALLOC_OPTIMISE_FOR_CURRENT_MACHINE "Compile for current machine architecture" Off "NOT SNMALLOC_HEADER_ONLY_LIBRARY" OFF) if (NOT SNMALLOC_HEADER_ONLY_LIBRARY) # Pick a sensible default for the thread cleanup mechanism @@ -291,20 +293,27 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) target_link_options(${name} PRIVATE $<$:-Wl,--icf=all -fuse-ld=lld>) endif() + target_compile_definitions(${name} PRIVATE + SNMALLOC_CHECK_LOADS=$,true,false>) + install(TARGETS ${name} EXPORT snmallocConfig) endfunction() + set(SHIM_FILES src/override/new.cc) + if (SNMALLOC_MEMCPY_BOUNDS) + list(APPEND SHIM_FILES src/override/memcpy.cc) + endif () + if (SNMALLOC_STATIC_LIBRARY) - add_shim(snmallocshim-static STATIC src/override/new.cc) + add_shim(snmallocshim-static STATIC ${SHIM_FILES}) target_compile_definitions(snmallocshim-static PRIVATE SNMALLOC_STATIC_LIBRARY_PREFIX=${SNMALLOC_STATIC_LIBRARY_PREFIX}) endif () if(NOT WIN32) - set(SHARED_FILES src/override/new.cc) - add_shim(snmallocshim SHARED ${SHARED_FILES}) - add_shim(snmallocshim-checks SHARED ${SHARED_FILES}) + add_shim(snmallocshim SHARED ${SHIM_FILES}) + add_shim(snmallocshim-checks SHARED ${SHIM_FILES}) target_compile_definitions(snmallocshim-checks PRIVATE SNMALLOC_CHECK_CLIENT) endif() diff --git a/src/backend/commonconfig.h b/src/backend/commonconfig.h index 689a13946..2e118799b 100644 --- a/src/backend/commonconfig.h +++ b/src/backend/commonconfig.h @@ -115,4 +115,26 @@ namespace snmalloc &unused_remote != fake_large_remote, "Compilation should ensure these are different"); }; + + /** + * SFINAE helper. Matched only if `T` implements `is_initialised`. Calls + * it if it exists. + */ + template + SNMALLOC_FAST_PATH auto call_is_initialised(T*, int) + -> decltype(T::is_initialised()) + { + return T::is_initialised(); + } + + /** + * SFINAE helper. Matched only if `T` does not implement `is_initialised`. + * Unconditionally returns true if invoked. + */ + template + SNMALLOC_FAST_PATH auto call_is_initialised(T*, long) + { + return true; + } + } // namespace snmalloc diff --git a/src/backend/pagemap.h b/src/backend/pagemap.h index eae470306..7a794c12f 100644 --- a/src/backend/pagemap.h +++ b/src/backend/pagemap.h @@ -244,7 +244,14 @@ namespace snmalloc } // This means external pointer on Windows will be slow. - if constexpr (potentially_out_of_range && !pal_supports) + // TODO: With SNMALLOC_CHECK_CLIENT we should map that region read-only, + // not no-access, but this requires a PAL change. + if constexpr ( + potentially_out_of_range +#ifndef SNMALLOC_CHECK_CLIENT + && !pal_supports +#endif + ) { register_range(p, 1); } diff --git a/src/ds/address.h b/src/ds/address.h index bfa1ac534..81739ac45 100644 --- a/src/ds/address.h +++ b/src/ds/address.h @@ -231,11 +231,11 @@ namespace snmalloc * expected to point to the base of some (sub)allocation into which cursor * points; would-be negative answers trip an assertion in debug builds. */ - inline size_t pointer_diff(void* base, void* cursor) + inline size_t pointer_diff(const void* base, const void* cursor) { SNMALLOC_ASSERT(cursor >= base); return static_cast( - static_cast(cursor) - static_cast(base)); + static_cast(cursor) - static_cast(base)); } template< diff --git a/src/ds/defines.h b/src/ds/defines.h index d8c495556..b0aab14c6 100644 --- a/src/ds/defines.h +++ b/src/ds/defines.h @@ -15,6 +15,7 @@ # define SNMALLOC_PURE # define SNMALLOC_COLD # define SNMALLOC_REQUIRE_CONSTINIT +# define SNMALLOC_UNUSED_FUNCTION #else # define likely(x) __builtin_expect(!!(x), 1) # define unlikely(x) __builtin_expect(!!(x), 0) @@ -25,6 +26,7 @@ # define SNMALLOC_FAST_PATH_LAMBDA SNMALLOC_FAST_PATH # define SNMALLOC_PURE __attribute__((const)) # define SNMALLOC_COLD __attribute__((cold)) +# define SNMALLOC_UNUSED_FUNCTION __attribute((unused)) # ifdef __clang__ # define SNMALLOC_REQUIRE_CONSTINIT \ [[clang::require_constant_initialization]] diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 36f4544b9..761754ee5 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -287,27 +287,6 @@ namespace snmalloc return local_cache.remote_allocator; } - /** - * SFINAE helper. Matched only if `T` implements `is_initialised`. Calls - * it if it exists. - */ - template - SNMALLOC_FAST_PATH auto call_is_initialised(T*, int) - -> decltype(T::is_initialised()) - { - return T::is_initialised(); - } - - /** - * SFINAE helper. Matched only if `T` does not implement `is_initialised`. - * Unconditionally returns true if invoked. - */ - template - SNMALLOC_FAST_PATH auto call_is_initialised(T*, long) - { - return true; - } - /** * Call `SharedStateHandle::is_initialised()` if it is implemented, * unconditionally returns true otherwise. @@ -639,9 +618,9 @@ namespace snmalloc if constexpr (location == Start) return start; else if constexpr (location == End) - return pointer_offset(start, rsize); - else return pointer_offset(start, rsize - 1); + else + return pointer_offset(start, rsize); } #else UNUSED(p_raw); @@ -649,7 +628,7 @@ namespace snmalloc if constexpr ((location == End) || (location == OnePastEnd)) // We don't know the End, so return MAX_PTR - return pointer_offset(nullptr, UINTPTR_MAX); + return reinterpret_cast(UINTPTR_MAX); else // We don't know the Start, so return MIN_PTR return nullptr; diff --git a/src/override/malloc.cc b/src/override/malloc.cc index 0a9ad3630..6db61ae8e 100644 --- a/src/override/malloc.cc +++ b/src/override/malloc.cc @@ -1,35 +1,10 @@ -// Core implementation of snmalloc independent of the configuration mode -#include "../snmalloc_core.h" - -#ifndef SNMALLOC_PROVIDE_OWN_CONFIG -# include "../backend/globalconfig.h" -// The default configuration for snmalloc is used if alternative not defined -namespace snmalloc -{ - using Alloc = snmalloc::LocalAllocator; -} // namespace snmalloc -#endif - -// User facing API surface, needs to know what `Alloc` is. -#include "../snmalloc_front.h" +#include "override.h" #include #include using namespace snmalloc; -#ifndef SNMALLOC_EXPORT -# define SNMALLOC_EXPORT -#endif -#ifdef SNMALLOC_STATIC_LIBRARY_PREFIX -# define __SN_CONCAT(a, b) a##b -# define __SN_EVALUATE(a, b) __SN_CONCAT(a, b) -# define SNMALLOC_NAME_MANGLE(a) \ - __SN_EVALUATE(SNMALLOC_STATIC_LIBRARY_PREFIX, a) -#elif !defined(SNMALLOC_NAME_MANGLE) -# define SNMALLOC_NAME_MANGLE(a) a -#endif - #ifndef MALLOC_USABLE_SIZE_QUALIFIER # define MALLOC_USABLE_SIZE_QUALIFIER #endif diff --git a/src/override/memcpy.cc b/src/override/memcpy.cc new file mode 100644 index 000000000..4d41bf35a --- /dev/null +++ b/src/override/memcpy.cc @@ -0,0 +1,232 @@ +#include "override.h" + +#include +#include +#include +#if __has_include() +# include +#endif + +using namespace snmalloc; + +// glibc lacks snprintf_l +#ifdef __linux__ +# define snprintf_l(buf, size, loc, msg, ...) \ + snprintf(buf, size, msg, __VA_ARGS__) +// Windows has it with an underscore prefix +#elif defined(_MSC_VER) +# define snprintf_l(buf, size, loc, msg, ...) \ + _snprintf_l(buf, size, msg, loc, __VA_ARGS__) +#endif + +namespace +{ + /** + * Should we check loads? This defaults to on in debug builds, off in + * release (store-only checks) + */ + static constexpr bool CheckReads = +#ifdef SNMALLOC_CHECK_LOADS + SNMALLOC_CHECK_LOADS +#else +# ifdef NDEBUG + false +# else + true +# endif +#endif + ; + + /** + * Should we fail fast when we encounter an error? With this set to true, we + * just issue a trap instruction and crash the process once we detect an + * error. With it set to false we print a helpful error message and then crash + * the process. The process may be in an undefined state by the time the + * check fails, so there are potentially security implications to turning this + * off. It defaults to true for debug builds, false for release builds. + */ + static constexpr bool FailFast = +#ifdef SNMALLOC_FAIL_FAST + SNMALLOC_FAIL_FAST +#else +# ifdef NDEBUG + true +# else + false +# endif +#endif + ; + + /** + * The largest register size that we can use for loads and stores. These + * types are expected to work for overlapping copies: we can always load them + * into a register and store them. Note that this is at the C abstract + * machine level: the compiler may spill temporaries to the stack, just not + * to the source or destination object. + */ + static constexpr size_t LargestRegisterSize = +#ifdef __AVX__ + 32 +#elif defined(__SSE__) + 16 +#else + sizeof(uint64_t) +#endif + ; + + /** + * Copy a single element of a specified size. Uses a compiler builtin that + * expands to a single load and store. + */ + template + SNMALLOC_FAST_PATH inline void copy_one(void* dst, const void* src) + { +#if __has_builtin(__builtin_memcpy_inline) + __builtin_memcpy_inline(dst, src, Size); +#else + // Define a structure of size `Size` that has alignment 1 and a default + // copy-assignment operator. We can then copy the data as this type. The + // compiler knows the exact width and so will generate the correct wide + // instruction for us (clang 10 and gcc 12 both generate movups for the + // 16-byte version of this when targeting SSE. + struct Block + { + char data[Size]; + }; + auto* d = static_cast(dst); + auto* s = static_cast(src); + *d = *s; +#endif + } + + SNMALLOC_SLOW_PATH SNMALLOC_UNUSED_FUNCTION void crashWithMessage + [[noreturn]] ( + void* p, size_t len, const char* msg, decltype(ThreadAlloc::get())& alloc) + { + // We're going to crash the program now, but try to avoid heap + // allocations if possible, since the heap may be in an undefined + // state. + std::array buffer; + snprintf_l( + buffer.data(), + buffer.size(), + /* Force C locale */ nullptr, + "%s: %p is in allocation %p--%p, offset 0x%zx is past the end.\n", + msg, + p, + alloc.template external_pointer(p), + alloc.template external_pointer(p), + len); + Pal::error(buffer.data()); + } + + /** + * Check whether a pointer + length is in the same object as the pointer. + * Fail with the error message from the third argument if not. + * + * The template parameter indicates whether this is a read. If so, this + * function is a no-op when `CheckReads` is false. + */ + template + SNMALLOC_FAST_PATH inline void + check_bounds(const void* ptr, size_t len, const char* msg = "") + { + if constexpr (!IsRead || CheckReads) + { + auto& alloc = ThreadAlloc::get(); + void* p = const_cast(ptr); + + if (unlikely( + pointer_diff(ptr, alloc.external_pointer(p)) < len)) + { + if constexpr (FailFast) + { + UNUSED(ptr); + UNUSED(len); + UNUSED(msg); + __builtin_trap(); + } + else + { + crashWithMessage(p, len, msg, alloc); + } + } + } + else + { + UNUSED(ptr); + UNUSED(len); + UNUSED(msg); + } + } + + /** + * Copy a block using the specified size. This copies as many complete + * chunks of size `Size` as are possible from `len`. + */ + template + SNMALLOC_FAST_PATH inline void + block_copy(void* dst, const void* src, size_t len) + { + for (size_t i = 0; (i + Size) <= len; i += Size) + { + copy_one(pointer_offset(dst, i), pointer_offset(src, i)); + } + } + + /** + * Perform an overlapping copy of the end. This will copy one (potentially + * unaligned) `T` from the end of the source to the end of the destination. + * This may overlap other bits of the copy. + */ + template + SNMALLOC_FAST_PATH inline void + copy_end(void* dst, const void* src, size_t len) + { + copy_one( + pointer_offset(dst, len - Size), pointer_offset(src, len - Size)); + } + + /** + * Predicate indicating whether the source and destination are sufficiently + * aligned to be copied as aligned chunks of `Size` bytes. + */ + template + SNMALLOC_FAST_PATH bool is_aligned_memcpy(void* dst, const void* src) + { + return (pointer_align_down(const_cast(src)) == src) && + (pointer_align_down(dst) == dst); + } +} + +extern "C" +{ + /** + * Snmalloc checked memcpy. + */ + SNMALLOC_EXPORT void* + SNMALLOC_NAME_MANGLE(memcpy)(void* dst, const void* src, size_t len) + { + // 0 is a very common size for memcpy and we don't need to do external + // pointer checks if we hit it. It's also the fastest case, to encourage + // the compiler to favour the other cases. + if (unlikely(len == 0)) + { + return dst; + } + // Check the bounds of the arguments. + check_bounds( + dst, len, "memcpy with destination out of bounds of heap allocation"); + check_bounds( + src, len, "memcpy with source out of bounds of heap allocation"); + // If this is a small size, do byte-by-byte copies. + if (len < LargestRegisterSize) + { + block_copy<1>(dst, src, len); + return dst; + } + block_copy(dst, src, len); + copy_end(dst, src, len); + return dst; + } +} diff --git a/src/override/override.h b/src/override/override.h new file mode 100644 index 000000000..a9aedbe9d --- /dev/null +++ b/src/override/override.h @@ -0,0 +1,28 @@ +#pragma once + +// Core implementation of snmalloc independent of the configuration mode +#include "../snmalloc_core.h" + +#ifndef SNMALLOC_PROVIDE_OWN_CONFIG +# include "../backend/globalconfig.h" +// The default configuration for snmalloc is used if alternative not defined +namespace snmalloc +{ + using Alloc = snmalloc::LocalAllocator; +} // namespace snmalloc +#endif + +// User facing API surface, needs to know what `Alloc` is. +#include "../snmalloc_front.h" + +#ifndef SNMALLOC_EXPORT +# define SNMALLOC_EXPORT +#endif +#ifdef SNMALLOC_STATIC_LIBRARY_PREFIX +# define __SN_CONCAT(a, b) a##b +# define __SN_EVALUATE(a, b) __SN_CONCAT(a, b) +# define SNMALLOC_NAME_MANGLE(a) \ + __SN_EVALUATE(SNMALLOC_STATIC_LIBRARY_PREFIX, a) +#elif !defined(SNMALLOC_NAME_MANGLE) +# define SNMALLOC_NAME_MANGLE(a) a +#endif diff --git a/src/pal/pal_posix.h b/src/pal/pal_posix.h index b087c9298..93b951854 100644 --- a/src/pal/pal_posix.h +++ b/src/pal/pal_posix.h @@ -134,7 +134,7 @@ namespace snmalloc static void print_stack_trace() { -#ifdef BACKTRACE_HEADER +#ifdef SNMALLOC_BACKTRACE_HEADER constexpr int SIZE = 1024; void* buffer[SIZE]; auto nptrs = backtrace(buffer, SIZE); diff --git a/src/test/func/memcpy/func-memcpy.cc b/src/test/func/memcpy/func-memcpy.cc new file mode 100644 index 000000000..1727d3825 --- /dev/null +++ b/src/test/func/memcpy/func-memcpy.cc @@ -0,0 +1,155 @@ +// Windows doesn't like changing the linkage spec of abort. +#if defined(_MSC_VER) +int main() +{ + return 0; +} +#else +// QEMU user mode does not support the code that generates backtraces and so we +// also need to skip this test if we are doing a debug build and targeting +// QEMU. +# if defined(SNMALLOC_QEMU_WORKAROUND) && defined(SNMALLOC_BACKTRACE_HEADER) +# undef SNMALLOC_BACKTRACE_HEADER +# endif +# ifdef SNMALLOC_STATIC_LIBRARY_PREFIX +# undef SNMALLOC_STATIC_LIBRARY_PREFIX +# endif +# ifdef SNMALLOC_FAIL_FAST +# undef SNMALLOC_FAIL_FAST +# endif +# define SNMALLOC_FAIL_FAST false +# define SNMALLOC_STATIC_LIBRARY_PREFIX my_ +# include "ds/defines.h" +# ifndef SNMALLOC_PASS_THROUGH +# include "override/malloc.cc" +# else +# define my_malloc(x) malloc(x) +# define my_free(x) free(x) +# endif +# include "override/memcpy.cc" + +# include +# include +# include +# include +# include + +/** + * Jump buffer used to jump out of `abort()` for recoverable errors. + */ +static std::jmp_buf jmp; + +/** + * Flag indicating whether `jmp` is valid. If this is set then calls to + * `abort` will jump to the jump buffer, rather than exiting. + */ +static bool can_longjmp; + +/** + * Replacement for the C standard `abort` that returns to the `setjmp` call for + * recoverable errors. + */ +extern "C" void abort() +{ + if (can_longjmp) + { + longjmp(jmp, 1); + } + exit(-1); +} + +/** + * Check that memcpy works in correct use. This allocates a pair of buffers, + * fills one with a well-known pattern, and then copies subsets of this at + * one-byte increments to a target. This gives us unaligned starts. + */ +void check_size(size_t size) +{ + auto* s = reinterpret_cast(my_malloc(size + 1)); + auto* d = reinterpret_cast(my_malloc(size + 1)); + d[size] = 0; + s[size] = 255; + for (size_t start = 0; start < size; start++) + { + unsigned char* src = s + start; + unsigned char* dst = d + start; + size_t sz = (size - start); + for (size_t i = 0; i < sz; ++i) + { + src[i] = static_cast(i); + } + for (size_t i = 0; i < sz; ++i) + { + dst[i] = 0; + } + my_memcpy(dst, src, sz); + for (size_t i = 0; i < sz; ++i) + { + if (dst[i] != static_cast(i)) + { + fprintf( + stderr, + "Testing size %zd %hhx == %hhx\n", + sz, + static_cast(i), + dst[i]); + } + SNMALLOC_CHECK(dst[i] == (unsigned char)i); + } + SNMALLOC_CHECK(d[size] == 0); + } + my_free(s); + my_free(d); +} + +void check_bounds(size_t size, size_t out_of_bounds) +{ + auto* s = reinterpret_cast(my_malloc(size)); + auto* d = reinterpret_cast(my_malloc(size)); + for (size_t i = 0; i < size; ++i) + { + s[i] = static_cast(i); + } + for (size_t i = 0; i < size; ++i) + { + d[i] = 0; + } + bool bounds_error = false; + can_longjmp = true; + if (setjmp(jmp) == 0) + { + my_memcpy(d, s, size + out_of_bounds); + } + else + { + bounds_error = true; + } + can_longjmp = false; + SNMALLOC_CHECK(bounds_error == (out_of_bounds > 0)); + my_free(s); + my_free(d); +} + +int main() +{ + // Skip the checks that expect bounds checks to fail when we are not the + // malloc implementation. +# if !defined(SNMALLOC_PASS_THROUGH) + // Some sizes to check for out-of-bounds access + std::initializer_list sizes = {16, 1024, 2 * 1024 * 1024}; + for (auto sz : sizes) + { + // Check in bounds + check_bounds(sz, 0); + // Check one byte out + check_bounds(sz, 1); + // Check one object out of bounds + check_bounds(sz, sz); + } +# endif + for (size_t x = 0; x < 2048; x++) + { + check_size(x); + } +} +#endif From 7f71f80cce3e61b3d44a576cad423574da344db3 Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Mon, 20 Sep 2021 20:25:15 +0100 Subject: [PATCH 075/302] Add compiler abstractions over fast fail. (#392) * Add compiler abstractions over fast fail. * Fix MSVC / GCC's disagreement over inline. * Rework the inline definitions. * Use _snprintf_s_l. --- src/ds/defines.h | 19 +++++++++++++++++++ src/override/memcpy.cc | 12 ++++++------ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/ds/defines.h b/src/ds/defines.h index b0aab14c6..30194935a 100644 --- a/src/ds/defines.h +++ b/src/ds/defines.h @@ -1,12 +1,20 @@ #pragma once #if defined(_MSC_VER) && !defined(__clang__) +// 28 is FAST_FAIL_INVALID_BUFFER_ACCESS. Not using the symbolic constant to +// avoid depending on winnt.h +# define SNMALLOC_FAST_FAIL() __fastfail(28) # define ALWAYSINLINE __forceinline # define NOINLINE __declspec(noinline) # define likely(x) !!(x) # define unlikely(x) !!(x) # define SNMALLOC_SLOW_PATH NOINLINE # define SNMALLOC_FAST_PATH ALWAYSINLINE +/** + * Fast path with inline linkage. MSVC assumes that `__forceinline` implies + * `inline` and complains if you specify `SNMALLOC_FAST_PATH` and `inline`. + */ +# define SNMALLOC_FAST_PATH_INLINE ALWAYSINLINE # if _MSC_VER >= 1927 && !defined(SNMALLOC_USE_CXX17) # define SNMALLOC_FAST_PATH_LAMBDA [[msvc::forceinline]] # else @@ -17,12 +25,23 @@ # define SNMALLOC_REQUIRE_CONSTINIT # define SNMALLOC_UNUSED_FUNCTION #else +# define SNMALLOC_FAST_FAIL() __builtin_trap() # define likely(x) __builtin_expect(!!(x), 1) # define unlikely(x) __builtin_expect(!!(x), 0) # define ALWAYSINLINE __attribute__((always_inline)) # define NOINLINE __attribute__((noinline)) # define SNMALLOC_SLOW_PATH NOINLINE # define SNMALLOC_FAST_PATH ALWAYSINLINE +/** + * Fast path with inline linkage. GCC assumes that + * `__attribute__((always_inline))` is orthogonal to `inline` and complains if + * you specify `SNMALLOC_FAST_PATH` and don't specify `inline` in places where + * `inline` would be required for the one definition rule. The error message + * in this case is confusing: always-inline function may not be inlined. If + * you see this error message when using `SNMALLOC_FAST_PATH` then switch to + * `SNMALLOC_FAST_PATH_INLINE`. + */ +# define SNMALLOC_FAST_PATH_INLINE ALWAYSINLINE inline # define SNMALLOC_FAST_PATH_LAMBDA SNMALLOC_FAST_PATH # define SNMALLOC_PURE __attribute__((const)) # define SNMALLOC_COLD __attribute__((cold)) diff --git a/src/override/memcpy.cc b/src/override/memcpy.cc index 4d41bf35a..82bbf9a0d 100644 --- a/src/override/memcpy.cc +++ b/src/override/memcpy.cc @@ -16,7 +16,7 @@ using namespace snmalloc; // Windows has it with an underscore prefix #elif defined(_MSC_VER) # define snprintf_l(buf, size, loc, msg, ...) \ - _snprintf_l(buf, size, msg, loc, __VA_ARGS__) + _snprintf_s_l(buf, size, _TRUNCATE, msg, loc, __VA_ARGS__) #endif namespace @@ -79,7 +79,7 @@ namespace * expands to a single load and store. */ template - SNMALLOC_FAST_PATH inline void copy_one(void* dst, const void* src) + SNMALLOC_FAST_PATH_INLINE void copy_one(void* dst, const void* src) { #if __has_builtin(__builtin_memcpy_inline) __builtin_memcpy_inline(dst, src, Size); @@ -128,7 +128,7 @@ namespace * function is a no-op when `CheckReads` is false. */ template - SNMALLOC_FAST_PATH inline void + SNMALLOC_FAST_PATH_INLINE void check_bounds(const void* ptr, size_t len, const char* msg = "") { if constexpr (!IsRead || CheckReads) @@ -144,7 +144,7 @@ namespace UNUSED(ptr); UNUSED(len); UNUSED(msg); - __builtin_trap(); + SNMALLOC_FAST_FAIL(); } else { @@ -165,7 +165,7 @@ namespace * chunks of size `Size` as are possible from `len`. */ template - SNMALLOC_FAST_PATH inline void + SNMALLOC_FAST_PATH_INLINE void block_copy(void* dst, const void* src, size_t len) { for (size_t i = 0; (i + Size) <= len; i += Size) @@ -180,7 +180,7 @@ namespace * This may overlap other bits of the copy. */ template - SNMALLOC_FAST_PATH inline void + SNMALLOC_FAST_PATH_INLINE void copy_end(void* dst, const void* src, size_t len) { copy_one( From 1baf675adbb3f3e10a8c8ff47c37f7e53ab3393d Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Wed, 21 Jul 2021 15:38:30 +0100 Subject: [PATCH 076/302] PALNoAlloc should delegate more to underlying PAL --- src/pal/pal_concept.h | 27 ++++++++++++++++++++------- src/pal/pal_noalloc.h | 20 +++++++++++++------- 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/src/pal/pal_concept.h b/src/pal/pal_concept.h index 42e034bc3..d8a328e43 100644 --- a/src/pal/pal_concept.h +++ b/src/pal/pal_concept.h @@ -8,17 +8,29 @@ namespace snmalloc { + + /* + * These concepts enforce that these are indeed constants that fit in the + * desired types. (This is subtly different from saying that they are the + * required types; C++ may handle constants without much regard for their + * claimed type.) + */ + /** - * PALs must advertize the bit vector of their supported features and the - * platform's page size. This concept enforces that these are indeed - * constants that fit in the desired types. (This is subtly different from - * saying that they are the required types; C++ may handle constants without - * much regard for their claimed type.) + * PALs must advertize the bit vector of their supported features. */ template - concept ConceptPAL_static_members = requires() + concept ConceptPAL_static_features = requires() { typename std::integral_constant; + }; + + /** + * PALs must advertise their page size + */ + template + concept ConceptPAL_static_sizes = requires() + { typename std::integral_constant; }; @@ -93,7 +105,8 @@ namespace snmalloc */ template concept ConceptPAL = - ConceptPAL_static_members && + ConceptPAL_static_features && + ConceptPAL_static_sizes && ConceptPAL_error && ConceptPAL_memops && (!pal_supports || diff --git a/src/pal/pal_noalloc.h b/src/pal/pal_noalloc.h index b5c286ee1..2c5c61af8 100644 --- a/src/pal/pal_noalloc.h +++ b/src/pal/pal_noalloc.h @@ -4,23 +4,29 @@ #pragma once #include "../aal/aal.h" +#include "pal_concept.h" #include "pal_consts.h" #include namespace snmalloc { +#ifdef __cpp_concepts + /** + * The minimal subset of a PAL that we need for delegation + */ + template + concept PALNoAllocBase = ConceptPAL_static_sizes&& ConceptPAL_error; +#endif + /** * Platform abstraction layer that does not allow allocation. * * This is a minimal PAL for pre-reserved memory regions, where the * address-space manager is initialised with all of the memory that it will * ever use. - * - * It takes an error handler delegate as a template argument. This is - * expected to forward to the default PAL in most cases. */ - template + template struct PALNoAlloc { /** @@ -29,14 +35,14 @@ namespace snmalloc */ static constexpr uint64_t pal_features = NoAllocation; - static constexpr size_t page_size = Aal::smallest_page_size; + static constexpr size_t page_size = BasePAL::page_size; /** * Print a stack trace. */ static void print_stack_trace() { - ErrorHandler::print_stack_trace(); + BasePAL::print_stack_trace(); } /** @@ -44,7 +50,7 @@ namespace snmalloc */ [[noreturn]] static void error(const char* const str) noexcept { - ErrorHandler::error(str); + BasePAL::error(str); } /** From 4a4ca96125e77eabe236acad99e2a428e76c4d59 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Wed, 21 Jul 2021 16:13:12 +0100 Subject: [PATCH 077/302] Prepare for AAL bits / address_bits --- src/aal/aal.h | 60 +++++++++++++++++++++++++++++++++++++++++++ src/aal/aal_concept.h | 4 ++- 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/src/aal/aal.h b/src/aal/aal.h index 3d35b6676..00f636236 100644 --- a/src/aal/aal.h +++ b/src/aal/aal.h @@ -67,6 +67,66 @@ namespace snmalloc default_address_t, Arch>::address_t; + private: + /** + * SFINAE template and default case. T will be Arch and the second template + * argument defaults at the call site (below). + */ + template + struct default_bits_t + { + static constexpr size_t value = sizeof(size_t) * 8; + }; + + /** + * SFINAE override case. T will be Arch, and the computation in the second + * position yields the type int iff T::bits exists and is a substituion + * failure otherwise. That is, if T::bits exists, this specialization + * shadows the default; otherwise, this specialization has no effect. + */ + template + struct default_bits_t + { + static constexpr size_t value = T::bits; + }; + + public: + /** + * Architectural word width as overridden by the underlying Arch-itecture or + * defaulted as per above. + */ + static constexpr size_t bits = default_bits_t::value; + + private: + /** + * Architectures have a default opinion of their address space size, but + * this is mediated by the platform (e.g., the kernel may cleave the address + * space in twain or my use only some of the radix points available to + * hardware paging mechanisms). + * + * This is more SFINAE-based type-level trickery; see default_bits_t, above, + * for more details. + */ + template + struct default_address_bits_t + { + static constexpr size_t value = (bits == 64) ? 48 : 32; + }; + + /** + * Yet more SFINAE; see default_bits_t for more discussion. Here, the + * computation in the second parameter yields the type int iff + * T::address_bits exists. + */ + template + struct default_address_bits_t + { + static constexpr size_t value = T::address_bits; + }; + + public: + static constexpr size_t address_bits = default_address_bits_t::value; + /** * Prefetch a specific address. * diff --git a/src/aal/aal_concept.h b/src/aal/aal_concept.h index 6b85772ee..180e8930c 100644 --- a/src/aal/aal_concept.h +++ b/src/aal/aal_concept.h @@ -12,13 +12,15 @@ namespace snmalloc { /** * AALs must advertise the bit vector of supported features, their name, - * + * machine word size, and an upper bound on the address space size */ template concept ConceptAAL_static_members = requires() { typename std::integral_constant; typename std::integral_constant; + typename std::integral_constant; + typename std::integral_constant; }; /** From e212ddd0e0e73311f93e288eb80ba6a00fb19dc5 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Wed, 21 Jul 2021 16:14:03 +0100 Subject: [PATCH 078/302] Prepare for PAL address_bits --- src/pal/pal_concept.h | 3 ++- src/pal/pal_noalloc.h | 2 ++ src/pal/pal_posix.h | 12 ++++++++++++ src/pal/pal_windows.h | 5 +++++ 4 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/pal/pal_concept.h b/src/pal/pal_concept.h index d8a328e43..aff935469 100644 --- a/src/pal/pal_concept.h +++ b/src/pal/pal_concept.h @@ -26,11 +26,12 @@ namespace snmalloc }; /** - * PALs must advertise their page size + * PALs must advertise the size of the address space and their page size */ template concept ConceptPAL_static_sizes = requires() { + typename std::integral_constant; typename std::integral_constant; }; diff --git a/src/pal/pal_noalloc.h b/src/pal/pal_noalloc.h index 2c5c61af8..a98add62c 100644 --- a/src/pal/pal_noalloc.h +++ b/src/pal/pal_noalloc.h @@ -37,6 +37,8 @@ namespace snmalloc static constexpr size_t page_size = BasePAL::page_size; + static constexpr size_t address_bits = Aal::address_bits; + /** * Print a stack trace. */ diff --git a/src/pal/pal_posix.h b/src/pal/pal_posix.h index 93b951854..dd3932b66 100644 --- a/src/pal/pal_posix.h +++ b/src/pal/pal_posix.h @@ -132,6 +132,18 @@ namespace snmalloc static constexpr size_t page_size = Aal::smallest_page_size; + /** + * Address bits are potentially mediated by some POSIX OSes, but generally + * default to the architecture's. + * + * Unlike the AALs, which are composited by explicitly delegating to their + * template parameters and so play a SFINAE-based game to achieve similar + * ends, for the PALPOSIX<> classes we instead use more traditional + * inheritance (e.g., PALLinux is subtype of PALPOSIX) and so we + * can just use that mechanism here, too. + */ + static constexpr size_t address_bits = Aal::address_bits; + static void print_stack_trace() { #ifdef SNMALLOC_BACKTRACE_HEADER diff --git a/src/pal/pal_windows.h b/src/pal/pal_windows.h index 98654d1a1..86286d063 100644 --- a/src/pal/pal_windows.h +++ b/src/pal/pal_windows.h @@ -62,6 +62,11 @@ namespace snmalloc static constexpr size_t page_size = 0x1000; + /** + * Windows always inherits its underlying architecture's full address range. + */ + static constexpr size_t address_bits = Aal::address_bits; + /** * Check whether the low memory state is still in effect. This is an * expensive operation and should not be on any fast paths. From 15e30520873ce34e5ab8bd6e6688c4a95d292f7a Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Wed, 21 Jul 2021 16:55:25 +0100 Subject: [PATCH 079/302] Move to AAL/PAL bits and address_bits --- src/backend/pagemap.h | 4 ++-- src/ds/bits.h | 9 +-------- src/mem/sizeclasstable.h | 11 ++++++++--- src/mem/slaballocator.h | 2 +- src/test/func/memory/memory.cc | 2 +- src/test/perf/external_pointer/externalpointer.cc | 2 +- 6 files changed, 14 insertions(+), 16 deletions(-) diff --git a/src/backend/pagemap.h b/src/backend/pagemap.h index 7a794c12f..c3e56674e 100644 --- a/src/backend/pagemap.h +++ b/src/backend/pagemap.h @@ -83,7 +83,7 @@ namespace snmalloc { static_assert( has_bounds_ == has_bounds, "Don't set SFINAE template parameter!"); - constexpr size_t COVERED_BITS = bits::ADDRESS_BITS - GRANULARITY_BITS; + constexpr size_t COVERED_BITS = PAL::address_bits - GRANULARITY_BITS; constexpr size_t ENTRIES = bits::one_at_bit(COVERED_BITS); return ENTRIES * sizeof(T); } @@ -204,7 +204,7 @@ namespace snmalloc } else { - return bits::one_at_bit(bits::ADDRESS_BITS - GRANULARITY_BITS); + return bits::one_at_bit(PAL::address_bits - GRANULARITY_BITS); } } diff --git a/src/ds/bits.h b/src/ds/bits.h index 20e1b4fa9..455ec4ab7 100644 --- a/src/ds/bits.h +++ b/src/ds/bits.h @@ -30,12 +30,7 @@ namespace snmalloc namespace bits { - static constexpr size_t BITS = sizeof(size_t) * 8; - - static constexpr bool is64() - { - return BITS == 64; - } + static constexpr size_t BITS = Aal::bits; /** * Returns a value of type T that has a single bit set, @@ -52,8 +47,6 @@ namespace snmalloc return (static_cast(1)) << shift; } - static constexpr size_t ADDRESS_BITS = is64() ? 48 : 32; - inline SNMALLOC_FAST_PATH size_t clz(size_t x) { SNMALLOC_ASSERT(x != 0); // Calling with 0 is UB on some implementations diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index 3e08c44a8..c406ac1f2 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -41,7 +41,7 @@ namespace snmalloc // Large classes range from [SUPERSLAB, ADDRESS_SPACE).// TODO static constexpr size_t NUM_LARGE_CLASSES = - bits::ADDRESS_BITS - MAX_SIZECLASS_BITS; + Pal::address_bits - MAX_SIZECLASS_BITS; inline SNMALLOC_FAST_PATH static size_t aligned_size(size_t alignment, size_t size) @@ -179,7 +179,7 @@ namespace snmalloc auto rsize = sizeclass_to_size(sc); - if constexpr (bits::is64()) + if constexpr (sizeof(offset) >= 8) { // Only works for 64 bit multiplication, as the following will overflow in // 32bit. @@ -192,6 +192,10 @@ namespace snmalloc // SUPERSLAB_BITS <= 24, "The following code assumes max of 24 bits"); // TODO 24 hack + static_assert(bits::BITS >= 24, "About to attempt a negative shift"); + static_assert( + (8 * sizeof(offset)) >= (bits::BITS - 24), + "About to shift further than the type"); return (((offset >> MIN_ALLOC_BITS) * sizeclass_metadata.fast[sc].div_mult) >> (bits::BITS - 24)) * @@ -209,7 +213,7 @@ namespace snmalloc // SUPERSLAB_SIZE. // SNMALLOC_ASSERT(offset <= SUPERSLAB_SIZE); - if constexpr (bits::is64()) + if constexpr (sizeof(offset) >= 8) { // Only works for 64 bit multiplication, as the following will overflow in // 32bit. @@ -219,6 +223,7 @@ namespace snmalloc // square of the offset to be representable. // TODO 24 hack. Redo the maths given the multiple // slab sizes + static_assert(bits::BITS >= 25); static constexpr size_t MASK = ~(bits::one_at_bit(bits::BITS - 1 - 24) - 1); diff --git a/src/mem/slaballocator.h b/src/mem/slaballocator.h index f714c07ac..5c91b4420 100644 --- a/src/mem/slaballocator.h +++ b/src/mem/slaballocator.h @@ -24,7 +24,7 @@ namespace snmalloc /** * How many slab sizes that can be provided. */ - constexpr size_t NUM_SLAB_SIZES = bits::ADDRESS_BITS - MIN_CHUNK_BITS; + constexpr size_t NUM_SLAB_SIZES = Pal::address_bits - MIN_CHUNK_BITS; /** * Used to ensure the per slab meta data is large enough for both use cases. diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index cf65096ac..a0daef725 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -292,7 +292,7 @@ void test_external_pointer_large() auto& alloc = ThreadAlloc::get(); - constexpr size_t count_log = snmalloc::bits::is64() ? 5 : 3; + constexpr size_t count_log = Pal::address_bits > 32 ? 5 : 3; constexpr size_t count = 1 << count_log; // Pre allocate all the objects size_t* objects[count]; diff --git a/src/test/perf/external_pointer/externalpointer.cc b/src/test/perf/external_pointer/externalpointer.cc index 6a8616d77..e2fdb06be 100644 --- a/src/test/perf/external_pointer/externalpointer.cc +++ b/src/test/perf/external_pointer/externalpointer.cc @@ -19,7 +19,7 @@ namespace test { size_t rand = (size_t)r.next(); size_t offset = bits::clz(rand); - if constexpr (bits::is64()) + if constexpr (Pal::address_bits > 32) { if (offset > 30) offset = 30; From 0af1ee3bef5443c6b2a98c41b05351e91d9426c6 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 13 Aug 2021 15:45:45 +0100 Subject: [PATCH 080/302] Tidy TODOs from Free List * Add extra key to freelist. This follows the encoding Cedric suggested for a signature of two things. Free list key now has a pair of keys for encoding previous pointer. This makes it harder to extract the underlying keys out of the multiplication. * Apply SFINAE to the extract_segment. --- src/mem/entropy.h | 16 ++++++++++------ src/mem/freelist.h | 18 +++++++----------- src/mem/remoteallocator.h | 2 +- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/mem/entropy.h b/src/mem/entropy.h index 9a56cb289..7f1491950 100644 --- a/src/mem/entropy.h +++ b/src/mem/entropy.h @@ -31,11 +31,13 @@ namespace snmalloc struct FreeListKey { - address_t key; + address_t key1; + address_t key2; address_t key_next; - constexpr FreeListKey(uint64_t key, uint64_t key_next) - : key(static_cast(key)), + constexpr FreeListKey(uint64_t key1, uint64_t key2, uint64_t key_next) + : key1(static_cast(key1)), + key2(static_cast(key2)), key_next(static_cast(key_next)) {} }; @@ -47,7 +49,7 @@ namespace snmalloc uint64_t local_counter{0}; uint64_t fresh_bits{0}; uint64_t count{0}; - FreeListKey key{0, 0}; + FreeListKey key{0, 0, 0}; public: constexpr LocalEntropy() = default; @@ -59,12 +61,14 @@ namespace snmalloc local_counter = get_entropy64(); if constexpr (bits::BITS == 64) { - key.key = get_next(); + key.key1 = get_next(); + key.key2 = get_next(); key.key_next = get_next(); } else { - key.key = get_next() & 0xffff'ffff; + key.key1 = get_next() & 0xffff'ffff; + key.key2 = get_next() & 0xffff'ffff; key.key_next = get_next() & 0xffff'ffff; } bit_source = get_next(); diff --git a/src/mem/freelist.h b/src/mem/freelist.h index 31500bb8d..a8bc4dfd6 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -43,20 +43,13 @@ namespace snmalloc { /** * This function is used to sign back pointers in the free list. - * - * TODO - Needs review. Should we shift low bits out as they help - * guess the key. - * - * TODO - We now have space in the FreeListBuilder for a fresh key for each - * list. */ inline static uintptr_t signed_prev(address_t curr, address_t next, const FreeListKey& key) { auto c = curr; auto n = next; - auto k = key.key; - return (c + k) * (n - k); + return (c + key.key1) * (n + key.key2); } /** @@ -65,7 +58,7 @@ namespace snmalloc * back pointer in a doubly linked list, however, it is encoded * to prevent corruption. * - * TODO: Consider put prev_encoded at the end of the object, would + * TODO: Consider putting prev_encoded at the end of the object, would * require size to be threaded through, but would provide more OOB * detection. */ @@ -443,11 +436,14 @@ namespace snmalloc } } - std::pair, CapPtr> + template + std::enable_if_t< + !RANDOM_, + std::pair, CapPtr>> extract_segment(const FreeListKey& key) { + static_assert(RANDOM_ == RANDOM, "Don't set SFINAE parameter!"); SNMALLOC_ASSERT(!empty()); - SNMALLOC_ASSERT(!RANDOM); // TODO: Turn this into a static failure. auto first = read_head(0, key); // end[0] is pointing to the first field in the object, diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index 6e8ea4735..f467cfc7b 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -27,7 +27,7 @@ namespace snmalloc /** * Global key for all remote lists. */ - inline static FreeListKey key_global(0xdeadbeef, 0xdeadbeef); + inline static FreeListKey key_global(0xdeadbeef, 0xbeefdead, 0xdeadbeef); struct alignas(REMOTE_MIN_ALIGN) RemoteAllocator { From b4efc40aa66a60b9ccada117e8758d088aee589b Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 23 Sep 2021 15:56:59 +0100 Subject: [PATCH 081/302] Expose notify_using_readonly This exposes a readonly notify using, so that the underlying platform can map the range of pages readonly into the application. This improves performance of external pointer on platforms that support lazy commit of pages as it can access anything in the range. --- docs/PORTING.md | 9 +++++++++ src/backend/pagemap.h | 27 +++++++++++++++++---------- src/pal/pal_posix.h | 20 +++++++++++++++++++- 3 files changed, 45 insertions(+), 11 deletions(-) diff --git a/docs/PORTING.md b/docs/PORTING.md index 8e394511c..f6c4567f4 100644 --- a/docs/PORTING.md +++ b/docs/PORTING.md @@ -33,6 +33,15 @@ function may not be required to do anything. If the template parameter is set to `YesZero` then this function is also responsible for ensuring that the newly requested memory is full of zeros. +```c++ +static void notify_using_readonly(void* p, size_t size) noexcept; +``` +Notify the system that the range of memory from `p` to `p` + `size` is now in use +for read-only access. This is currently only requried on platforms that support +`LazyCommit`. +On systems that lazily provide physical memory to virtual mappings, this +function may not be required to do anything. + ```c++ template static void zero(void* p, size_t size) noexcept; diff --git a/src/backend/pagemap.h b/src/backend/pagemap.h index c3e56674e..72041624f 100644 --- a/src/backend/pagemap.h +++ b/src/backend/pagemap.h @@ -159,7 +159,7 @@ namespace snmalloc #ifdef SNMALLOC_CHECK_CLIENT // Allocate a power of two extra to allow the placement of the // pagemap be difficult to guess. - size_t additional_size = bits::next_pow2(REQUIRED_SIZE) * 2; + size_t additional_size = bits::next_pow2(REQUIRED_SIZE) * 4; size_t request_size = REQUIRED_SIZE + additional_size; #else size_t request_size = REQUIRED_SIZE; @@ -178,6 +178,18 @@ namespace snmalloc size_t offset = get_entropy64() & (additional_size - sizeof(T)); auto new_body = reinterpret_cast(pointer_offset(new_body_untyped, offset)); + + if constexpr (pal_supports) + { + void* start_page = pointer_align_down(new_body); + void* end_page = pointer_align_up( + pointer_offset(new_body, REQUIRED_SIZE)); + // Only commit readonly memory for this range, if the platform supports + // lazy commit. Otherwise, this would be a lot of memory to have + // mapped. + PAL::notify_using_readonly( + start_page, pointer_diff(start_page, end_page)); + } #else auto new_body = reinterpret_cast(new_body_untyped); #endif @@ -243,15 +255,10 @@ namespace snmalloc p = p - base; } - // This means external pointer on Windows will be slow. - // TODO: With SNMALLOC_CHECK_CLIENT we should map that region read-only, - // not no-access, but this requires a PAL change. - if constexpr ( - potentially_out_of_range -#ifndef SNMALLOC_CHECK_CLIENT - && !pal_supports -#endif - ) + // If this is potentially_out_of_range, then the pages will not have + // been mapped. With Lazy commit they will at least be mapped read-only + // Note that: this means external pointer on Windows will be slow. + if constexpr (potentially_out_of_range && !pal_supports) { register_range(p, 1); } diff --git a/src/pal/pal_posix.h b/src/pal/pal_posix.h index dd3932b66..ff4cd52d4 100644 --- a/src/pal/pal_posix.h +++ b/src/pal/pal_posix.h @@ -196,7 +196,7 @@ namespace snmalloc * * On POSIX platforms, lazy commit means that this is a no-op, unless we * are also zeroing the pages in which case we call the platform's `zero` - * function. + * function, or we have initially mapped the pages as PROT_NONE. */ template static void notify_using(void* p, size_t size) noexcept @@ -215,6 +215,24 @@ namespace snmalloc zero(p, size); } + /** + * Notify platform that we will be using these pages for reading. + * + * On POSIX platforms, lazy commit means that this is a no-op, unless + * we have initially mapped the pages as PROT_NONE. + */ + static void notify_using_readonly(void* p, size_t size) noexcept + { + SNMALLOC_ASSERT(is_aligned_block(p, size)); + +#ifdef SNMALLOC_CHECK_CLIENT + mprotect(p, size, PROT_READ); +#else + UNUSED(p); + UNUSED(size); +#endif + } + /** * OS specific function for zeroing memory. * From 55a7ad2d588c256dab573c715ba7d983d1861736 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 27 Sep 2021 13:32:46 +0100 Subject: [PATCH 082/302] Introduce PalEnforceAccess The various Pals were given different meanings in CHECK_CLIENT and non-CHECK_CLIENT builds. This was because it is essential that in the CHECK_CLIENT builds access is prevented, when not requested. This PR separates the CHECK_CLIENT concept from how the Pal should be implemented. --- docs/PORTING.md | 51 +++++++++++++++++-------------- src/pal/pal_apple.h | 38 +++++++++++------------ src/pal/pal_bsd.h | 12 +++++--- src/pal/pal_bsd_aligned.h | 9 ++---- src/pal/pal_consts.h | 15 +++++++++ src/pal/pal_freebsd_kernel.h | 4 +-- src/pal/pal_posix.h | 59 +++++++++++++++++++----------------- src/pal/pal_windows.h | 4 +-- 8 files changed, 108 insertions(+), 84 deletions(-) diff --git a/docs/PORTING.md b/docs/PORTING.md index f6c4567f4..0f1f2f0a6 100644 --- a/docs/PORTING.md +++ b/docs/PORTING.md @@ -16,31 +16,37 @@ The PAL must implement the following methods: ``` Report a fatal error and exit. +The memory that snmalloc is supplied from the Pal should be in one of three +states + +* `using` +* `using readonly` +* `not using` + +Before accessing the memory for a read, `snmalloc` will change the state to +either `using` or `using readonly`, +and before a write by it will change the state to `using`. +If memory is not required any more, then `snmalloc` will change the state to +`not using`, and will ensure that it notifies the `Pal` again +before it every accesses that memory again. +The `not using` state allows the `Pal` to recycle the memory for other purposes. +If `PalEnforceAccess` is set to true, then accessing that has not been notified +correctly should trigger an exception/segfault. + +The state for a particular region of memory is set with ```c++ static void notify_not_using(void* p, size_t size) noexcept; -``` -Notify the system that the range of memory from `p` to `p` + `size` is no -longer in use, allowing the underlying physical pages to recycled for other -purposes. -```c++ template static void notify_using(void* p, size_t size) noexcept; -``` -Notify the system that the range of memory from `p` to `p` + `size` is now in use. -On systems that lazily provide physical memory to virtual mappings, this -function may not be required to do anything. -If the template parameter is set to `YesZero` then this function is also -responsible for ensuring that the newly requested memory is full of zeros. -```c++ static void notify_using_readonly(void* p, size_t size) noexcept; ``` -Notify the system that the range of memory from `p` to `p` + `size` is now in use -for read-only access. This is currently only requried on platforms that support -`LazyCommit`. -On systems that lazily provide physical memory to virtual mappings, this -function may not be required to do anything. +These functions notify the system that the range of memory from `p` to `p` + +`size` is in the relevant state. + +If the template parameter is set to `YesZero` then `notify_using` must ensure +the range is full of zeros. ```c++ template @@ -54,18 +60,17 @@ efficient to request that the operating system provides background-zeroed pages, rather than zeroing them synchronously in this call ```c++ -template +template static void* reserve_aligned(size_t size) noexcept; static void* reserve(size_t size) noexcept; ``` All platforms should provide `reserve` and can optionally provide `reserve_aligned` if the underlying system can provide strongly aligned memory regions. -If the system guarantees only page alignment, implement only the second. The Pal is -free to overallocate based on the platform's desire and snmalloc -will find suitably aligned blocks inside the region. `reserve` should -not commit memory as snmalloc will commit the range of memory it requires of what -is returned. +If the system guarantees only page alignment, implement only the second. `snmalloc` will +overallocate to ensure it can find suitably aligned blocks inside the region. +`reserve` should consider memory initially as `not_using`, and `snmalloc` will notify when it +needs the range of memory. If the system provides strong alignment, implement the first to return memory at the desired alignment. If providing the first, then the `Pal` should also diff --git a/src/pal/pal_apple.h b/src/pal/pal_apple.h index a36e10c91..14fb9f0c0 100644 --- a/src/pal/pal_apple.h +++ b/src/pal/pal_apple.h @@ -113,7 +113,7 @@ namespace snmalloc { SNMALLOC_ASSERT(is_aligned_block(p, size)); -# if defined(SNMALLOC_CHECK_CLIENT) && !defined(NDEBUG) +# if !defined(NDEBUG) memset(p, 0x5a, size); # endif @@ -126,12 +126,13 @@ namespace snmalloc while (madvise(p, size, MADV_FREE_REUSABLE) == -1 && errno == EAGAIN) ; -# ifdef SNMALLOC_CHECK_CLIENT - // This must occur after `MADV_FREE_REUSABLE`. - // - // `mach_vm_protect` is observably slower in benchmarks. - mprotect(p, size, PROT_NONE); -# endif + if constexpr (PalEnforceAccess) + { + // This must occur after `MADV_FREE_REUSABLE`. + // + // `mach_vm_protect` is observably slower in benchmarks. + mprotect(p, size, PROT_NONE); + } } /** @@ -180,12 +181,13 @@ namespace snmalloc } } -# ifdef SNMALLOC_CHECK_CLIENT - // Mark pages as writable for `madvise` below. - // - // `mach_vm_protect` is observably slower in benchmarks. - mprotect(p, size, PROT_READ | PROT_WRITE); -# endif + if constexpr (PalEnforceAccess) + { + // Mark pages as writable for `madvise` below. + // + // `mach_vm_protect` is observably slower in benchmarks. + mprotect(p, size, PROT_READ | PROT_WRITE); + } // `MADV_FREE_REUSE` can only be applied to writable pages, // otherwise it's an error. @@ -205,7 +207,7 @@ namespace snmalloc // Apple's `mmap` doesn't support user-specified alignment and only // guarantees mappings are aligned to the system page size, so we use // `mach_vm_map` instead. - template + template static void* reserve_aligned(size_t size) noexcept { SNMALLOC_ASSERT(bits::is_pow2(size)); @@ -219,11 +221,9 @@ namespace snmalloc // must be initialized to 0 or addr is interepreted as a lower-bound. mach_vm_address_t addr = 0; -# ifdef SNMALLOC_CHECK_CLIENT - vm_prot_t prot = committed ? VM_PROT_READ | VM_PROT_WRITE : VM_PROT_NONE; -# else - vm_prot_t prot = VM_PROT_READ | VM_PROT_WRITE; -# endif + vm_prot_t prot = (state_using || !PalEnforceAccess) ? + VM_PROT_READ | VM_PROT_WRITE : + VM_PROT_NONE; kern_return_t kr = mach_vm_map( mach_task_self(), diff --git a/src/pal/pal_bsd.h b/src/pal/pal_bsd.h index 38b357165..cf6424499 100644 --- a/src/pal/pal_bsd.h +++ b/src/pal/pal_bsd.h @@ -34,14 +34,16 @@ namespace snmalloc static void notify_not_using(void* p, size_t size) noexcept { SNMALLOC_ASSERT(is_aligned_block(p, size)); - // Call this Pal to simulate the Windows decommit in CI. -#if defined(SNMALLOC_CHECK_CLIENT) && !defined(NDEBUG) + +#if !defined(NDEBUG) memset(p, 0x5a, size); #endif madvise(p, size, MADV_FREE); -#ifdef SNMALLOC_CHECK_CLIENT - mprotect(p, size, PROT_NONE); -#endif + + if constexpr (PalEnforceAccess) + { + mprotect(p, size, PROT_NONE); + } } }; } // namespace snmalloc diff --git a/src/pal/pal_bsd_aligned.h b/src/pal/pal_bsd_aligned.h index 592b842bd..7f1c1ea72 100644 --- a/src/pal/pal_bsd_aligned.h +++ b/src/pal/pal_bsd_aligned.h @@ -28,7 +28,7 @@ namespace snmalloc /** * Reserve memory at a specific alignment. */ - template + template static void* reserve_aligned(size_t size) noexcept { // Alignment must be a power of 2. @@ -37,11 +37,8 @@ namespace snmalloc int log2align = static_cast(bits::next_pow2_bits(size)); -#ifdef SNMALLOC_CHECK_CLIENT - auto prot = committed ? PROT_READ | PROT_WRITE : PROT_NONE; -#else - auto prot = PROT_READ | PROT_WRITE; -#endif + auto prot = + state_using || !PalEnforceAccess ? PROT_READ | PROT_WRITE : PROT_NONE; void* p = mmap( nullptr, diff --git a/src/pal/pal_consts.h b/src/pal/pal_consts.h index 84823fed6..0cb9e81d4 100644 --- a/src/pal/pal_consts.h +++ b/src/pal/pal_consts.h @@ -6,6 +6,21 @@ namespace snmalloc { + /** + * Pal implementations should query this flag to see whether they + * are allowed to optimise memory access, or that they must provide + * exceptions/segfaults if accesses do not obey the + * - using + * - using_readonly + * - not_using + * model. + */ +#ifdef SNMALLOC_CHECK_CLIENT + static constexpr bool PalEnforceAccess = true; +#else + static constexpr bool PalEnforceAccess = false; +#endif + /** * Flags in a bitfield of optional features that a PAL may support. These * should be set in the PAL's `pal_features` static constexpr field. diff --git a/src/pal/pal_freebsd_kernel.h b/src/pal/pal_freebsd_kernel.h index 084b38b39..72f84870c 100644 --- a/src/pal/pal_freebsd_kernel.h +++ b/src/pal/pal_freebsd_kernel.h @@ -59,7 +59,7 @@ namespace snmalloc ::bzero(p, size); } - template + template static void* reserve_aligned(size_t size) noexcept { SNMALLOC_ASSERT(bits::is_pow2(size)); @@ -80,7 +80,7 @@ namespace snmalloc { return nullptr; } - if (committed) + if (state_using) { if ( kmem_back(kernel_object, addr, size, M_ZERO | M_WAITOK) != diff --git a/src/pal/pal_posix.h b/src/pal/pal_posix.h index ff4cd52d4..55d96a5e2 100644 --- a/src/pal/pal_posix.h +++ b/src/pal/pal_posix.h @@ -178,17 +178,21 @@ namespace snmalloc static void notify_not_using(void* p, size_t size) noexcept { SNMALLOC_ASSERT(is_aligned_block(p, size)); -#ifdef SNMALLOC_CHECK_CLIENT - // Fill memory so that when we switch the pages back on we don't make - // assumptions on the content. -# if !defined(NDEBUG) - memset(p, 0x5a, size); -# endif - mprotect(p, size, PROT_NONE); -#else - UNUSED(p); - UNUSED(size); + + if constexpr (PalEnforceAccess) + { +#if !defined(NDEBUG) + // Fill memory so that when we switch the pages back on we don't make + // assumptions on the content. + memset(p, 0x5a, size); #endif + mprotect(p, size, PROT_NONE); + } + else + { + UNUSED(p); + UNUSED(size); + } } /** @@ -204,12 +208,13 @@ namespace snmalloc SNMALLOC_ASSERT( is_aligned_block(p, size) || (zero_mem == NoZero)); -#ifdef SNMALLOC_CHECK_CLIENT - mprotect(p, size, PROT_READ | PROT_WRITE); -#else - UNUSED(p); - UNUSED(size); -#endif + if constexpr (PalEnforceAccess) + mprotect(p, size, PROT_READ | PROT_WRITE); + else + { + UNUSED(p); + UNUSED(size); + } if constexpr (zero_mem == YesZero) zero(p, size); @@ -225,12 +230,13 @@ namespace snmalloc { SNMALLOC_ASSERT(is_aligned_block(p, size)); -#ifdef SNMALLOC_CHECK_CLIENT - mprotect(p, size, PROT_READ); -#else - UNUSED(p); - UNUSED(size); -#endif + if constexpr (PalEnforceAccess) + mprotect(p, size, PROT_READ); + else + { + UNUSED(p); + UNUSED(size); + } } /** @@ -286,11 +292,10 @@ namespace snmalloc */ static void* reserve(size_t size) noexcept { -#ifdef SNMALLOC_CHECK_CLIENT - auto prot = PROT_NONE; -#else - auto prot = PROT_READ | PROT_WRITE; -#endif + // If enforcing access, map pages initially as None, and then + // add permissions as required. Otherwise, immediately give all + // access as this is the most efficient to implement. + auto prot = PalEnforceAccess ? PROT_NONE : PROT_READ | PROT_WRITE; void* p = mmap( nullptr, diff --git a/src/pal/pal_windows.h b/src/pal/pal_windows.h index 86286d063..425810bb6 100644 --- a/src/pal/pal_windows.h +++ b/src/pal/pal_windows.h @@ -158,7 +158,7 @@ namespace snmalloc } # ifdef PLATFORM_HAS_VIRTUALALLOC2 - template + template static void* reserve_aligned(size_t size) noexcept { SNMALLOC_ASSERT(bits::is_pow2(size)); @@ -166,7 +166,7 @@ namespace snmalloc DWORD flags = MEM_RESERVE; - if (committed) + if (state_using) flags |= MEM_COMMIT; // If we're on Windows 10 or newer, we can use the VirtualAlloc2 From dbb796550726e3683bb7c37907234ff8e04960f9 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 28 Sep 2021 11:17:53 +0100 Subject: [PATCH 083/302] Minor. --- src/mem/corealloc.h | 2 +- src/mem/metaslab.h | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 87c6aa78a..60d29ebd6 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -747,7 +747,7 @@ namespace snmalloc else error("debug_is_empty: found non-empty allocator"); } - curr = currmeta->link.get_next(); + curr = curr->get_next(); } } }; diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 6212faa8f..0c9954a19 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -32,8 +32,6 @@ namespace snmalloc /** * Data-structure for building the free list for this slab. - * - * Spare 32bits are used for the fields in MetaslabEnd. */ #ifdef SNMALLOC_CHECK_CLIENT FreeListBuilder free_queue; From 8ac2adc4e597ff7a39f6b6acecac8d70a9ecc724 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 29 Sep 2021 10:01:31 +0100 Subject: [PATCH 084/302] Added a sequential queue This changes the slab lists to use a sequential queue. They were previously stored in a stack. This commit also tidies up some incomplete refactoring from the initial snmalloc2 work. --- src/ds/cdllist.h | 93 --------------------------------------- src/ds/seqqueue.h | 105 ++++++++++++++++++++++++++++++++++++++++++++ src/mem/corealloc.h | 71 ++++++++++++------------------ src/mem/metaslab.h | 25 +++-------- 4 files changed, 139 insertions(+), 155 deletions(-) delete mode 100644 src/ds/cdllist.h create mode 100644 src/ds/seqqueue.h diff --git a/src/ds/cdllist.h b/src/ds/cdllist.h deleted file mode 100644 index 56c601f97..000000000 --- a/src/ds/cdllist.h +++ /dev/null @@ -1,93 +0,0 @@ -#pragma once - -#include "address.h" -#include "defines.h" -#include "ptrwrap.h" - -#include -#include - -namespace snmalloc -{ - /** - * TODO Rewrite for actual use, no longer Cyclic or doubly linked. - * - * Special class for cyclic doubly linked non-empty linked list - * - * This code assumes there is always one element in the list. The client - * must ensure there is a sentinal element. - */ - template typename Ptr = Pointer> - class CDLLNode - { - Ptr next{nullptr}; - - constexpr void set_next(Ptr c) - { - next = c; - } - - public: - /** - * Single element cyclic list. This is the empty case. - */ - constexpr CDLLNode() - { - this->set_next(nullptr); - } - - SNMALLOC_FAST_PATH bool is_empty() - { - return next == nullptr; - } - - SNMALLOC_FAST_PATH Ptr get_next() - { - return next; - } - - /** - * Single element cyclic list. This is the uninitialised case. - * - * This entry should never be accessed and is only used to make - * a fake metaslab. - */ - constexpr CDLLNode(bool) {} - - SNMALLOC_FAST_PATH Ptr pop() - { - SNMALLOC_ASSERT(!this->is_empty()); - auto result = get_next(); - set_next(result->get_next()); - return result; - } - - SNMALLOC_FAST_PATH void insert(Ptr item) - { - debug_check(); - item->set_next(this->get_next()); - set_next(item); - debug_check(); - } - - /** - * Checks the lists invariants - * x->next->prev = x - * for all x in the list. - */ - void debug_check() - { -#ifndef NDEBUG - // Ptr item = this->get_next(); - // auto p = Ptr(this); - - // do - // { - // SNMALLOC_ASSERT(item->prev == p); - // p = item; - // item = item->get_next(); - // } while (item != Ptr(this)); -#endif - } - }; -} // namespace snmalloc diff --git a/src/ds/seqqueue.h b/src/ds/seqqueue.h new file mode 100644 index 000000000..420e4e780 --- /dev/null +++ b/src/ds/seqqueue.h @@ -0,0 +1,105 @@ +#pragma once + +#include "address.h" +#include "defines.h" +#include "ptrwrap.h" + +#include +#include + +namespace snmalloc +{ + /** + * Simple sequential queue of T. + * + * Linked using the T::next field. + */ + template + class SeqQueue + { + static_assert( + std::is_same::value, + "T->next must be a queue pointer to T"); + T* head{nullptr}; + T** end{&head}; + + public: + /** + * Empty queue + */ + constexpr SeqQueue() = default; + + /** + * Check for empty + */ + SNMALLOC_FAST_PATH bool is_empty() + { + SNMALLOC_ASSERT(end != nullptr); + return &head == end; + } + + /** + * Remove an element from the queue + * + * Assumes queue is non-empty + */ + SNMALLOC_FAST_PATH T* pop() + { + SNMALLOC_ASSERT(!this->is_empty()); + auto result = head; + if (&(head->next) == end) + end = &head; + else + head = head->next; + return result; + } + + /** + * Filter + * + * Removes all elements that f returns true for. + * If f returns true, then filter is not allowed to look at the + * object again, and f is responsible for its lifetime. + */ + template + SNMALLOC_FAST_PATH void filter(Fn&& f) + { + T** prev = &head; + // Check for empty case. + if (prev == end) + return; + + while (true) + { + T* curr = *prev; + // Note must read curr->next before calling `f` as `f` is allowed to + // mutate that field. + T* next = curr->next; + if (f(curr)) + { + // Remove element; + *prev = next; + } + else + { + // Keep element + prev = &(curr->next); + } + + if (&(curr->next) == end) + break; + } + + end = prev; + } + + /** + * Add an element to the queue. + */ + SNMALLOC_FAST_PATH void insert(T* item) + { + *end = item; + end = &(item->next); + } + }; +} // namespace snmalloc diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 60d29ebd6..9b3909bd5 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -299,32 +299,23 @@ namespace snmalloc SNMALLOC_SLOW_PATH void dealloc_local_slabs(sizeclass_t sizeclass) { // Return unused slabs of sizeclass_t back to global allocator - SlabLink* prev = &alloc_classes[sizeclass]; - auto curr = prev->get_next(); - while (curr != nullptr) - { - auto nxt = curr->get_next(); - auto meta = Metaslab::from_link(curr); - if (meta->needed() == 0) - { - prev->pop(); - alloc_classes[sizeclass].length--; - alloc_classes[sizeclass].unused--; + alloc_classes[sizeclass].queue.filter([this, sizeclass](Metaslab* meta) { + if (meta->needed() != 0) + return false; - // TODO delay the clear to the next user of the slab, or teardown so - // don't touch the cache lines at this point in check_client. - auto chunk_record = clear_slab(meta, sizeclass); - ChunkAllocator::dealloc( - get_backend_local_state(), - chunk_record, - sizeclass_to_slab_sizeclass(sizeclass)); - } - else - { - prev = curr; - } - curr = nxt; - } + alloc_classes[sizeclass].length--; + alloc_classes[sizeclass].unused--; + + // TODO delay the clear to the next user of the slab, or teardown so + // don't touch the cache lines at this point in check_client. + auto chunk_record = clear_slab(meta, sizeclass); + ChunkAllocator::dealloc( + get_backend_local_state(), + chunk_record, + sizeclass_to_slab_sizeclass(sizeclass)); + + return true; + }); } /** @@ -348,7 +339,7 @@ namespace snmalloc // Wake slab up. meta->set_not_sleeping(sizeclass); - alloc_classes[sizeclass].insert(&meta->link); + alloc_classes[sizeclass].queue.insert(meta); alloc_classes[sizeclass].length++; #ifdef SNMALLOC_TRACING @@ -590,10 +581,10 @@ namespace snmalloc size_t rsize = sizeclass_to_size(sizeclass); // Look to see if we can grab a free list. - auto& sl = alloc_classes[sizeclass]; + auto& sl = alloc_classes[sizeclass].queue; if (likely(!(sl.is_empty()))) { - auto meta = Metaslab::from_link(sl.pop()); + auto meta = sl.pop(); // Drop length of sl, and empty count if it was empty. alloc_classes[sizeclass].length--; if (meta->needed() == 0) @@ -734,29 +725,23 @@ namespace snmalloc bool debug_is_empty_impl(bool* result) { auto test = [&result](auto& queue) { - if (!queue.is_empty()) - { - auto curr = queue.get_next(); - while (curr != nullptr) + queue.filter([&result](auto metaslab) { + if (metaslab->needed() != 0) { - auto currmeta = Metaslab::from_link(curr); - if (currmeta->needed() != 0) - { - if (result != nullptr) - *result = false; - else - error("debug_is_empty: found non-empty allocator"); - } - curr = curr->get_next(); + if (result != nullptr) + *result = false; + else + error("debug_is_empty: found non-empty allocator"); } - } + return false; + }); }; bool sent_something = flush(true); for (auto& alloc_class : alloc_classes) { - test(alloc_class); + test(alloc_class.queue); } // Place the static stub message on the queue. diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 0c9954a19..88f3721ae 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -1,8 +1,8 @@ #pragma once -#include "../ds/cdllist.h" #include "../ds/dllist.h" #include "../ds/helpers.h" +#include "../ds/seqqueue.h" #include "../mem/remoteallocator.h" #include "freelist.h" #include "ptrhelpers.h" @@ -12,23 +12,15 @@ namespace snmalloc { class Slab; - using SlabLink = CDLLNode<>; - // The Metaslab represent the status of a single slab. // This can be either a short or a standard slab. class alignas(CACHELINE_SIZE) Metaslab { public: - // TODO: Annotate with CHERI subobject unbound for pointer arithmetic - SlabLink link; - - constexpr Metaslab() : link(true) {} + // Used to link metaslabs together in various other data-structures. + Metaslab* next{nullptr}; - /** - * Metaslab::link points at another link field. To get the actual Metaslab, - * use this encapsulation of the container-of logic. - */ - static Metaslab* from_link(SlabLink* ptr); + constexpr Metaslab() = default; /** * Data-structure for building the free list for this slab. @@ -167,12 +159,6 @@ namespace snmalloc } }; - inline Metaslab* Metaslab::from_link(SlabLink* lptr) - { - return pointer_offset_signed( - lptr, -static_cast(offsetof(Metaslab, link))); - } - struct RemoteAllocator; /** @@ -232,8 +218,9 @@ namespace snmalloc } }; - struct MetaslabCache : public CDLLNode<> + struct MetaslabCache { + SeqQueue queue; uint16_t unused = 0; uint16_t length = 0; }; From bba66e4f7eb470ad771f61f38070bff2c80bd806 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 7 Oct 2021 15:51:18 +0100 Subject: [PATCH 085/302] Randomise slab filling (#397) # Free List builder track length This commit makes the free list builder track the length of the lists in the Random case. # Refactor free list creation. Minor refactoring to share code between the new free list and existing path. # Randomise slab filling Knowing when a slab is going to become full makes it easier to by pass the free list entries as protection for OOB writes. This commit randomises when a slab will become full. This commit changes two things * the free list builder can return some fraction of the deallocations on a slab. * when there is a single free slab, we can with some probability allocate an additional slab. These two combine to make it difficult to predict when a slab will be free. # Apply suggestions from code review Co-authored-by: Nathaniel Wesley Filardo --- src/mem/corealloc.h | 65 ++++++++++++++++++++++++++++++++++----------- src/mem/freelist.h | 53 +++++++++++++++++++++--------------- src/mem/metaslab.h | 48 ++++++++++++++++++++++++--------- 3 files changed, 117 insertions(+), 49 deletions(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 9b3909bd5..cb3869d8d 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -188,7 +188,7 @@ namespace snmalloc static SNMALLOC_FAST_PATH void alloc_new_list( CapPtr& bumpptr, - FreeListIter& fast_free_list, + Metaslab* meta, size_t rsize, size_t slab_size, LocalEntropy& entropy) @@ -197,8 +197,7 @@ namespace snmalloc auto& key = entropy.get_free_list_key(); - FreeListBuilder b; - SNMALLOC_ASSERT(b.empty()); + auto& b = meta->free_queue; #ifdef SNMALLOC_CHECK_CLIENT // Structure to represent the temporary list elements @@ -243,7 +242,7 @@ namespace snmalloc auto curr_ptr = start_ptr; do { - b.add(FreeObject::make(curr_ptr.as_void()), key); + b.add(FreeObject::make(curr_ptr.as_void()), key, entropy); curr_ptr = curr_ptr->next; } while (curr_ptr != start_ptr); #else @@ -256,16 +255,14 @@ namespace snmalloc #endif // This code consumes everything up to slab_end. bumpptr = slab_end; - - SNMALLOC_ASSERT(!b.empty()); - b.close(fast_free_list, key); } ChunkRecord* clear_slab(Metaslab* meta, sizeclass_t sizeclass) { auto& key = entropy.get_free_list_key(); FreeListIter fl; - meta->free_queue.close(fl, key); + auto more = meta->free_queue.close(fl, key); + UNUSED(more); void* p = finish_alloc_no_zero(fl.take(key), sizeclass); #ifdef SNMALLOC_CHECK_CLIENT @@ -278,6 +275,21 @@ namespace snmalloc count++; } // Check the list contains all the elements + SNMALLOC_ASSERT( + (count + more) == snmalloc::sizeclass_to_slab_object_count(sizeclass)); + + if (more > 0) + { + auto no_more = meta->free_queue.close(fl, key); + SNMALLOC_ASSERT(no_more == 0); + UNUSED(no_more); + + while (!fl.empty()) + { + fl.take(key); + count++; + } + } SNMALLOC_ASSERT( count == snmalloc::sizeclass_to_slab_object_count(sizeclass)); #endif @@ -582,15 +594,32 @@ namespace snmalloc // Look to see if we can grab a free list. auto& sl = alloc_classes[sizeclass].queue; - if (likely(!(sl.is_empty()))) + if (likely(alloc_classes[sizeclass].length > 0)) { +#ifdef SNMALLOC_CHECK_CLIENT + // Occassionally don't use the last list. + if ( + unlikely(alloc_classes[sizeclass].length == 1) && + (entropy.next_bit() == 0)) + { + return small_alloc_slow(sizeclass, fast_free_list, rsize); + } +#endif + auto meta = sl.pop(); // Drop length of sl, and empty count if it was empty. alloc_classes[sizeclass].length--; if (meta->needed() == 0) alloc_classes[sizeclass].unused--; - auto p = Metaslab::alloc(meta, fast_free_list, entropy, sizeclass); + auto [p, still_active] = + Metaslab::alloc_free_list(meta, fast_free_list, entropy, sizeclass); + + if (still_active) + { + alloc_classes[sizeclass].length++; + sl.insert(meta); + } return finish_alloc(p, sizeclass); } @@ -641,16 +670,20 @@ namespace snmalloc return nullptr; } - // Build a free list for the slab - alloc_new_list(slab, fast_free_list, rsize, slab_size, entropy); - // Set meta slab to empty. meta->initialise(sizeclass); - auto& key = entropy.get_free_list_key(); + // Build a free list for the slab + alloc_new_list(slab, meta, rsize, slab_size, entropy); + + auto [p, still_active] = + Metaslab::alloc_free_list(meta, fast_free_list, entropy, sizeclass); - // take an allocation from the free list - auto p = fast_free_list.take(key); + if (still_active) + { + alloc_classes[sizeclass].length++; + alloc_classes[sizeclass].queue.insert(meta); + } return finish_alloc(p, sizeclass); } diff --git a/src/mem/freelist.h b/src/mem/freelist.h index a8bc4dfd6..9cfa49bf9 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -301,6 +301,8 @@ namespace snmalloc // This enables branch free enqueuing. std::array*, LENGTH> end{nullptr}; + std::array length{}; + public: constexpr FreeListBuilder() { @@ -336,6 +338,10 @@ namespace snmalloc index = 0; end[index] = FreeObject::store_next(end[index], n, key); + if constexpr (RANDOM) + { + length[index]++; + } } /** @@ -388,41 +394,42 @@ namespace snmalloc /** * Close a free list, and set the iterator parameter * to iterate it. + * + * In the RANDOM case, it may return only part of the freelist. + * + * The return value is how many entries are still contained in the builder. */ - SNMALLOC_FAST_PATH void close(FreeListIter& fl, const FreeListKey& key) + SNMALLOC_FAST_PATH uint16_t close(FreeListIter& fl, const FreeListKey& key) { + uint32_t i; if constexpr (RANDOM) { SNMALLOC_ASSERT(end[1] != &head[0]); SNMALLOC_ASSERT(end[0] != &head[1]); - // If second list is non-empty, perform append. - if (end[1] != &head[1]) - { - // The start token has been corrupted. - // TOCTTOU issue, but small window here. - read_head(1, key)->check_prev(get_fake_signed_prev(1, key)); + // Select longest list. + i = length[0] > length[1] ? 0 : 1; + } + else + { + i = 0; + } - terminate_list(1, key); + terminate_list(i, key); - // Append 1 to 0 - FreeObject::store_next(end[0], read_head(1, key), key); + fl = {read_head(i, key), get_fake_signed_prev(i, key)}; - SNMALLOC_ASSERT(end[1] != &head[0]); - SNMALLOC_ASSERT(end[0] != &head[1]); - } - else - { - terminate_list(0, key); - } + end[i] = &head[i]; + + if constexpr (RANDOM) + { + length[i] = 0; + return length[1 - i]; } else { - terminate_list(0, key); + return 0; } - - fl = {read_head(0, key), get_fake_signed_prev(0, key)}; - init(); } /** @@ -433,6 +440,10 @@ namespace snmalloc for (size_t i = 0; i < LENGTH; i++) { end[i] = &head[i]; + if (RANDOM) + { + length[i] = 0; + } } } diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 88f3721ae..66dbecbdb 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -71,7 +71,7 @@ namespace snmalloc // returned all the elements, but this is a slab that is still being bump // allocated from. Hence, the bump allocator slab will never be returned // for use in another size class. - set_sleeping(sizeclass); + set_sleeping(sizeclass, 0); } /** @@ -96,14 +96,28 @@ namespace snmalloc return sleeping(); } - SNMALLOC_FAST_PATH void set_sleeping(sizeclass_t sizeclass) + /** + * Try to set this metaslab to sleep. If the remaining elements are fewer + * than the threshold, then it will actually be set to the sleeping state, + * and will return true, otherwise it will return false. + */ + SNMALLOC_FAST_PATH bool + set_sleeping(sizeclass_t sizeclass, uint16_t remaining) { - SNMALLOC_ASSERT(free_queue.empty()); + auto threshold = threshold_for_waking_slab(sizeclass); + if (remaining >= threshold) + { + // Set needed to at least one, possibly more so we only use + // a slab when it has a reasonable amount of free elements + auto allocated = sizeclass_to_slab_object_count(sizeclass); + needed() = allocated - remaining; + sleeping() = false; + return false; + } - // Set needed to at least one, possibly more so we only use - // a slab when it has a reasonable amount of free elements - needed() = threshold_for_waking_slab(sizeclass); sleeping() = true; + needed() = threshold - remaining; + return true; } SNMALLOC_FAST_PATH void set_not_sleeping(sizeclass_t sizeclass) @@ -129,9 +143,18 @@ namespace snmalloc } /** - * TODO + * Allocates a free list from the meta data. + * + * Returns a freshly allocated object of the correct size, and a bool that + * specifies if the metaslab should be placed in the queue for that + * sizeclass. + * + * If Randomisation is not used, it will always return false for the second + * component, but with randomisation, it may only return part of the + * available objects for this metaslab. */ - static SNMALLOC_FAST_PATH CapPtr alloc( + static SNMALLOC_FAST_PATH std::pair, bool> + alloc_free_list( Metaslab* meta, FreeListIter& fast_free_list, LocalEntropy& entropy, @@ -140,7 +163,7 @@ namespace snmalloc auto& key = entropy.get_free_list_key(); FreeListIter tmp_fl; - meta->free_queue.close(tmp_fl, key); + auto remaining = meta->free_queue.close(tmp_fl, key); auto p = tmp_fl.take(key); fast_free_list = tmp_fl; @@ -150,12 +173,13 @@ namespace snmalloc UNUSED(entropy); #endif - // Treat stealing the free list as allocating it all. // This marks the slab as sleeping, and sets a wakeup // when sufficient deallocations have occurred to this slab. - meta->set_sleeping(sizeclass); + // Takes how many deallocations were not grabbed on this call + // This will be zero if there is no randomisation. + auto sleeping = meta->set_sleeping(sizeclass, remaining); - return p; + return {p, !sleeping}; } }; From 6470d62635ca39ea083839540cdfbe5512220ec5 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 7 Oct 2021 17:17:53 +0100 Subject: [PATCH 086/302] Fix teardown for main thread when using pthread destructors. Co-authored with David Chisnall --- src/mem/threadalloc.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/mem/threadalloc.h b/src/mem/threadalloc.h index bf5f822a1..4f6b674e9 100644 --- a/src/mem/threadalloc.h +++ b/src/mem/threadalloc.h @@ -104,13 +104,29 @@ namespace snmalloc { ThreadAlloc::get().teardown(); } + + /** + * Used to give correct signature to teardown required by atexit. + */ + inline void pthread_cleanup_main_thread() + { + ThreadAlloc::get().teardown(); + } + /** * Used to give correct signature to the pthread call for the Singleton class. */ inline void pthread_create(pthread_key_t* key) noexcept { pthread_key_create(key, &pthread_cleanup); + // Main thread does not call pthread_cleanup if `main` returns or `exit` is + // called, so use an atexit handler to guarantee that the cleanup is run at + // least once. If the main thread exits with `pthread_exit` then it will be + // called twice but this case is already handled because other destructors + // can cause the per-thread allocator to be recreated. + atexit(&pthread_cleanup_main_thread); } + /** * Performs thread local teardown for the allocator using the pthread library. * From 73ebb69955134aae29941f8651cc141e3450aad2 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Sun, 26 Sep 2021 18:13:53 +0100 Subject: [PATCH 087/302] localcache: drop some useless snmalloc:: namespace qualifiers --- src/mem/localcache.h | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/mem/localcache.h b/src/mem/localcache.h index 74a35fbab..65a61073f 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -12,9 +12,8 @@ namespace snmalloc { using Stats = AllocStats; - inline static SNMALLOC_FAST_PATH void* finish_alloc_no_zero( - snmalloc::CapPtr p, - sizeclass_t sizeclass) + inline static SNMALLOC_FAST_PATH void* + finish_alloc_no_zero(CapPtr p, sizeclass_t sizeclass) { SNMALLOC_ASSERT(Metaslab::is_start_of_object(sizeclass, address_cast(p))); UNUSED(sizeclass); @@ -25,9 +24,8 @@ namespace snmalloc } template - inline static SNMALLOC_FAST_PATH void* finish_alloc( - snmalloc::CapPtr p, - sizeclass_t sizeclass) + inline static SNMALLOC_FAST_PATH void* + finish_alloc(CapPtr p, sizeclass_t sizeclass) { auto r = finish_alloc_no_zero(p, sizeclass); From 3109ae9f72ae2c8e2a28cb7af842bd731f8c4a08 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Tue, 28 Sep 2021 15:57:03 +0100 Subject: [PATCH 088/302] NFC: Accumulated nits in comments Mostly, promote some inline commentary to doc comments. A typo and some stale text can go, too. --- src/mem/freelist.h | 8 +------- src/mem/localcache.h | 5 +++-- src/mem/remoteallocator.h | 10 +++++++--- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/mem/freelist.h b/src/mem/freelist.h index 9cfa49bf9..eab71eb44 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -271,12 +271,6 @@ namespace snmalloc * * Adds signing of pointers in the SNMALLOC_CHECK_CLIENT mode * - * We use the template parameter, so that an enclosing - * class can make use of the remaining bytes, which may not - * be aligned. On 64bit ptr architectures, this structure - * is a multiple of 8 bytes in the checked and random more. - * But on 128bit ptr architectures this may be a benefit. - * * If RANDOM is enabled, the builder uses two queues, and * "randomly" decides to add to one of the two queues. This * means that we will maintain a randomisation of the order @@ -348,7 +342,7 @@ namespace snmalloc * Adds an element to the builder, if we are guaranteed that * RANDOM is false. This is useful in certain construction * cases that do not need to introduce randomness, such as - * during the initialation construction of a free list, which + * during the initialisation construction of a free list, which * uses its own algorithm, or during building remote deallocation * lists, which will be randomised at the other end. */ diff --git a/src/mem/localcache.h b/src/mem/localcache.h index 65a61073f..eaaf6c106 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -68,6 +68,9 @@ namespace snmalloc : remote_allocator(remote_allocator) {} + /** + * Return all the free lists to the allocator. Used during thread teardown. + */ template< size_t allocator_size, typename SharedStateHandle, @@ -77,8 +80,6 @@ namespace snmalloc { auto& key = entropy.get_free_list_key(); - // Return all the free lists to the allocator. - // Used during thread teardown for (size_t i = 0; i < NUM_SIZECLASSES; i++) { // TODO could optimise this, to return the whole list in one append diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index f467cfc7b..e02c2c884 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -71,13 +71,15 @@ namespace snmalloc return bk == front; } + /** + * Pushes a list of messages to the queue. Each message from first to + * last should be linked together through their next pointers. + */ void enqueue( CapPtr first, CapPtr last, const FreeListKey& key) { - // Pushes a list of messages to the queue. Each message from first to - // last should be linked together through their next pointers. invariant(); last->atomic_store_null(key); @@ -93,9 +95,11 @@ namespace snmalloc return front; } + /** + * Returns the front message, or null if not possible to return a message. + */ std::pair, bool> dequeue(const FreeListKey& key) { - // Returns the front message, or null if not possible to return a message. invariant(); CapPtr first = front; CapPtr next = first->atomic_read_next(key); From b57390663e991daa6f1329f2d52c2f286116a9b0 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Wed, 13 Oct 2021 15:29:10 +0100 Subject: [PATCH 089/302] address_cast SNMALLOC_FAST_PATH --- src/ds/address.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ds/address.h b/src/ds/address.h index 81739ac45..ae1c553d1 100644 --- a/src/ds/address.h +++ b/src/ds/address.h @@ -63,7 +63,7 @@ namespace snmalloc * Cast from a pointer type to an address. */ template - inline address_t address_cast(T* ptr) + inline SNMALLOC_FAST_PATH address_t address_cast(T* ptr) { return reinterpret_cast(ptr); } @@ -77,7 +77,7 @@ namespace snmalloc */ template - inline address_t address_cast(CapPtr a) + inline SNMALLOC_FAST_PATH address_t address_cast(CapPtr a) { return address_cast(a.unsafe_ptr()); } From 906589318126acd285f60d82e7cca035a9d522b9 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Mon, 19 Jul 2021 09:49:18 +0100 Subject: [PATCH 090/302] Overhaul CapPtr * Switch to a multidimensional taxonomy. Rather than encoding the abstract bound states in a single enum, move to a more algebraic treatment. The dimensions themselves are within the snmalloc::capptr_bounds namespace so that their fairly generic names do not conflict with consumer code. Aliases for many points in the space are established outside that namespace for ease of use elsewhere. * Introduce several new namespaces: * snmalloc::capptr::dimension holds each of the dimension enums * snmalloc::capptr holds the bound<> type itself and a ConceptBound * snmalloc::capptr::bounds gives convenient specializations of bound<> * snmalloc::capptr also has aliases for CapPtr<> itself All told, rather than `CapPtr`, we now expect client code to read `capptr::Chunk` in almost all cases (and this is just an alias for the appropriate `CapPtr>` type). When the bound<>s themselves are necessary, as when calling capptr_bound, we expect that they will almost always be pronounced using an alias (e.g., `capptr::bounds::Alloc`). * Chase consequences. * Prune old taxa and aliases that are no longer in use in snmalloc2. --- src/aal/aal.h | 23 ++- src/aal/aal_concept.h | 9 +- src/backend/address_space.h | 18 +- src/backend/address_space_core.h | 35 ++-- src/backend/backend.h | 20 +- src/ds/address.h | 28 ++- src/ds/ptrwrap.h | 339 ++++++++++++++++++++----------- src/mem/corealloc.h | 26 ++- src/mem/freelist.h | 53 ++--- src/mem/localalloc.h | 8 +- src/mem/localcache.h | 6 +- src/mem/metaslab.h | 3 +- src/mem/ptrhelpers.h | 98 --------- src/mem/remoteallocator.h | 27 +-- src/mem/remotecache.h | 2 +- src/mem/slaballocator.h | 6 +- src/pal/pal.h | 46 +++-- 17 files changed, 399 insertions(+), 348 deletions(-) delete mode 100644 src/mem/ptrhelpers.h diff --git a/src/aal/aal.h b/src/aal/aal.h index 00f636236..7e2af3755 100644 --- a/src/aal/aal.h +++ b/src/aal/aal.h @@ -189,32 +189,33 @@ namespace snmalloc */ template< typename T, - enum capptr_bounds nbounds, - enum capptr_bounds obounds, + SNMALLOC_CONCEPT(capptr::ConceptBound) BOut, + SNMALLOC_CONCEPT(capptr::ConceptBound) BIn, typename U = T> - static SNMALLOC_FAST_PATH CapPtr - capptr_bound(CapPtr a, size_t size) noexcept + static SNMALLOC_FAST_PATH CapPtr + capptr_bound(CapPtr a, size_t size) noexcept { // Impose constraints on bounds annotations. - static_assert( - obounds == CBArena || obounds == CBChunkD || obounds == CBChunk || - obounds == CBChunkE); - static_assert(capptr_is_bounds_refinement()); + static_assert(BIn::spatial >= capptr::dimension::Spatial::Chunk); + static_assert(capptr_is_spatial_refinement()); UNUSED(size); - return CapPtr(a.template as_static().unsafe_ptr()); + return CapPtr(a.template as_static().unsafe_capptr); } /** * For architectures which do not enforce StrictProvenance, there's nothing * to be done, so just return the pointer unmodified with new annotation. */ - template + template< + typename T, + SNMALLOC_CONCEPT(capptr::ConceptBound) BOut, + SNMALLOC_CONCEPT(capptr::ConceptBound) BIn> static SNMALLOC_FAST_PATH CapPtr capptr_rebound(CapPtr a, CapPtr r) noexcept { UNUSED(a); - return CapPtr(r.unsafe_ptr()); + return CapPtr(r.unsafe_capptr); } }; } // namespace snmalloc diff --git a/src/aal/aal_concept.h b/src/aal/aal_concept.h index 180e8930c..2a14a94b4 100644 --- a/src/aal/aal_concept.h +++ b/src/aal/aal_concept.h @@ -43,21 +43,22 @@ namespace snmalloc template concept ConceptAAL_capptr_methods = - requires(CapPtr auth, CapPtr ret, size_t sz) + requires(capptr::Chunk auth, capptr::AllocFull ret, size_t sz) { /** * Produce a pointer with reduced authority from a more privilged pointer. * The resulting pointer will have base at auth's address and length of * exactly sz. auth+sz must not exceed auth's limit. */ - { AAL::template capptr_bound(auth, sz) } noexcept - -> ConceptSame>; + { AAL::template capptr_bound(auth, sz) } + noexcept + -> ConceptSame>; /** * Construct a copy of auth with its target set to that of ret. */ { AAL::capptr_rebound(auth, ret) } noexcept - -> ConceptSame>; + -> ConceptSame>; }; template diff --git a/src/backend/address_space.h b/src/backend/address_space.h index ada529590..e197e7709 100644 --- a/src/backend/address_space.h +++ b/src/backend/address_space.h @@ -43,7 +43,7 @@ namespace snmalloc * arena_map for use in subsequent amplification. */ template - CapPtr + capptr::Chunk reserve(typename Pagemap::LocalState* local_state, size_t size) { #ifdef SNMALLOC_TRACING @@ -62,21 +62,21 @@ namespace snmalloc { if (size >= PAL::minimum_alloc_size) { - auto base = CapPtr( - PAL::template reserve_aligned(size)); + auto base = + capptr::Chunk(PAL::template reserve_aligned(size)); Pagemap::register_range(local_state, address_cast(base), size); return base; } } - CapPtr res; + capptr::Chunk res; { FlagLock lock(spin_lock); res = core.template reserve(local_state, size); if (res == nullptr) { // Allocation failed ask OS for more memory - CapPtr block = nullptr; + capptr::Chunk block = nullptr; size_t block_size = 0; if constexpr (pal_supports) { @@ -92,7 +92,7 @@ namespace snmalloc // It's a bit of a lie to convert without applying bounds, but the // platform will have bounded block for us and it's better that // the rest of our internals expect CBChunk bounds. - block = CapPtr(block_raw); + block = capptr::Chunk(block_raw); } else if constexpr (!pal_supports) { @@ -110,7 +110,7 @@ namespace snmalloc size_request >= needed_size; size_request = size_request / 2) { - block = CapPtr(PAL::reserve(size_request)); + block = capptr::Chunk(PAL::reserve(size_request)); if (block != nullptr) { block_size = size_request; @@ -158,7 +158,7 @@ namespace snmalloc * used, by smaller objects. */ template - CapPtr reserve_with_left_over( + capptr::Chunk reserve_with_left_over( typename Pagemap::LocalState* local_state, size_t size) { SNMALLOC_ASSERT(size >= sizeof(void*)); @@ -198,7 +198,7 @@ namespace snmalloc template void add_range( typename Pagemap::LocalState* local_state, - CapPtr base, + capptr::Chunk base, size_t length) { FlagLock lock(spin_lock); diff --git a/src/backend/address_space_core.h b/src/backend/address_space_core.h index 0aa5872aa..d3088c0fe 100644 --- a/src/backend/address_space_core.h +++ b/src/backend/address_space_core.h @@ -25,7 +25,7 @@ namespace snmalloc { struct FreeChunk { - CapPtr next; + capptr::Chunk next; }; /** @@ -43,12 +43,12 @@ namespace snmalloc * bits::BITS is used for simplicity, we do not use below the pointer size, * and large entries will be unlikely to be supported by the platform. */ - std::array, bits::BITS> ranges = {}; + std::array, bits::BITS> ranges = {}; /** * Checks a block satisfies its invariant. */ - inline void check_block(CapPtr base, size_t align_bits) + inline void check_block(capptr::Chunk base, size_t align_bits) { SNMALLOC_ASSERT( address_cast(base) == @@ -72,8 +72,8 @@ namespace snmalloc void set_next( typename Pagemap::LocalState* local_state, size_t align_bits, - CapPtr base, - CapPtr next) + capptr::Chunk base, + capptr::Chunk next) { if (align_bits >= MIN_CHUNK_BITS) { @@ -102,16 +102,16 @@ namespace snmalloc * particular size. */ template - CapPtr get_next( + capptr::Chunk get_next( typename Pagemap::LocalState* local_state, size_t align_bits, - CapPtr base) + capptr::Chunk base) { if (align_bits >= MIN_CHUNK_BITS) { const MetaEntry& t = Pagemap::template get_metaentry( local_state, address_cast(base)); - return CapPtr( + return capptr::Chunk( reinterpret_cast(t.get_metaslab())); } @@ -127,7 +127,7 @@ namespace snmalloc void add_block( typename Pagemap::LocalState* local_state, size_t align_bits, - CapPtr base) + capptr::Chunk base) { check_block(base, align_bits); SNMALLOC_ASSERT(align_bits < 64); @@ -143,10 +143,10 @@ namespace snmalloc template< SNMALLOC_CONCEPT(ConceptPAL) PAL, SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> - CapPtr + capptr::Chunk remove_block(typename Pagemap::LocalState* local_state, size_t align_bits) { - CapPtr first = ranges[align_bits]; + capptr::Chunk first = ranges[align_bits]; if (first == nullptr) { if (align_bits == (bits::BITS - 1)) @@ -156,7 +156,7 @@ namespace snmalloc } // Look for larger block and split up recursively - CapPtr bigger = + capptr::Chunk bigger = remove_block(local_state, align_bits + 1); if (bigger != nullptr) { @@ -174,7 +174,8 @@ namespace snmalloc add_block( local_state, align_bits, - Aal::capptr_bound(left_over, left_over_size)); + Aal::capptr_bound( + left_over, left_over_size)); check_block(left_over.as_static(), align_bits); } check_block(bigger.as_static(), align_bits + 1); @@ -196,7 +197,7 @@ namespace snmalloc SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> void add_range( typename Pagemap::LocalState* local_state, - CapPtr base, + capptr::Chunk base, size_t length) { // For start and end that are not chunk sized, we need to @@ -233,7 +234,7 @@ namespace snmalloc * Commit a block of memory */ template - void commit_block(CapPtr base, size_t size) + void commit_block(capptr::Chunk base, size_t size) { // Rounding required for sub-page allocations. auto page_start = pointer_align_down(base); @@ -257,7 +258,7 @@ namespace snmalloc template< SNMALLOC_CONCEPT(ConceptPAL) PAL, SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> - CapPtr + capptr::Chunk reserve(typename Pagemap::LocalState* local_state, size_t size) { #ifdef SNMALLOC_TRACING @@ -281,7 +282,7 @@ namespace snmalloc template< SNMALLOC_CONCEPT(ConceptPAL) PAL, SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> - CapPtr reserve_with_left_over( + capptr::Chunk reserve_with_left_over( typename Pagemap::LocalState* local_state, size_t size) { SNMALLOC_ASSERT(size >= sizeof(void*)); diff --git a/src/backend/backend.h b/src/backend/backend.h index 8f36b7e2f..daf036393 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -39,7 +39,7 @@ namespace snmalloc * Backend allocator may use guard pages and separate area of * address space to protect this from corruption. */ - static CapPtr alloc_meta_data( + static capptr::Chunk alloc_meta_data( AddressSpaceManager& global, LocalState* local_state, size_t size) { return reserve(global, local_state, size); @@ -54,7 +54,7 @@ namespace snmalloc * (remote, sizeclass, metaslab) * where metaslab, is the second element of the pair return. */ - static std::pair, Metaslab*> alloc_chunk( + static std::pair, Metaslab*> alloc_chunk( AddressSpaceManager& global, LocalState* local_state, size_t size, @@ -70,7 +70,7 @@ namespace snmalloc if (meta == nullptr) return {nullptr, nullptr}; - CapPtr p = reserve(global, local_state, size); + capptr::Chunk p = reserve(global, local_state, size); #ifdef SNMALLOC_TRACING std::cout << "Alloc chunk: " << p.unsafe_ptr() << " (" << size << ")" @@ -98,7 +98,7 @@ namespace snmalloc * space managers. */ template - static CapPtr reserve( + static capptr::Chunk reserve( AddressSpaceManager& global, LocalState* local_state, size_t size) { #ifdef SNMALLOC_CHECK_CLIENT @@ -108,7 +108,7 @@ namespace snmalloc constexpr auto MAX_CACHED_SIZE = LOCAL_CACHE_BLOCK; #endif - CapPtr p; + capptr::Chunk p; if ((local_state != nullptr) && (size <= MAX_CACHED_SIZE)) { #ifdef SNMALLOC_CHECK_CLIENT @@ -182,8 +182,8 @@ namespace snmalloc * the range [base, base+full_size]. The first and last slot are not used * so that the edges can be used for guard pages. */ - static CapPtr - sub_range(CapPtr base, size_t full_size, size_t sub_size) + static capptr::Chunk + sub_range(capptr::Chunk base, size_t full_size, size_t sub_size) { SNMALLOC_ASSERT(bits::is_pow2(full_size)); SNMALLOC_ASSERT(bits::is_pow2(sub_size)); @@ -318,7 +318,7 @@ namespace snmalloc auto [heap_base, heap_length] = Pagemap::concretePagemap.init(base, length); address_space.template add_range( - local_state, CapPtr(heap_base), heap_length); + local_state, capptr::Chunk(heap_base), heap_length); } /** @@ -332,7 +332,7 @@ namespace snmalloc * places or with different policies. */ template - static CapPtr + static capptr::Chunk alloc_meta_data(LocalState* local_state, size_t size) { return AddressSpaceAllocator::alloc_meta_data( @@ -348,7 +348,7 @@ namespace snmalloc * (remote, sizeclass, metaslab) * where metaslab, is the second element of the pair return. */ - static std::pair, Metaslab*> alloc_chunk( + static std::pair, Metaslab*> alloc_chunk( LocalState* local_state, size_t size, RemoteAllocator* remote, diff --git a/src/ds/address.h b/src/ds/address.h index ae1c553d1..90b0bb456 100644 --- a/src/ds/address.h +++ b/src/ds/address.h @@ -35,7 +35,7 @@ namespace snmalloc reinterpret_cast(base) + static_cast(diff)); } - template + template inline CapPtr pointer_offset(CapPtr base, size_t diff) { @@ -52,7 +52,7 @@ namespace snmalloc return reinterpret_cast(reinterpret_cast(base) + diff); } - template + template inline CapPtr pointer_offset_signed(CapPtr base, ptrdiff_t diff) { @@ -76,7 +76,7 @@ namespace snmalloc * capptr_bound. */ - template + template inline SNMALLOC_FAST_PATH address_t address_cast(CapPtr a) { return address_cast(a.unsafe_ptr()); @@ -132,7 +132,10 @@ namespace snmalloc pointer_align_down(reinterpret_cast(p))); } - template + template< + size_t alignment, + typename T, + SNMALLOC_CONCEPT(capptr::ConceptBound) bounds> inline CapPtr pointer_align_down(CapPtr p) { return CapPtr(pointer_align_down(p.unsafe_ptr())); @@ -166,7 +169,10 @@ namespace snmalloc } } - template + template< + size_t alignment, + typename T = void, + SNMALLOC_CONCEPT(capptr::ConceptBound) bounds> inline CapPtr pointer_align_up(CapPtr p) { return CapPtr(pointer_align_up(p.unsafe_ptr())); @@ -195,7 +201,7 @@ namespace snmalloc #endif } - template + template inline CapPtr pointer_align_down(CapPtr p, size_t alignment) { @@ -219,7 +225,7 @@ namespace snmalloc #endif } - template + template inline CapPtr pointer_align_up(CapPtr p, size_t alignment) { @@ -241,8 +247,8 @@ namespace snmalloc template< typename T = void, typename U = void, - enum capptr_bounds Tbounds, - enum capptr_bounds Ubounds> + SNMALLOC_CONCEPT(capptr::ConceptBound) Tbounds, + SNMALLOC_CONCEPT(capptr::ConceptBound) Ubounds> inline size_t pointer_diff(CapPtr base, CapPtr cursor) { return pointer_diff(base.unsafe_ptr(), cursor.unsafe_ptr()); @@ -261,8 +267,8 @@ namespace snmalloc template< typename T = void, typename U = void, - enum capptr_bounds Tbounds, - enum capptr_bounds Ubounds> + SNMALLOC_CONCEPT(capptr::ConceptBound) Tbounds, + SNMALLOC_CONCEPT(capptr::ConceptBound) Ubounds> inline ptrdiff_t pointer_diff_signed(CapPtr base, CapPtr cursor) { diff --git a/src/ds/ptrwrap.h b/src/ds/ptrwrap.h index ff073e942..11c9926a7 100644 --- a/src/ds/ptrwrap.h +++ b/src/ds/ptrwrap.h @@ -1,6 +1,7 @@ #pragma once -#include "defines.h" +#include "../ds/concept.h" +#include "../ds/defines.h" #include @@ -21,78 +22,150 @@ namespace snmalloc /** * Summaries of StrictProvenance metadata. We abstract away the particular * size and any offset into the bounds. - * - * CBArena is as powerful as our pointers get: they're results from mmap(), - * and so confer as much authority as the kernel has given us. - * - * CBChunk is restricted to either a single chunk (SUPERSLAB_SIZE) or perhaps - * to several if we've requesed a large allocation (see capptr_chunk_is_alloc - * and its uses). - * - * CBChunkD is curious: we often use CBArena-bounded pointers to derive - * pointers to Allocslab metadata, and on most fast paths these pointers end - * up being ephemeral. As such, on NDEBUG builds, we elide the capptr_bounds - * that would bound these to chunks and instead just unsafely inherit the - * CBArena bounds. The use of CBChunkD thus helps to ensure that we - * eventually do invoke capptr_bounds when these pointers end up being longer - * lived! - * - * *E forms are "exported" and have had platform constraints applied. That - * means, for example, on CheriBSD, that they have had their VMMAP permission - * stripped. - * - * Yes, I wish the start-of-comment characters were aligned below as well. - * I blame clang format. */ - enum capptr_bounds + + namespace capptr { - /* Spatial Notes */ - CBArena, /* Arena */ - CBChunkD, /* Arena Chunk-bounded in debug; internal use only! */ - CBChunk, /* Chunk */ - CBChunkE, /* Chunk (+ platform constraints) */ - CBAlloc, /* Alloc */ - CBAllocE /* Alloc (+ platform constraints) */ - }; + namespace dimension + { + /* + * Describe the spatial extent (intended to be) authorized by a pointer. + * + * Bounds dimensions are sorted so that < reflects authority. + */ + enum class Spatial + { + /** + * Bounded to a particular allocation (which might be Large!) + */ + Alloc, + /** + * Bounded to one or more particular chunk granules + */ + Chunk, + }; + + /** + * On some platforms (e.g., CHERI), pointers can be checked to see whether + * they authorize control of the address space. See the PAL's + * capptr_export(). + */ + enum class AddressSpaceControl + { + /** + * All intended control constraints have been applied. For example, on + * CheriBSD, the VMMAP permission has been stripped and so this CapPtr<> + * cannot authorize manipulation of the address space itself, though it + * continues to authorize loads and stores. + */ + User, + /** + * No control constraints have been applied. On CheriBSD, specifically, + * this grants control of the address space (via mmap and friends) and + * in Cornucopia exempts the pointer from revocation (as long as the + * mapping remains in place, but snmalloc does not presently tear down + * its own mappings.) + */ + Full + }; + + } // namespace dimension + + /** + * The aggregate type of a bound: a Cartesian product of the individual + * dimensions, above. + */ + template + struct bound + { + static constexpr enum dimension::Spatial spatial = S; + static constexpr enum dimension::AddressSpaceControl + address_space_control = AS; + + /** + * Set just the spatial component of the bounds + */ + template + using with_spatial = bound; + + /** + * Set just the address space control component of the bounds + */ + template + using with_address_space_control = bound; + }; + + // clang-format off +#ifdef __cpp_concepts + /* + * This is spelled a little differently from our other concepts because GCC + * treats "{ T::spatial }" as generating a reference and then complains that + * it isn't "ConceptSame", though clang is perfectly happy + * with that spelling. Both seem happy with this formulation. + */ + template + concept ConceptBound = + ConceptSame && + ConceptSame; +#endif + // clang-format on + + /* + * Several combinations are used often enough that we give convenient + * aliases for them. + */ + namespace bounds + { + /** + * Internal access to a Chunk of memory. These flow between the ASM and + * the slab allocators, for example. + */ + using Chunk = + bound; + + /** + * User access to an entire Chunk. Used as an ephemeral state when + * returning a large allocation. See capptr_chunk_is_alloc. + */ + using ChunkUser = + Chunk::with_address_space_control; + + /** + * Internal access to just one allocation (usually, within a slab). + */ + using AllocFull = Chunk::with_spatial; + + /** + * User access to just one allocation (usually, within a slab). + */ + using Alloc = AllocFull::with_address_space_control< + dimension::AddressSpaceControl::User>; + } // namespace bounds + } // namespace capptr /** - * Compute the "exported" variant of a capptr_bounds annotation. This is - * used by the PAL's capptr_export function to compute its return value's - * annotation. + * Determine whether BI is a spatial refinement of BO. + * Chunk and ChunkD are considered eqivalent here. */ - template - SNMALLOC_CONSTEVAL capptr_bounds capptr_export_type() + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BI, + SNMALLOC_CONCEPT(capptr::ConceptBound) BO> + SNMALLOC_CONSTEVAL bool capptr_is_spatial_refinement() { - static_assert( - (B == CBChunk) || (B == CBAlloc), "capptr_export_type of bad type"); - - switch (B) + if (BI::address_space_control != BO::address_space_control) { - case CBChunk: - return CBChunkE; - case CBAlloc: - return CBAllocE; + return false; } - } - template - SNMALLOC_CONSTEVAL bool capptr_is_bounds_refinement() - { - switch (BI) + switch (BI::spatial) { - case CBAllocE: - return BO == CBAllocE; - case CBAlloc: - return BO == CBAlloc; - case CBChunkE: - return BO == CBAllocE || BO == CBChunkE; - case CBChunk: - return BO == CBAlloc || BO == CBChunk || BO == CBChunkD; - case CBChunkD: - return BO == CBAlloc || BO == CBChunk || BO == CBChunkD; - case CBArena: - return BO == CBAlloc || BO == CBChunk || BO == CBChunkD || - BO == CBArena; + using namespace capptr::dimension; + case Spatial::Chunk: + return true; + + case Spatial::Alloc: + return BO::spatial == Spatial::Alloc; } } @@ -100,18 +173,19 @@ namespace snmalloc * A pointer annotated with a "phantom type parameter" carrying a static * summary of its StrictProvenance metadata. */ - template - class CapPtr + template + struct CapPtr { - uintptr_t unsafe_capptr; + T* unsafe_capptr; - public: /** * nullptr is implicitly constructable at any bounds type */ - constexpr CapPtr(const std::nullptr_t) : unsafe_capptr(0) {} + constexpr SNMALLOC_FAST_PATH CapPtr(const std::nullptr_t n) + : unsafe_capptr(n) + {} - constexpr CapPtr() : CapPtr(nullptr){}; + constexpr SNMALLOC_FAST_PATH CapPtr() : CapPtr(nullptr) {} /** * all other constructions must be explicit @@ -127,23 +201,21 @@ namespace snmalloc # pragma warning(push) # pragma warning(disable : 4702) #endif - constexpr explicit CapPtr(uintptr_t p) : unsafe_capptr(p) {} + constexpr explicit SNMALLOC_FAST_PATH CapPtr(T* p) : unsafe_capptr(p) {} #ifdef _MSC_VER # pragma warning(pop) #endif - explicit CapPtr(T* p) : unsafe_capptr(reinterpret_cast(p)) {} - /** * Allow static_cast<>-s that preserve bounds but vary the target type. */ template - SNMALLOC_FAST_PATH CapPtr as_static() + [[nodiscard]] SNMALLOC_FAST_PATH CapPtr as_static() const { - return CapPtr(this->unsafe_capptr); + return CapPtr(static_cast(this->unsafe_capptr)); } - SNMALLOC_FAST_PATH CapPtr as_void() + [[nodiscard]] SNMALLOC_FAST_PATH CapPtr as_void() const { return this->as_static(); } @@ -152,9 +224,9 @@ namespace snmalloc * A more aggressive bounds-preserving cast, using reinterpret_cast */ template - SNMALLOC_FAST_PATH CapPtr as_reinterpret() + [[nodiscard]] SNMALLOC_FAST_PATH CapPtr as_reinterpret() const { - return CapPtr(this->unsafe_capptr); + return CapPtr(reinterpret_cast(this->unsafe_capptr)); } SNMALLOC_FAST_PATH bool operator==(const CapPtr& rhs) const @@ -172,60 +244,82 @@ namespace snmalloc return this->unsafe_capptr < rhs.unsafe_capptr; } - [[nodiscard]] SNMALLOC_FAST_PATH T* unsafe_ptr() const + SNMALLOC_FAST_PATH T* operator->() const { - return reinterpret_cast(this->unsafe_capptr); + /* + * Alloc-bounded, platform-constrained pointers are associated with + * objects coming from or going to the client; we should be doing nothing + * with them. + */ + static_assert( + (bounds::spatial != capptr::dimension::Spatial::Alloc) || + (bounds::address_space_control != + capptr::dimension::AddressSpaceControl::User)); + return this->unsafe_capptr; } - [[nodiscard]] SNMALLOC_FAST_PATH uintptr_t unsafe_uintptr() const + [[nodiscard]] SNMALLOC_FAST_PATH T* unsafe_ptr() const { return this->unsafe_capptr; } - SNMALLOC_FAST_PATH T* operator->() const + [[nodiscard]] SNMALLOC_FAST_PATH uintptr_t unsafe_uintptr() const { - /* - * CBAllocE bounds are associated with objects coming from or going to the - * client; we should be doing nothing with them. - */ - static_assert(bounds != CBAllocE); - return unsafe_ptr(); + return reinterpret_cast(this->unsafe_capptr); } }; - static_assert(sizeof(CapPtr) == sizeof(void*)); - static_assert(alignof(CapPtr) == alignof(void*)); + namespace capptr + { + /* + * Aliases for CapPtr<> types with particular bounds. + */ - template - using CapPtrCBArena = CapPtr; + template + using Chunk = CapPtr; - template - using CapPtrCBChunk = CapPtr; + template + using ChunkUser = CapPtr; - template - using CapPtrCBChunkE = CapPtr; + template + using AllocFull = CapPtr; - template - using CapPtrCBAlloc = CapPtr; + template + using Alloc = CapPtr; + + } // namespace capptr + + static_assert(sizeof(capptr::Chunk) == sizeof(void*)); + static_assert(alignof(capptr::Chunk) == alignof(void*)); /** * Sometimes (with large allocations) we really mean the entire chunk (or even * several chunks) to be the allocation. */ template - inline SNMALLOC_FAST_PATH CapPtr - capptr_chunk_is_alloc(CapPtr p) + inline SNMALLOC_FAST_PATH capptr::Alloc + capptr_chunk_is_alloc(capptr::ChunkUser p) { - return CapPtr(p.unsafe_capptr); + return capptr::Alloc(p.unsafe_capptr); } /** * With all the bounds and constraints in place, it's safe to extract a void - * pointer (to reveal to the client). + * pointer (to reveal to the client). Roughly dual to capptr_from_client(). + */ + inline SNMALLOC_FAST_PATH void* capptr_reveal(capptr::Alloc p) + { + return p.unsafe_capptr; + } + + /** + * Given a void* from the client, it's fine to call it Alloc. Roughly + * dual to capptr_reveal(). */ - inline SNMALLOC_FAST_PATH void* capptr_reveal(CapPtr p) + static inline SNMALLOC_FAST_PATH capptr::Alloc + capptr_from_client(void* p) { - return p.unsafe_ptr(); + return capptr::Alloc(p); } /** @@ -237,7 +331,7 @@ namespace snmalloc * annotations around an un-annotated std::atomic, to appease C++, yet * will expose or consume only CapPtr with the same bounds annotation. */ - template + template struct AtomicCapPtr { std::atomic unsafe_capptr; @@ -245,12 +339,16 @@ namespace snmalloc /** * nullptr is constructable at any bounds type */ - constexpr AtomicCapPtr(const std::nullptr_t n) : unsafe_capptr(n) {} + constexpr SNMALLOC_FAST_PATH AtomicCapPtr(const std::nullptr_t n) + : unsafe_capptr(n) + {} /** * Interconversion with CapPtr */ - AtomicCapPtr(CapPtr p) : unsafe_capptr(p.unsafe_ptr()) {} + constexpr SNMALLOC_FAST_PATH AtomicCapPtr(CapPtr p) + : unsafe_capptr(p.unsafe_capptr) + {} operator CapPtr() const noexcept { @@ -260,7 +358,7 @@ namespace snmalloc // Our copy-assignment operator follows std::atomic and returns a copy of // the RHS. Clang finds this surprising; we suppress the warning. // NOLINTNEXTLINE(misc-unconventional-assign-operator) - CapPtr operator=(CapPtr p) noexcept + SNMALLOC_FAST_PATH CapPtr operator=(CapPtr p) noexcept { this->store(p); return p; @@ -276,7 +374,7 @@ namespace snmalloc CapPtr desired, std::memory_order order = std::memory_order_seq_cst) noexcept { - this->unsafe_capptr.store(desired.unsafe_ptr(), order); + this->unsafe_capptr.store(desired.unsafe_capptr, order); } SNMALLOC_FAST_PATH CapPtr exchange( @@ -284,7 +382,7 @@ namespace snmalloc std::memory_order order = std::memory_order_seq_cst) noexcept { return CapPtr( - this->unsafe_capptr.exchange(desired.unsafe_ptr(), order)); + this->unsafe_capptr.exchange(desired.unsafe_capptr, order)); } SNMALLOC_FAST_PATH bool operator==(const AtomicCapPtr& rhs) const @@ -303,13 +401,24 @@ namespace snmalloc } }; - template - using AtomicCapPtrCBArena = AtomicCapPtr; + namespace capptr + { + /* + * Aliases for AtomicCapPtr<> types with particular bounds. + */ - template - using AtomicCapPtrCBChunk = AtomicCapPtr; + template + using AtomicChunk = AtomicCapPtr; - template - using AtomicCapPtrCBAlloc = AtomicCapPtr; + template + using AtomicChunkUser = AtomicCapPtr; + + template + using AtomicAllocFull = AtomicCapPtr; + + template + using AtomicAlloc = AtomicCapPtr; + + } // namespace capptr } // namespace snmalloc diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index cb3869d8d..881ce1008 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -161,7 +161,7 @@ namespace snmalloc { // Manufacture an allocation to prime the queue // Using an actual allocation removes a conditional from a critical path. - auto dummy = CapPtr(small_alloc_one(MIN_ALLOC_SIZE)) + auto dummy = capptr::AllocFull(small_alloc_one(MIN_ALLOC_SIZE)) .template as_static(); if (dummy == nullptr) { @@ -187,7 +187,7 @@ namespace snmalloc } static SNMALLOC_FAST_PATH void alloc_new_list( - CapPtr& bumpptr, + capptr::Chunk& bumpptr, Metaslab* meta, size_t rsize, size_t slab_size, @@ -203,7 +203,7 @@ namespace snmalloc // Structure to represent the temporary list elements struct PreAllocObject { - CapPtr next; + capptr::AllocFull next; }; // The following code implements Sattolo's algorithm for generating // random cyclic permutations. This implementation is in the opposite @@ -213,9 +213,10 @@ namespace snmalloc // Note the wide bounds on curr relative to each of the ->next fields; // curr is not persisted once the list is built. - CapPtr curr = + capptr::Chunk curr = pointer_offset(bumpptr, 0).template as_static(); - curr->next = Aal::capptr_bound(curr, rsize); + curr->next = Aal::capptr_bound( + curr, rsize); uint16_t count = 1; for (curr = @@ -229,12 +230,13 @@ namespace snmalloc pointer_offset(bumpptr, insert_index * rsize) .template as_static() ->next, - Aal::capptr_bound(curr, rsize)); + Aal::capptr_bound( + curr, rsize)); count++; } // Pick entry into space, and then build linked list by traversing cycle - // to the start. Use ->next to jump from CBArena to CBAlloc. + // to the start. Use ->next to jump from Chunk to Alloc. auto start_index = entropy.sample(count); auto start_ptr = pointer_offset(bumpptr, start_index * rsize) .template as_static() @@ -249,7 +251,9 @@ namespace snmalloc auto p = bumpptr; do { - b.add(Aal::capptr_bound(p, rsize), key); + b.add( + Aal::capptr_bound(p, rsize), + key); p = pointer_offset(p, rsize); } while (p < slab_end); #endif @@ -299,7 +303,7 @@ namespace snmalloc auto start_of_slab = pointer_align_down( p, snmalloc::sizeclass_to_slab_size(sizeclass)); // TODO Add bounds correctly here - chunk_record->chunk = CapPtr(start_of_slab); + chunk_record->chunk = capptr::Chunk(start_of_slab); #ifdef SNMALLOC_TRACING std::cout << "Slab " << start_of_slab << " is unused, Object sizeclass " @@ -422,7 +426,7 @@ namespace snmalloc * need_post will be set to true, if capacity is exceeded. */ void handle_dealloc_remote( - const MetaEntry& entry, CapPtr p, bool& need_post) + const MetaEntry& entry, capptr::AllocFull p, bool& need_post) { // TODO this needs to not double count stats // TODO this needs to not double revoke if using MTE @@ -576,7 +580,7 @@ namespace snmalloc Metaslab::is_start_of_object(entry.get_sizeclass(), address_cast(p)), "Not deallocating start of an object"); - auto cp = CapPtr(reinterpret_cast(p)); + auto cp = capptr::AllocFull(reinterpret_cast(p)); auto& key = entropy.get_free_list_key(); diff --git a/src/mem/freelist.h b/src/mem/freelist.h index eab71eb44..09db13804 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -66,9 +66,9 @@ namespace snmalloc { union { - CapPtr next_object; + capptr::AllocFull next_object; // TODO: Should really use C++20 atomic_ref rather than a union. - AtomicCapPtr atomic_next_object; + capptr::AtomicAllocFull atomic_next_object; }; #ifdef SNMALLOC_CHECK_CLIENT // Encoded representation of a back pointer. @@ -78,7 +78,7 @@ namespace snmalloc #endif public: - static CapPtr make(CapPtr p) + static capptr::AllocFull make(capptr::AllocFull p) { return p.template as_static(); } @@ -86,8 +86,10 @@ namespace snmalloc /** * Encode next */ - inline static CapPtr encode_next( - address_t curr, CapPtr next, const FreeListKey& key) + inline static capptr::AllocFull encode_next( + address_t curr, + capptr::AllocFull next, + const FreeListKey& key) { // Note we can consider other encoding schemes here. // * XORing curr and next. This doesn't require any key material @@ -98,7 +100,8 @@ namespace snmalloc if constexpr (CHECK_CLIENT && !aal_supports) { - return CapPtr(address_cast(next) ^ key.key_next); + return capptr::AllocFull(reinterpret_cast( + reinterpret_cast(next.unsafe_ptr()) ^ key.key_next)); } else { @@ -115,9 +118,9 @@ namespace snmalloc * optimization for repeated snoc operations (in which * next->next_object is nullptr). */ - static CapPtr* store_next( - CapPtr* curr, - CapPtr next, + static capptr::AllocFull* store_next( + capptr::AllocFull* curr, + capptr::AllocFull next, const FreeListKey& key) { #ifdef SNMALLOC_CHECK_CLIENT @@ -131,7 +134,7 @@ namespace snmalloc } static void - store_null(CapPtr* curr, const FreeListKey& key) + store_null(capptr::AllocFull* curr, const FreeListKey& key) { *curr = encode_next(address_cast(curr), nullptr, key); } @@ -141,8 +144,8 @@ namespace snmalloc * * Uses the atomic view of next, so can be used in the message queues. */ - void - atomic_store_next(CapPtr next, const FreeListKey& key) + void atomic_store_next( + capptr::AllocFull next, const FreeListKey& key) { #ifdef SNMALLOC_CHECK_CLIENT next->prev_encoded = @@ -164,7 +167,7 @@ namespace snmalloc std::memory_order_relaxed); } - CapPtr atomic_read_next(const FreeListKey& key) + capptr::AllocFull atomic_read_next(const FreeListKey& key) { auto n = encode_next( address_cast(&next_object), @@ -194,7 +197,7 @@ namespace snmalloc /** * Read the next pointer */ - CapPtr read_next(const FreeListKey& key) + capptr::AllocFull read_next(const FreeListKey& key) { return encode_next(address_cast(&next_object), next_object, key); } @@ -211,14 +214,14 @@ namespace snmalloc */ class FreeListIter { - CapPtr curr{nullptr}; + capptr::AllocFull curr{nullptr}; #ifdef SNMALLOC_CHECK_CLIENT address_t prev{0}; #endif public: constexpr FreeListIter( - CapPtr head, address_t prev_value) + capptr::AllocFull head, address_t prev_value) : curr(head) { #ifdef SNMALLOC_CHECK_CLIENT @@ -240,7 +243,7 @@ namespace snmalloc /** * Returns current head without affecting the iterator. */ - CapPtr peek() + capptr::AllocFull peek() { return curr; } @@ -248,7 +251,7 @@ namespace snmalloc /** * Moves the iterator on, and returns the current value. */ - CapPtr take(const FreeListKey& key) + capptr::AllocFull take(const FreeListKey& key) { auto c = curr; auto next = curr->read_next(key); @@ -289,11 +292,11 @@ namespace snmalloc static constexpr size_t LENGTH = RANDOM ? 2 : 1; // Pointer to the first element. - std::array, LENGTH> head; + std::array, LENGTH> head; // Pointer to the reference to the last element. // In the empty case end[i] == &head[i] // This enables branch free enqueuing. - std::array*, LENGTH> end{nullptr}; + std::array*, LENGTH> end{nullptr}; std::array length{}; @@ -321,7 +324,7 @@ namespace snmalloc * Adds an element to the builder */ void add( - CapPtr n, + capptr::AllocFull n, const FreeListKey& key, LocalEntropy& entropy) { @@ -348,7 +351,7 @@ namespace snmalloc */ template std::enable_if_t - add(CapPtr n, const FreeListKey& key) + add(capptr::AllocFull n, const FreeListKey& key) { static_assert(RANDOM_ == RANDOM, "Don't set template parameter"); end[0] = FreeObject::store_next(end[0], n, key); @@ -372,7 +375,7 @@ namespace snmalloc * and is thus subject to encoding if the next_object pointers * encoded. */ - CapPtr + capptr::AllocFull read_head(uint32_t index, const FreeListKey& key) { return FreeObject::encode_next( @@ -444,7 +447,7 @@ namespace snmalloc template std::enable_if_t< !RANDOM_, - std::pair, CapPtr>> + std::pair, capptr::AllocFull>> extract_segment(const FreeListKey& key) { static_assert(RANDOM_ == RANDOM, "Don't set SFINAE parameter!"); @@ -456,7 +459,7 @@ namespace snmalloc // to the actual object. This isn't true if the builder is // empty, but you are not allowed to call this in the empty case. auto last = - CapPtr(reinterpret_cast(end[0])); + capptr::AllocFull(reinterpret_cast(end[0])); init(); return {first, last}; } diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 761754ee5..b20b3d3a4 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -263,7 +263,9 @@ namespace snmalloc MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry( core_alloc->backend_state_ptr(), address_cast(p)); local_cache.remote_dealloc_cache.template dealloc( - entry.get_remote()->trunc_id(), CapPtr(p), key_global); + entry.get_remote()->trunc_id(), + capptr::AllocFull(p), + key_global); post_remote_cache(); return; } @@ -469,7 +471,7 @@ namespace snmalloc { local_cache.remote_dealloc_cache.template dealloc( entry.get_remote()->trunc_id(), - CapPtr(p), + capptr::AllocFull(p), key_global); # ifdef SNMALLOC_TRACING std::cout << "Remote dealloc fast" << p << " size " << alloc_size(p) @@ -501,7 +503,7 @@ namespace snmalloc # endif ChunkRecord* slab_record = reinterpret_cast(entry.get_metaslab()); - slab_record->chunk = CapPtr(p); + slab_record->chunk = capptr::Chunk(p); check_init( []( CoreAlloc* core_alloc, diff --git a/src/mem/localcache.h b/src/mem/localcache.h index eaaf6c106..a655cfbab 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -13,19 +13,19 @@ namespace snmalloc using Stats = AllocStats; inline static SNMALLOC_FAST_PATH void* - finish_alloc_no_zero(CapPtr p, sizeclass_t sizeclass) + finish_alloc_no_zero(capptr::AllocFull p, sizeclass_t sizeclass) { SNMALLOC_ASSERT(Metaslab::is_start_of_object(sizeclass, address_cast(p))); UNUSED(sizeclass); - auto r = capptr_reveal(capptr_export(p.as_void())); + auto r = capptr_reveal(capptr_to_user_address_control(p.as_void())); return r; } template inline static SNMALLOC_FAST_PATH void* - finish_alloc(CapPtr p, sizeclass_t sizeclass) + finish_alloc(capptr::AllocFull p, sizeclass_t sizeclass) { auto r = finish_alloc_no_zero(p, sizeclass); diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 66dbecbdb..a6015b2b2 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -5,7 +5,6 @@ #include "../ds/seqqueue.h" #include "../mem/remoteallocator.h" #include "freelist.h" -#include "ptrhelpers.h" #include "sizeclasstable.h" namespace snmalloc @@ -153,7 +152,7 @@ namespace snmalloc * component, but with randomisation, it may only return part of the * available objects for this metaslab. */ - static SNMALLOC_FAST_PATH std::pair, bool> + static SNMALLOC_FAST_PATH std::pair, bool> alloc_free_list( Metaslab* meta, FreeListIter& fast_free_list, diff --git a/src/mem/ptrhelpers.h b/src/mem/ptrhelpers.h deleted file mode 100644 index 8b0cee925..000000000 --- a/src/mem/ptrhelpers.h +++ /dev/null @@ -1,98 +0,0 @@ -#pragma once - -#include "../aal/aal.h" -#include "../ds/ptrwrap.h" -#include "allocconfig.h" - -namespace snmalloc -{ - /* - * At various points, we do pointer math on high-authority pointers to find - * some metadata. `capptr_bound_chunkd` and `capptr_chunk_from_chunkd` - * encapsulate the notion that the result of these accesses is left unbounded - * in non-debug builds, because most codepaths do not reveal these pointers or - * any progeny to the application. However, in some cases we have already - * (partially) bounded these high-authority pointers (to CBChunk) and wish to - * preserve this annotation (rather than always returning a CBChunkD-annotated - * pointer); `capptr_bound_chunkd_bounds` does the computation for us and is - * used in the signatures of below and in those of wrappers around them. - */ - - template - constexpr capptr_bounds capptr_bound_chunkd_bounds() - { - switch (B) - { - case CBArena: - return CBChunkD; - case CBChunkD: - return CBChunkD; - case CBChunk: - return CBChunk; - } - } - - /** - * Construct an CapPtr from an CapPtr or - * CapPtr input. For an CapPtr input, simply pass - * it through (preserving the static notion of bounds). - * - * Applies bounds on debug builds, otherwise is just sleight of hand. - * - * Requires that `p` point at a multiple of `sz` (that is, at the base of a - * highly-aligned object) to avoid representability issues. - */ - template - SNMALLOC_FAST_PATH CapPtr()> - capptr_bound_chunkd(CapPtr p, size_t sz) - { - static_assert(B == CBArena || B == CBChunkD || B == CBChunk); - SNMALLOC_ASSERT((address_cast(p) % sz) == 0); - -#ifndef NDEBUG - // On Debug builds, apply bounds if not already there - if constexpr (B == CBArena) - return Aal::capptr_bound(p, sz); - else // quiesce MSVC's warnings about unreachable code below -#endif - { - UNUSED(sz); - return CapPtr()>(p.unsafe_ptr()); - } - } - - /** - * Apply bounds that might not have been applied when constructing an - * CapPtr. That is, on non-debug builds, apply bounds; debug - * builds have already had them applied. - * - * Requires that `p` point at a multiple of `sz` (that is, at the base of a - * highly-aligned object) to avoid representability issues. - */ - template - SNMALLOC_FAST_PATH CapPtr - capptr_chunk_from_chunkd(CapPtr p, size_t sz) - { - SNMALLOC_ASSERT((address_cast(p) % sz) == 0); - -#ifndef NDEBUG - // On debug builds, CBChunkD are already bounded as if CBChunk. - UNUSED(sz); - return CapPtr(p.unsafe_ptr()); -#else - // On non-debug builds, apply bounds now, as they haven't been already. - return Aal::capptr_bound(p, sz); -#endif - } - - /** - * Very rarely, while debugging, it's both useful and acceptable to forget - * that we have applied chunk bounds to something. - */ - template - SNMALLOC_FAST_PATH CapPtr - capptr_debug_chunkd_from_chunk(CapPtr p) - { - return CapPtr(p.unsafe_ptr()); - } -} // namespace snmalloc diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index e02c2c884..f9596c796 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -35,10 +35,10 @@ namespace snmalloc // Store the message queue on a separate cacheline. It is mutable data that // is read by other threads. - alignas(CACHELINE_SIZE) AtomicCapPtr back{nullptr}; + alignas(CACHELINE_SIZE) capptr::AtomicAllocFull back{nullptr}; // Store the two ends on different cache lines as access by different // threads. - alignas(CACHELINE_SIZE) CapPtr front{nullptr}; + alignas(CACHELINE_SIZE) capptr::AllocFull front{nullptr}; constexpr RemoteAllocator() = default; @@ -48,7 +48,7 @@ namespace snmalloc SNMALLOC_ASSERT(front != nullptr); } - void init(CapPtr stub) + void init(capptr::AllocFull stub) { stub->atomic_store_null(key_global); front = stub; @@ -56,9 +56,9 @@ namespace snmalloc invariant(); } - CapPtr destroy() + capptr::AllocFull destroy() { - CapPtr fnt = front; + capptr::AllocFull fnt = front; back.store(nullptr, std::memory_order_relaxed); front = nullptr; return fnt; @@ -66,7 +66,7 @@ namespace snmalloc inline bool is_empty() { - CapPtr bk = back.load(std::memory_order_relaxed); + capptr::AllocFull bk = back.load(std::memory_order_relaxed); return bk == front; } @@ -76,21 +76,21 @@ namespace snmalloc * last should be linked together through their next pointers. */ void enqueue( - CapPtr first, - CapPtr last, + capptr::AllocFull first, + capptr::AllocFull last, const FreeListKey& key) { invariant(); last->atomic_store_null(key); // exchange needs to be a release, so nullptr in next is visible. - CapPtr prev = + capptr::AllocFull prev = back.exchange(last, std::memory_order_release); prev->atomic_store_next(first, key); } - CapPtr peek() + capptr::AllocFull peek() { return front; } @@ -98,11 +98,12 @@ namespace snmalloc /** * Returns the front message, or null if not possible to return a message. */ - std::pair, bool> dequeue(const FreeListKey& key) + std::pair, bool> + dequeue(const FreeListKey& key) { invariant(); - CapPtr first = front; - CapPtr next = first->atomic_read_next(key); + capptr::AllocFull first = front; + capptr::AllocFull next = first->atomic_read_next(key); if (next != nullptr) { diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index 4b88d23bf..0bf11ed74 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -66,7 +66,7 @@ namespace snmalloc template SNMALLOC_FAST_PATH void dealloc( RemoteAllocator::alloc_id_t target_id, - CapPtr p, + capptr::AllocFull p, const FreeListKey& key) { SNMALLOC_ASSERT(initialised); diff --git a/src/mem/slaballocator.h b/src/mem/slaballocator.h index 5c91b4420..aa4d47fcf 100644 --- a/src/mem/slaballocator.h +++ b/src/mem/slaballocator.h @@ -18,7 +18,7 @@ namespace snmalloc struct ChunkRecord { std::atomic next; - CapPtr chunk; + capptr::Chunk chunk; }; /** @@ -76,7 +76,7 @@ namespace snmalloc { public: template - static std::pair, Metaslab*> alloc_chunk( + static std::pair, Metaslab*> alloc_chunk( typename SharedStateHandle::LocalState& local_state, sizeclass_t sizeclass, sizeclass_t slab_sizeclass, // TODO sizeclass_t @@ -163,7 +163,7 @@ namespace snmalloc // Cache line align size_t size = bits::align_up(sizeof(U), 64); - CapPtr p = + capptr::Chunk p = SharedStateHandle::template alloc_meta_data(local_state, size); if (p == nullptr) diff --git a/src/pal/pal.h b/src/pal/pal.h index d95ef0f3b..a2ad11556 100644 --- a/src/pal/pal.h +++ b/src/pal/pal.h @@ -69,28 +69,46 @@ namespace snmalloc // Used to keep Superslab metadata committed. static constexpr size_t OS_PAGE_SIZE = Pal::page_size; + /** + * Compute the AddressSpaceControl::User variant of a capptr::bound + * annotation. This is used by the PAL's capptr_export function to compute + * its return value's annotation. + */ + template + using capptr_user_address_control_type = + typename B::template with_address_space_control< + capptr::dimension::AddressSpaceControl::User>; + /** * Perform platform-specific adjustment of return pointers. * * This is here, rather than in every PAL proper, merely to minimize * disruption to PALs for platforms that do not support StrictProvenance AALs. */ - template + template< + typename PAL = Pal, + typename AAL = Aal, + typename T, + SNMALLOC_CONCEPT(capptr::ConceptBound) B> static inline typename std::enable_if_t< !aal_supports, - CapPtr()>> - capptr_export(CapPtr p) + CapPtr>> + capptr_to_user_address_control(CapPtr p) { - return CapPtr()>(p.unsafe_ptr()); + return CapPtr>(p.unsafe_capptr); } - template - static inline typename std::enable_if_t< + template< + typename PAL = Pal, + typename AAL = Aal, + typename T, + SNMALLOC_CONCEPT(capptr::ConceptBound) B> + static SNMALLOC_FAST_PATH typename std::enable_if_t< aal_supports, - CapPtr()>> - capptr_export(CapPtr p) + CapPtr>> + capptr_to_user_address_control(CapPtr p) { - return PAL::capptr_export(p); + return PAL::capptr_to_user_address_control(p); } /** @@ -101,12 +119,16 @@ namespace snmalloc * disruption and avoid code bloat. This wrapper ought to compile down to * nothing if SROA is doing its job. */ - template + template< + typename PAL, + bool page_aligned = false, + typename T, + SNMALLOC_CONCEPT(capptr::ConceptBound) B> static SNMALLOC_FAST_PATH void pal_zero(CapPtr p, size_t sz) { static_assert( - !page_aligned || B == CBArena || B == CBChunkD || B == CBChunk); - PAL::template zero(p.unsafe_ptr(), sz); + !page_aligned || B::spatial >= capptr::dimension::Spatial::Chunk); + PAL::template zero(p.unsafe_capptr, sz); } static_assert( From ca70856dc186b919aaaddb084f58a9570038aa2c Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Mon, 16 Aug 2021 20:58:11 +0100 Subject: [PATCH 091/302] CapPtr: remove a stale static assert In the new world order, we will actually operate on AllocUser-bound pointers, such as slabs' free lists. --- src/ds/ptrwrap.h | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/ds/ptrwrap.h b/src/ds/ptrwrap.h index 11c9926a7..359d4e9c0 100644 --- a/src/ds/ptrwrap.h +++ b/src/ds/ptrwrap.h @@ -246,15 +246,6 @@ namespace snmalloc SNMALLOC_FAST_PATH T* operator->() const { - /* - * Alloc-bounded, platform-constrained pointers are associated with - * objects coming from or going to the client; we should be doing nothing - * with them. - */ - static_assert( - (bounds::spatial != capptr::dimension::Spatial::Alloc) || - (bounds::address_space_control != - capptr::dimension::AddressSpaceControl::User)); return this->unsafe_capptr; } From d76f5fdd28a6a53c5913293c1e518b601a37800d Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Mon, 16 Aug 2021 19:06:19 +0100 Subject: [PATCH 092/302] NFC: CapPtr: Introduce "Tame"/"Wild" split This is just the changes to the taxonomy, no use of it in the tree yet --- src/ds/ptrwrap.h | 89 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 81 insertions(+), 8 deletions(-) diff --git a/src/ds/ptrwrap.h b/src/ds/ptrwrap.h index 359d4e9c0..b08745e1a 100644 --- a/src/ds/ptrwrap.h +++ b/src/ds/ptrwrap.h @@ -69,30 +69,67 @@ namespace snmalloc Full }; + /** + * Distinguish pointers proximate provenance: pointers given to us by + * clients can be arbitrarily malformed while pointers from the kernel or + * internally can be presumed well-formed. See the Backend's + * capptr_domesticate(). + */ + enum class Wildness + { + /** + * The purported "pointer" here may just be a pile of bits. On CHERI + * architectures, for example, it may not have a set tag or may be out + * of bounds. + */ + Wild, + /** + * Either this pointer has provenance from the kernel or it has been + * checked by capptr_dewild. + */ + Tame + }; } // namespace dimension /** * The aggregate type of a bound: a Cartesian product of the individual * dimensions, above. */ - template + template< + dimension::Spatial S, + dimension::AddressSpaceControl AS, + dimension::Wildness W> struct bound { static constexpr enum dimension::Spatial spatial = S; static constexpr enum dimension::AddressSpaceControl address_space_control = AS; + static constexpr enum dimension::Wildness wildness = W; /** * Set just the spatial component of the bounds */ template - using with_spatial = bound; + using with_spatial = bound; /** * Set just the address space control component of the bounds */ template - using with_address_space_control = bound; + using with_address_space_control = bound; + + /** + * Set just the wild component of the bounds + */ + template + using with_wildness = bound; + + /* The dimensions here are not used completely orthogonally */ + static_assert( + !(W == dimension::Wildness::Wild) || + (S == dimension::Spatial::Alloc && + AS == dimension::AddressSpaceControl::User), + "Wild pointers must be annotated as tightly bounded"); }; // clang-format off @@ -105,9 +142,10 @@ namespace snmalloc */ template concept ConceptBound = - ConceptSame && + ConceptSame && ConceptSame; + const dimension::AddressSpaceControl> && + ConceptSame; #endif // clang-format on @@ -121,8 +159,10 @@ namespace snmalloc * Internal access to a Chunk of memory. These flow between the ASM and * the slab allocators, for example. */ - using Chunk = - bound; + using Chunk = bound< + dimension::Spatial::Chunk, + dimension::AddressSpaceControl::Full, + dimension::Wildness::Tame>; /** * User access to an entire Chunk. Used as an ephemeral state when @@ -141,6 +181,12 @@ namespace snmalloc */ using Alloc = AllocFull::with_address_space_control< dimension::AddressSpaceControl::User>; + + /** + * A wild (i.e., putative) CBAllocExport pointer handed back by the + * client. See capptr_from_client() and capptr_domesticate(). + */ + using AllocWild = Alloc::with_wildness; } // namespace bounds } // namespace capptr @@ -158,6 +204,11 @@ namespace snmalloc return false; } + if (BI::wildness != BO::wildness) + { + return false; + } + switch (BI::spatial) { using namespace capptr::dimension; @@ -246,6 +297,9 @@ namespace snmalloc SNMALLOC_FAST_PATH T* operator->() const { + static_assert( + bounds::wildness != capptr::dimension::Wildness::Wild, + "Trying to dereference a Wild pointer"); return this->unsafe_capptr; } @@ -278,6 +332,9 @@ namespace snmalloc template using Alloc = CapPtr; + template + using AllocWild = CapPtr; + } // namespace capptr static_assert(sizeof(capptr::Chunk) == sizeof(void*)); @@ -296,7 +353,8 @@ namespace snmalloc /** * With all the bounds and constraints in place, it's safe to extract a void - * pointer (to reveal to the client). Roughly dual to capptr_from_client(). + * pointer (to reveal to the client). Roughly dual to capptr_from_client(), + * but we stop oursevles from revealing anything not known to be domesticated. */ inline SNMALLOC_FAST_PATH void* capptr_reveal(capptr::Alloc p) { @@ -313,6 +371,21 @@ namespace snmalloc return capptr::Alloc(p); } + /** + * It's safe to mark any CapPtr as Wild. + */ + template + static inline SNMALLOC_FAST_PATH CapPtr< + T, + typename B::template with_wildness> + capptr_rewild(CapPtr p) + { + return CapPtr< + T, + typename B::template with_wildness>( + p.unsafe_capptr); + } + /** * * Wrap a std::atomic with bounds annotation and speak in terms of From e25db7b832e158ec21394e4fbbb68cff18797c82 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Fri, 17 Sep 2021 14:32:14 +0100 Subject: [PATCH 093/302] Move to FreeObject::T> FreeObject itself is now just a namespace (but `friend`-ly); the actual free list nodes are FreeObject::T-s that are templatized on the (perceived) `capptr::bound<>` of the pointer they contain. (These may differ across an instantiated snmalloc; for example, in the sandboxing design, the in-sandbox allocators may perceive all remotes to be full of `AllocUser` while the privileged allocator of sandbox memory should perceive its remote queue as holding `AllocUserWild` pointers in need of domestication.) The interfaces to `FreeObject::T`-s now let us distinguish between the base and inductive cases of the queues: * in the inductive case, the pointer we hold to a `FreeObject::T` and its next_object have the same bounds * in the base case, the pointer we hold has different bounds (typically, domesticated by contrast to the wild pointers in the queues). To keep the clutter down a bit, we occasionally use raw pointers when we can be reasonably certain that domestication is assured. Moreover, we define some type aliases, `FreeObject::{HeadPtr, QueuePtr, AtomicQueuePtr}`, that are slightly more convenient labels than, e.g., `CapPtr, BView>`. Because we are using template parameters for the `capptr::bound<>`s themselves, we cannot use the aliases for `CapPtr<>s` provided within `capptr::`. The two primary interfaces around free objects (`FreeListIter` AND `FreeListBuilder`) are adjusted appropriately and their `BView` and `BQueue` template paramters are plumbed explicitly around the tree. This makes for quite a bit of noise at the moment, but means that we'll be able to evolve parts of the tree separately and can consider putting defaults in once that's done. --- src/mem/corealloc.h | 43 +++-- src/mem/freelist.h | 376 ++++++++++++++++++++++++++++---------- src/mem/localalloc.h | 44 +++-- src/mem/localcache.h | 13 +- src/mem/metaslab.h | 22 ++- src/mem/remoteallocator.h | 37 ++-- src/mem/remotecache.h | 15 +- 7 files changed, 386 insertions(+), 164 deletions(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 881ce1008..7870d7389 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -161,8 +161,9 @@ namespace snmalloc { // Manufacture an allocation to prime the queue // Using an actual allocation removes a conditional from a critical path. - auto dummy = capptr::AllocFull(small_alloc_one(MIN_ALLOC_SIZE)) - .template as_static(); + auto dummy = + capptr::AllocFull(small_alloc_one(MIN_ALLOC_SIZE)) + .template as_static>(); if (dummy == nullptr) { error("Critical error: Out-of-memory during initialisation."); @@ -181,9 +182,11 @@ namespace snmalloc SNMALLOC_ASSERT(attached_cache != nullptr); // Use attached cache, and fill it if it is empty. return attached_cache->template alloc( - size, [&](sizeclass_t sizeclass, FreeListIter* fl) { - return small_alloc(sizeclass, *fl); - }); + size, + [&]( + sizeclass_t sizeclass, + FreeListIter* + fl) { return small_alloc(sizeclass, *fl); }); } static SNMALLOC_FAST_PATH void alloc_new_list( @@ -244,7 +247,10 @@ namespace snmalloc auto curr_ptr = start_ptr; do { - b.add(FreeObject::make(curr_ptr.as_void()), key, entropy); + b.add( + FreeObject::make(curr_ptr.as_void()), + key, + entropy); curr_ptr = curr_ptr->next; } while (curr_ptr != start_ptr); #else @@ -252,7 +258,9 @@ namespace snmalloc do { b.add( - Aal::capptr_bound(p, rsize), + Aal::capptr_bound< + FreeObject::T, + capptr::bounds::AllocFull>(p, rsize), key); p = pointer_offset(p, rsize); } while (p < slab_end); @@ -264,7 +272,7 @@ namespace snmalloc ChunkRecord* clear_slab(Metaslab* meta, sizeclass_t sizeclass) { auto& key = entropy.get_free_list_key(); - FreeListIter fl; + FreeListIter fl; auto more = meta->free_queue.close(fl, key); UNUSED(more); void* p = finish_alloc_no_zero(fl.take(key), sizeclass); @@ -426,7 +434,9 @@ namespace snmalloc * need_post will be set to true, if capacity is exceeded. */ void handle_dealloc_remote( - const MetaEntry& entry, capptr::AllocFull p, bool& need_post) + const MetaEntry& entry, + FreeObject::QueuePtr p, + bool& need_post) { // TODO this needs to not double count stats // TODO this needs to not double revoke if using MTE @@ -580,7 +590,8 @@ namespace snmalloc Metaslab::is_start_of_object(entry.get_sizeclass(), address_cast(p)), "Not deallocating start of an object"); - auto cp = capptr::AllocFull(reinterpret_cast(p)); + auto cp = FreeObject::QueuePtr( + reinterpret_cast*>(p)); auto& key = entropy.get_free_list_key(); @@ -591,8 +602,10 @@ namespace snmalloc } template - SNMALLOC_SLOW_PATH void* - small_alloc(sizeclass_t sizeclass, FreeListIter& fast_free_list) + SNMALLOC_SLOW_PATH void* small_alloc( + sizeclass_t sizeclass, + FreeListIter& + fast_free_list) { size_t rsize = sizeclass_to_size(sizeclass); @@ -650,7 +663,10 @@ namespace snmalloc template SNMALLOC_SLOW_PATH void* small_alloc_slow( - sizeclass_t sizeclass, FreeListIter& fast_free_list, size_t rsize) + sizeclass_t sizeclass, + FreeListIter& + fast_free_list, + size_t rsize) { // No existing free list get a new slab. size_t slab_size = sizeclass_to_slab_size(sizeclass); @@ -712,6 +728,7 @@ namespace snmalloc auto& entry = SharedStateHandle::Pagemap::get_metaentry( backend_state_ptr(), snmalloc::address_cast(p)); handle_dealloc_remote(entry, p, need_post); + // XXX n is not known to be domesticated p = n; } } diff --git a/src/mem/freelist.h b/src/mem/freelist.h index 09db13804..dd15f10d1 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -52,44 +52,136 @@ namespace snmalloc return (c + key.key1) * (n + key.key2); } - /** - * Free objects within each slab point directly to the next. - * There is an optional second field that is effectively a - * back pointer in a doubly linked list, however, it is encoded - * to prevent corruption. - * - * TODO: Consider putting prev_encoded at the end of the object, would - * require size to be threaded through, but would provide more OOB - * detection. - */ class FreeObject { - union + public: + template + class T; + + /** + * This "inductive step" type -- a queue-annotated pointer to a FreeObject + * containing a queue-annotated pointer -- shows up all over the place. + * Give it a shorter name (FreeObject::QueuePtr) for convenience. + */ + template + using QueuePtr = CapPtr, BQueue>; + + /** + * As with QueuePtr, but atomic. + */ + template + using AtomicQueuePtr = AtomicCapPtr, BQueue>; + + /** + * This is the "base case" of that induction. While we can't get rid of the + * two different type parameters (in general), we can at least get rid of a + * bit of the clutter. "FreeObject::HeadPtr" looks a little + * nicer than "CapPtr, BView>". + */ + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + using HeadPtr = CapPtr, BView>; + + /** + * As with HeadPtr, but atomic. + */ + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + using AtomicHeadPtr = AtomicCapPtr, BView>; + + /** + * Free objects within each slab point directly to the next. + * There is an optional second field that is effectively a + * back pointer in a doubly linked list, however, it is encoded + * to prevent corruption. + * + * This is an inner class to avoid the need to specify BQueue when calling + * static methods. + * + * Raw C++ pointers to this type are *assumed to be domesticated*. In some + * cases we still explicitly annotate domesticated FreeObject*-s as + * CapPtr<>, but more often CapPtr,B> will have B = A. + * + * TODO: Consider putting prev_encoded at the end of the object, would + * require size to be threaded through, but would provide more OOB + * detection. + */ + template + class T { - capptr::AllocFull next_object; - // TODO: Should really use C++20 atomic_ref rather than a union. - capptr::AtomicAllocFull atomic_next_object; - }; + friend class FreeObject; + + union + { + QueuePtr next_object; + // TODO: Should really use C++20 atomic_ref rather than a union. + AtomicQueuePtr atomic_next_object; + }; #ifdef SNMALLOC_CHECK_CLIENT - // Encoded representation of a back pointer. - // Hard to fake, and provides consistency on - // the next pointers. - address_t prev_encoded; + // Encoded representation of a back pointer. + // Hard to fake, and provides consistency on + // the next pointers. + address_t prev_encoded; #endif - public: - static capptr::AllocFull make(capptr::AllocFull p) + public: + QueuePtr atomic_read_next(const FreeListKey& key) + { + auto n = FreeObject::decode_next( + address_cast(&this->next_object), + this->atomic_next_object.load(std::memory_order_acquire), + key); +#ifdef SNMALLOC_CHECK_CLIENT + // XXX n is not known to be domesticated here + if (n != nullptr) + { + n->check_prev(signed_prev(address_cast(this), address_cast(n), key)); + } +#else + UNUSED(key); +#endif + return n; + } + + /** + * Read the next pointer + */ + QueuePtr read_next(const FreeListKey& key) + { + return FreeObject::decode_next( + address_cast(&this->next_object), this->next_object, key); + } + + /** + * Check the signature of this FreeObject + */ + void check_prev(address_t signed_prev) + { + UNUSED(signed_prev); + check_client( + signed_prev == this->prev_encoded, + "Heap corruption - free list corrupted!"); + } + }; + + // Note the inverted template argument order, since BView is inferable. + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue, + SNMALLOC_CONCEPT(capptr::ConceptBound) BView> + static HeadPtr make(CapPtr p) { - return p.template as_static(); + return p.template as_static>(); } + private: /** - * Encode next + * Involutive encryption with raw pointers */ - inline static capptr::AllocFull encode_next( - address_t curr, - capptr::AllocFull next, - const FreeListKey& key) + template + inline static FreeObject::T* code_next( + address_t curr, FreeObject::T* next, const FreeListKey& key) { // Note we can consider other encoding schemes here. // * XORing curr and next. This doesn't require any key material @@ -100,8 +192,8 @@ namespace snmalloc if constexpr (CHECK_CLIENT && !aal_supports) { - return capptr::AllocFull(reinterpret_cast( - reinterpret_cast(next.unsafe_ptr()) ^ key.key_next)); + return reinterpret_cast*>( + reinterpret_cast(next) ^ key.key_next); } else { @@ -110,19 +202,82 @@ namespace snmalloc } } + public: + /** + * Encode next. We perform two convenient little bits of type-level + * sleight of hand here: + * + * 1) We convert the provided HeadPtr to a QueuePtr, forgetting BView in + * the result; all the callers write the result through a pointer to a + * QueuePtr, though, strictly, the result itself is no less domesticated + * than the input (even if it is obfuscated). + * + * 2) Speaking of obfuscation, we continue to use a CapPtr<> type even + * though the result is likely not safe to dereference, being an + * obfuscated bundle of bits (on non-CHERI architectures, anyway). That's + * additional motivation to consider the result BQueue-bounded, as that is + * likely (but not necessarily) Wild. + */ + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + inline static QueuePtr encode_next( + address_t curr, HeadPtr next, const FreeListKey& key) + { + return QueuePtr(code_next(curr, next.unsafe_ptr(), key)); + } + /** - * Assign next_object and update its prev_encoded if SNMALLOC_CHECK_CLIENT. - * Static so that it can be used on reference to a FreeObject. + * Decode next. While traversing a queue, BView and BQueue here will + * often be equal (i.e., CBAllocExportWild) rather than dichotomous. + * However, we do occasionally decode an actual head pointer, so be + * polymorphic here. + */ + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + inline static HeadPtr decode_next( + address_t curr, HeadPtr next, const FreeListKey& key) + { + return HeadPtr(code_next(curr, next.unsafe_ptr(), key)); + } + + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + static void assert_view_queue_bounds() + { + static_assert( + BView::wildness == capptr::dimension::Wildness::Tame, + "FreeObject View must be domesticated, justifying raw pointers"); + + static_assert( + std::is_same_v< + typename BQueue::template with_wildness< + capptr::dimension::Wildness::Tame>, + BView>, + "FreeObject Queue bounds must match View bounds (but may be Wild)"); + } + + /** + * Assign next_object and update its prev_encoded if + * SNMALLOC_CHECK_CLIENT. Static so that it can be used on reference to a + * FreeObject. * * Returns a pointer to the next_object field of the next parameter as an * optimization for repeated snoc operations (in which * next->next_object is nullptr). */ - static capptr::AllocFull* store_next( - capptr::AllocFull* curr, - capptr::AllocFull next, + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + static QueuePtr* store_next( + QueuePtr* curr, + HeadPtr next, const FreeListKey& key) { + assert_view_queue_bounds(); + #ifdef SNMALLOC_CHECK_CLIENT next->prev_encoded = signed_prev(address_cast(curr), address_cast(next), key); @@ -133,10 +288,10 @@ namespace snmalloc return &(next->next_object); } - static void - store_null(capptr::AllocFull* curr, const FreeListKey& key) + template + static void store_null(QueuePtr* curr, const FreeListKey& key) { - *curr = encode_next(address_cast(curr), nullptr, key); + *curr = encode_next(address_cast(curr), QueuePtr(nullptr), key); } /** @@ -144,62 +299,41 @@ namespace snmalloc * * Uses the atomic view of next, so can be used in the message queues. */ - void atomic_store_next( - capptr::AllocFull next, const FreeListKey& key) + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + static void atomic_store_next( + HeadPtr curr, + HeadPtr next, + const FreeListKey& key) { + static_assert(BView::wildness == capptr::dimension::Wildness::Tame); + #ifdef SNMALLOC_CHECK_CLIENT next->prev_encoded = - signed_prev(address_cast(this), address_cast(next), key); + signed_prev(address_cast(curr), address_cast(next), key); #else UNUSED(key); #endif // Signature needs to be visible before item is linked in // so requires release semantics. - atomic_next_object.store( - encode_next(address_cast(&next_object), next, key), + curr->atomic_next_object.store( + encode_next(address_cast(&curr->next_object), next, key), std::memory_order_release); } - void atomic_store_null(const FreeListKey& key) - { - atomic_next_object.store( - encode_next(address_cast(&next_object), nullptr, key), - std::memory_order_relaxed); - } - - capptr::AllocFull atomic_read_next(const FreeListKey& key) - { - auto n = encode_next( - address_cast(&next_object), - atomic_next_object.load(std::memory_order_acquire), - key); -#ifdef SNMALLOC_CHECK_CLIENT - if (n != nullptr) - { - n->check_prev(signed_prev(address_cast(this), address_cast(n), key)); - } -#else - UNUSED(key); -#endif - return n; - } - - /** - * Check the signature of this FreeObject - */ - void check_prev(address_t signed_prev) + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + static void + atomic_store_null(HeadPtr curr, const FreeListKey& key) { - UNUSED(signed_prev); - check_client( - signed_prev == prev_encoded, "Heap corruption - free list corrupted!"); - } + static_assert(BView::wildness == capptr::dimension::Wildness::Tame); - /** - * Read the next pointer - */ - capptr::AllocFull read_next(const FreeListKey& key) - { - return encode_next(address_cast(&next_object), next_object, key); + curr->atomic_next_object.store( + encode_next( + address_cast(&curr->next_object), QueuePtr(nullptr), key), + std::memory_order_relaxed); } }; @@ -212,16 +346,19 @@ namespace snmalloc * * Checks signing of pointers */ + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> class FreeListIter { - capptr::AllocFull curr{nullptr}; + FreeObject::HeadPtr curr{nullptr}; #ifdef SNMALLOC_CHECK_CLIENT address_t prev{0}; #endif public: constexpr FreeListIter( - capptr::AllocFull head, address_t prev_value) + FreeObject::HeadPtr head, address_t prev_value) : curr(head) { #ifdef SNMALLOC_CHECK_CLIENT @@ -243,7 +380,7 @@ namespace snmalloc /** * Returns current head without affecting the iterator. */ - capptr::AllocFull peek() + FreeObject::HeadPtr peek() { return curr; } @@ -251,7 +388,7 @@ namespace snmalloc /** * Moves the iterator on, and returns the current value. */ - capptr::AllocFull take(const FreeListKey& key) + FreeObject::HeadPtr take(const FreeListKey& key) { auto c = curr; auto next = curr->read_next(key); @@ -286,17 +423,49 @@ namespace snmalloc * If RANDOM is set to false, then the code does not perform any * randomisation. */ - template + template< + bool RANDOM, + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue, + bool INIT = true> class FreeListBuilder { static constexpr size_t LENGTH = RANDOM ? 2 : 1; + /* + * We use native pointers below so that we don't run afoul of strict + * aliasing rules. head is a FreeObject::HeadPtr -- that + * is, a known-domesticated pointer to a queue of wild pointers -- and + * it's usually the case that end is a FreeObject::QueuePtr* -- + * that is, a known-domesticated pointer to a wild pointer to a queue of + * wild pointers. However, in order to do branchless inserts, we set end + * = &head, which breaks strict aliasing rules with the types as given. + * Fortunately, these are private members and so we can use native + * pointers and just expose a more strongly typed interface. + */ + // Pointer to the first element. - std::array, LENGTH> head; + std::array head{nullptr}; // Pointer to the reference to the last element. // In the empty case end[i] == &head[i] // This enables branch free enqueuing. - std::array*, LENGTH> end{nullptr}; + std::array end{nullptr}; + + FreeObject::QueuePtr* cast_end(uint32_t ix) + { + return reinterpret_cast*>(end[ix]); + } + + void set_end(uint32_t ix, FreeObject::QueuePtr* p) + { + end[ix] = reinterpret_cast(p); + } + + FreeObject::HeadPtr cast_head(uint32_t ix) + { + return FreeObject::HeadPtr( + static_cast*>(head[ix])); + } std::array length{}; @@ -304,7 +473,9 @@ namespace snmalloc constexpr FreeListBuilder() { if (INIT) + { init(); + } } /** @@ -314,8 +485,10 @@ namespace snmalloc { for (size_t i = 0; i < LENGTH; i++) { - if (address_cast(end[i]) != address_cast(&head[i])) + if (end[i] != &head[i]) + { return false; + } } return true; } @@ -324,7 +497,7 @@ namespace snmalloc * Adds an element to the builder */ void add( - capptr::AllocFull n, + FreeObject::HeadPtr n, const FreeListKey& key, LocalEntropy& entropy) { @@ -334,7 +507,7 @@ namespace snmalloc else index = 0; - end[index] = FreeObject::store_next(end[index], n, key); + set_end(index, FreeObject::store_next(cast_end(index), n, key)); if constexpr (RANDOM) { length[index]++; @@ -351,10 +524,10 @@ namespace snmalloc */ template std::enable_if_t - add(capptr::AllocFull n, const FreeListKey& key) + add(FreeObject::HeadPtr n, const FreeListKey& key) { static_assert(RANDOM_ == RANDOM, "Don't set template parameter"); - end[0] = FreeObject::store_next(end[0], n, key); + set_end(0, FreeObject::store_next(cast_end(0), n, key)); } /** @@ -363,7 +536,7 @@ namespace snmalloc SNMALLOC_FAST_PATH void terminate_list(uint32_t index, const FreeListKey& key) { - FreeObject::store_null(end[index], key); + FreeObject::store_null(cast_end(index), key); } /** @@ -375,11 +548,11 @@ namespace snmalloc * and is thus subject to encoding if the next_object pointers * encoded. */ - capptr::AllocFull + FreeObject::HeadPtr read_head(uint32_t index, const FreeListKey& key) { - return FreeObject::encode_next( - address_cast(&head[index]), head[index], key); + return FreeObject::decode_next( + address_cast(&head[index]), cast_head(index), key); } address_t get_fake_signed_prev(uint32_t index, const FreeListKey& key) @@ -396,7 +569,8 @@ namespace snmalloc * * The return value is how many entries are still contained in the builder. */ - SNMALLOC_FAST_PATH uint16_t close(FreeListIter& fl, const FreeListKey& key) + SNMALLOC_FAST_PATH uint16_t + close(FreeListIter& fl, const FreeListKey& key) { uint32_t i; if constexpr (RANDOM) @@ -447,7 +621,9 @@ namespace snmalloc template std::enable_if_t< !RANDOM_, - std::pair, capptr::AllocFull>> + std::pair< + FreeObject::HeadPtr, + FreeObject::HeadPtr>> extract_segment(const FreeListKey& key) { static_assert(RANDOM_ == RANDOM, "Don't set SFINAE parameter!"); @@ -458,8 +634,8 @@ namespace snmalloc // this is doing a CONTAINING_RECORD like cast to get back // to the actual object. This isn't true if the builder is // empty, but you are not allowed to call this in the empty case. - auto last = - capptr::AllocFull(reinterpret_cast(end[0])); + auto last = FreeObject::HeadPtr( + reinterpret_cast*>(end[0])); init(); return {first, last}; } diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index b20b3d3a4..ac183f00d 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -211,25 +211,33 @@ namespace snmalloc SNMALLOC_FAST_PATH void* small_alloc(size_t size) { // SNMALLOC_ASSUME(size <= sizeclass_to_size(NUM_SIZECLASSES)); - auto slowpath = [&]( - sizeclass_t sizeclass, - FreeListIter* fl) SNMALLOC_FAST_PATH_LAMBDA { - if (likely(core_alloc != nullptr)) - { - return core_alloc->handle_message_queue( - [](CoreAlloc* core_alloc, sizeclass_t sizeclass, FreeListIter* fl) { - return core_alloc->template small_alloc(sizeclass, *fl); + auto slowpath = + [&]( + sizeclass_t sizeclass, + FreeListIter* + fl) SNMALLOC_FAST_PATH_LAMBDA { + if (likely(core_alloc != nullptr)) + { + return core_alloc->handle_message_queue( + []( + CoreAlloc* core_alloc, + sizeclass_t sizeclass, + FreeListIter< + capptr::bounds::AllocFull, + capptr::bounds::AllocFull>* fl) { + return core_alloc->template small_alloc( + sizeclass, *fl); + }, + core_alloc, + sizeclass, + fl); + } + return lazy_init( + [&](CoreAlloc*, sizeclass_t sizeclass) { + return small_alloc(sizeclass_to_size(sizeclass)); }, - core_alloc, - sizeclass, - fl); - } - return lazy_init( - [&](CoreAlloc*, sizeclass_t sizeclass) { - return small_alloc(sizeclass_to_size(sizeclass)); - }, - sizeclass); - }; + sizeclass); + }; return local_cache.template alloc( size, slowpath); diff --git a/src/mem/localcache.h b/src/mem/localcache.h index a655cfbab..8184141e2 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -12,8 +12,9 @@ namespace snmalloc { using Stats = AllocStats; - inline static SNMALLOC_FAST_PATH void* - finish_alloc_no_zero(capptr::AllocFull p, sizeclass_t sizeclass) + inline static SNMALLOC_FAST_PATH void* finish_alloc_no_zero( + FreeObject::HeadPtr p, + sizeclass_t sizeclass) { SNMALLOC_ASSERT(Metaslab::is_start_of_object(sizeclass, address_cast(p))); UNUSED(sizeclass); @@ -24,8 +25,9 @@ namespace snmalloc } template - inline static SNMALLOC_FAST_PATH void* - finish_alloc(capptr::AllocFull p, sizeclass_t sizeclass) + inline static SNMALLOC_FAST_PATH void* finish_alloc( + FreeObject::HeadPtr p, + sizeclass_t sizeclass) { auto r = finish_alloc_no_zero(p, sizeclass); @@ -46,7 +48,8 @@ namespace snmalloc // Free list per small size class. These are used for // allocation on the fast path. This part of the code is inspired by // mimalloc. - FreeListIter small_fast_free_lists[NUM_SIZECLASSES]; + FreeListIter + small_fast_free_lists[NUM_SIZECLASSES]; // This is the entropy for a particular thread. LocalEntropy entropy; diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index a6015b2b2..9a7165bb2 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -25,9 +25,11 @@ namespace snmalloc * Data-structure for building the free list for this slab. */ #ifdef SNMALLOC_CHECK_CLIENT - FreeListBuilder free_queue; + FreeListBuilder + free_queue; #else - FreeListBuilder free_queue; + FreeListBuilder + free_queue; #endif /** @@ -152,16 +154,18 @@ namespace snmalloc * component, but with randomisation, it may only return part of the * available objects for this metaslab. */ - static SNMALLOC_FAST_PATH std::pair, bool> - alloc_free_list( - Metaslab* meta, - FreeListIter& fast_free_list, - LocalEntropy& entropy, - sizeclass_t sizeclass) + static SNMALLOC_FAST_PATH + std::pair, bool> + alloc_free_list( + Metaslab* meta, + FreeListIter& + fast_free_list, + LocalEntropy& entropy, + sizeclass_t sizeclass) { auto& key = entropy.get_free_list_key(); - FreeListIter tmp_fl; + std::remove_reference_t tmp_fl; auto remaining = meta->free_queue.close(tmp_fl, key); auto p = tmp_fl.take(key); fast_free_list = tmp_fl; diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index f9596c796..2e4e73561 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -35,10 +35,12 @@ namespace snmalloc // Store the message queue on a separate cacheline. It is mutable data that // is read by other threads. - alignas(CACHELINE_SIZE) capptr::AtomicAllocFull back{nullptr}; + alignas(CACHELINE_SIZE) + FreeObject::AtomicQueuePtr back{nullptr}; // Store the two ends on different cache lines as access by different // threads. - alignas(CACHELINE_SIZE) capptr::AllocFull front{nullptr}; + alignas(CACHELINE_SIZE) + FreeObject::QueuePtr front{nullptr}; constexpr RemoteAllocator() = default; @@ -48,17 +50,17 @@ namespace snmalloc SNMALLOC_ASSERT(front != nullptr); } - void init(capptr::AllocFull stub) + void init(FreeObject::QueuePtr stub) { - stub->atomic_store_null(key_global); + FreeObject::atomic_store_null(stub, key_global); front = stub; back.store(stub, std::memory_order_relaxed); invariant(); } - capptr::AllocFull destroy() + FreeObject::QueuePtr destroy() { - capptr::AllocFull fnt = front; + FreeObject::QueuePtr fnt = front; back.store(nullptr, std::memory_order_relaxed); front = nullptr; return fnt; @@ -66,7 +68,8 @@ namespace snmalloc inline bool is_empty() { - capptr::AllocFull bk = back.load(std::memory_order_relaxed); + FreeObject::QueuePtr bk = + back.load(std::memory_order_relaxed); return bk == front; } @@ -76,21 +79,22 @@ namespace snmalloc * last should be linked together through their next pointers. */ void enqueue( - capptr::AllocFull first, - capptr::AllocFull last, + FreeObject::QueuePtr first, + FreeObject::QueuePtr last, const FreeListKey& key) { invariant(); - last->atomic_store_null(key); + FreeObject::atomic_store_null(last, key); // exchange needs to be a release, so nullptr in next is visible. - capptr::AllocFull prev = + FreeObject::QueuePtr prev = back.exchange(last, std::memory_order_release); - prev->atomic_store_next(first, key); + // XXX prev is not known to be domesticated + FreeObject::atomic_store_next(prev, first, key); } - capptr::AllocFull peek() + FreeObject::QueuePtr peek() { return front; } @@ -98,12 +102,13 @@ namespace snmalloc /** * Returns the front message, or null if not possible to return a message. */ - std::pair, bool> + std::pair, bool> dequeue(const FreeListKey& key) { invariant(); - capptr::AllocFull first = front; - capptr::AllocFull next = first->atomic_read_next(key); + FreeObject::QueuePtr first = front; + FreeObject::QueuePtr next = + first->atomic_read_next(key); if (next != nullptr) { diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index 0bf11ed74..7bcc57a56 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -16,7 +16,14 @@ namespace snmalloc */ struct RemoteDeallocCache { - std::array, REMOTE_SLOTS> list; + std::array< + FreeListBuilder< + false, + capptr::bounds::AllocFull, + capptr::bounds::AllocFull, + false>, + REMOTE_SLOTS> + list; /** * The total amount of memory we are waiting for before we will dispatch @@ -70,7 +77,8 @@ namespace snmalloc const FreeListKey& key) { SNMALLOC_ASSERT(initialised); - auto r = p.template as_reinterpret(); + auto r = + p.template as_reinterpret>(); list[get_slot(target_id, 0)].add(r, key); } @@ -110,7 +118,8 @@ namespace snmalloc // Entries could map back onto the "resend" list, // so take copy of the head, mark the last element, // and clear the original list. - FreeListIter resend; + FreeListIter + resend; list[my_slot].close(resend, key); post_round++; From 06e333a3a9d4d3cf611ef2df541909910306a67f Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Tue, 28 Sep 2021 12:49:48 +0100 Subject: [PATCH 094/302] NFC: FreeObject: shift readers to take domestication callbacks If we're going to check next's prev in atomic_read_next, we will need to domesticate the next pointer first. We could push the check up, but that opens boxes, so it's simpler to plumb domestication this far down. For symmetry, we also plumb to (non-atomic) read_next. --- src/mem/corealloc.h | 11 ++++++--- src/mem/freelist.h | 52 +++++++++++++++++++++++++++------------ src/mem/remoteallocator.h | 11 ++++++--- 3 files changed, 51 insertions(+), 23 deletions(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 7870d7389..ecc01b023 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -416,7 +416,7 @@ namespace snmalloc #ifdef SNMALLOC_TRACING std::cout << "Handling remote" << std::endl; #endif - handle_dealloc_remote(entry, p, need_post); + handle_dealloc_remote(entry, r.first.as_void(), need_post); } if (need_post) @@ -435,7 +435,7 @@ namespace snmalloc */ void handle_dealloc_remote( const MetaEntry& entry, - FreeObject::QueuePtr p, + CapPtr p, bool& need_post) { // TODO this needs to not double count stats @@ -716,6 +716,9 @@ namespace snmalloc bool flush(bool destroy_queue = false) { SNMALLOC_ASSERT(attached_cache != nullptr); + // TODO: Placeholder + auto domesticate = [](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { return p; }; if (destroy_queue) { @@ -724,10 +727,10 @@ namespace snmalloc while (p != nullptr) { bool need_post = true; // Always going to post, so ignore. - auto n = p->atomic_read_next(key_global); + auto n = p->atomic_read_next(key_global, domesticate); auto& entry = SharedStateHandle::Pagemap::get_metaentry( backend_state_ptr(), snmalloc::address_cast(p)); - handle_dealloc_remote(entry, p, need_post); + handle_dealloc_remote(entry, p.as_void(), need_post); // XXX n is not known to be domesticated p = n; } diff --git a/src/mem/freelist.h b/src/mem/freelist.h index dd15f10d1..0b882a36f 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -127,31 +127,40 @@ namespace snmalloc #endif public: - QueuePtr atomic_read_next(const FreeListKey& key) + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView = typename BQueue:: + template with_wildness, + typename Domesticator> + HeadPtr + atomic_read_next(const FreeListKey& key, Domesticator domesticate) { - auto n = FreeObject::decode_next( + auto n_wild = FreeObject::decode_next( address_cast(&this->next_object), this->atomic_next_object.load(std::memory_order_acquire), key); + auto n_tame = domesticate(n_wild); #ifdef SNMALLOC_CHECK_CLIENT - // XXX n is not known to be domesticated here - if (n != nullptr) + if (n_tame != nullptr) { - n->check_prev(signed_prev(address_cast(this), address_cast(n), key)); + n_tame->check_prev( + signed_prev(address_cast(this), address_cast(n_tame), key)); } -#else - UNUSED(key); #endif - return n; + return n_tame; } /** * Read the next pointer */ - QueuePtr read_next(const FreeListKey& key) + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView = typename BQueue:: + template with_wildness, + typename Domesticator> + HeadPtr + read_next(const FreeListKey& key, Domesticator domesticate) { - return FreeObject::decode_next( - address_cast(&this->next_object), this->next_object, key); + return domesticate(FreeObject::decode_next( + address_cast(&this->next_object), this->next_object, key)); } /** @@ -228,10 +237,18 @@ namespace snmalloc } /** - * Decode next. While traversing a queue, BView and BQueue here will - * often be equal (i.e., CBAllocExportWild) rather than dichotomous. - * However, we do occasionally decode an actual head pointer, so be - * polymorphic here. + * Decode next. While traversing a queue, BView and BQueue here will often + * be equal (i.e., CBAllocExportWild) rather than dichotomous. However, + * we do occasionally decode an actual head pointer, so be polymorphic here. + * + * TODO: We'd like, in some sense, to more tightly couple or integrate this + * into to the domestication process. We could introduce an additional + * state in the capptr_bounds::wild taxonomy (e.g, Obfuscated) so that the + * Domesticator-s below have to call through this function to get the Wild + * pointer they can then make Tame. It's not yet entirely clear what that + * would look like and whether/how the encode_next side of things should be + * exposed. For the moment, obfuscation is left encapsulated within + * FreeObject and we do not capture any of it statically. */ template< SNMALLOC_CONCEPT(capptr::ConceptBound) BView, @@ -391,7 +408,10 @@ namespace snmalloc FreeObject::HeadPtr take(const FreeListKey& key) { auto c = curr; - auto next = curr->read_next(key); + // TODO: Placeholder + auto domesticate = [](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { return p; }; + auto next = curr->read_next(key, domesticate); Aal::prefetch(next.unsafe_ptr()); curr = next; diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index 2e4e73561..2be554f36 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -102,13 +102,18 @@ namespace snmalloc /** * Returns the front message, or null if not possible to return a message. */ - std::pair, bool> + std::pair< + FreeObject::HeadPtr, + bool> dequeue(const FreeListKey& key) { + auto domesticate = [](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { return p; }; invariant(); - FreeObject::QueuePtr first = front; + FreeObject::HeadPtr + first = domesticate(front); FreeObject::QueuePtr next = - first->atomic_read_next(key); + first->atomic_read_next(key, domesticate); if (next != nullptr) { From 55cd5e87c0996b68ae9d7d469f674e7fb9f4d4ea Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Tue, 21 Sep 2021 12:23:10 +0100 Subject: [PATCH 095/302] NFC: Add FreeObject::from_next_ptr Rather than open-code the conversion from &f->next_object to f, add a static method to FreeObject and take the opportunity to add a static_assert that our reinterpret_cast is sound. --- src/mem/freelist.h | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/mem/freelist.h b/src/mem/freelist.h index 0b882a36f..9dd8d9b06 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -184,6 +184,17 @@ namespace snmalloc return p.template as_static>(); } + /** + * A container-of operation to convert &f->next_object to f + */ + template + static FreeObject::T* + from_next_ptr(CapPtr, BQueue>* ptr) + { + static_assert(offsetof(FreeObject::T, next_object) == 0); + return reinterpret_cast*>(ptr); + } + private: /** * Involutive encryption with raw pointers @@ -655,7 +666,7 @@ namespace snmalloc // to the actual object. This isn't true if the builder is // empty, but you are not allowed to call this in the empty case. auto last = FreeObject::HeadPtr( - reinterpret_cast*>(end[0])); + FreeObject::from_next_ptr(cast_end(0))); init(); return {first, last}; } From 7e53a2e82a2e6b0aa64e8d7defbc2790a44b50a4 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Tue, 21 Sep 2021 19:17:51 +0100 Subject: [PATCH 096/302] CapPtr: shift free lists to Alloc bounds This is incomplete, yet still more reflective of what's going on: we take the exported pointers back from userspace and thread them directly into the free lists. So: move capptr_to_user_address_control to list construction time rather than list consumption time. --- src/mem/corealloc.h | 41 ++++++++++--------- src/mem/freelist.h | 4 +- src/mem/localalloc.h | 83 ++++++++++++++++++++------------------- src/mem/localcache.h | 11 +++--- src/mem/metaslab.h | 8 ++-- src/mem/remoteallocator.h | 32 +++++++-------- src/mem/remotecache.h | 11 +++--- 7 files changed, 99 insertions(+), 91 deletions(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index ecc01b023..06858f84f 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -162,8 +162,8 @@ namespace snmalloc // Manufacture an allocation to prime the queue // Using an actual allocation removes a conditional from a critical path. auto dummy = - capptr::AllocFull(small_alloc_one(MIN_ALLOC_SIZE)) - .template as_static>(); + capptr::Alloc(small_alloc_one(MIN_ALLOC_SIZE)) + .template as_static>(); if (dummy == nullptr) { error("Critical error: Out-of-memory during initialisation."); @@ -185,8 +185,9 @@ namespace snmalloc size, [&]( sizeclass_t sizeclass, - FreeListIter* - fl) { return small_alloc(sizeclass, *fl); }); + FreeListIter* fl) { + return small_alloc(sizeclass, *fl); + }); } static SNMALLOC_FAST_PATH void alloc_new_list( @@ -248,7 +249,8 @@ namespace snmalloc do { b.add( - FreeObject::make(curr_ptr.as_void()), + FreeObject::make( + capptr_to_user_address_control(curr_ptr.as_void())), key, entropy); curr_ptr = curr_ptr->next; @@ -258,9 +260,10 @@ namespace snmalloc do { b.add( - Aal::capptr_bound< - FreeObject::T, - capptr::bounds::AllocFull>(p, rsize), + FreeObject::make( + capptr_to_user_address_control( + Aal::capptr_bound( + p.as_void(), rsize))), key); p = pointer_offset(p, rsize); } while (p < slab_end); @@ -272,7 +275,7 @@ namespace snmalloc ChunkRecord* clear_slab(Metaslab* meta, sizeclass_t sizeclass) { auto& key = entropy.get_free_list_key(); - FreeListIter fl; + FreeListIter fl; auto more = meta->free_queue.close(fl, key); UNUSED(more); void* p = finish_alloc_no_zero(fl.take(key), sizeclass); @@ -435,7 +438,7 @@ namespace snmalloc */ void handle_dealloc_remote( const MetaEntry& entry, - CapPtr p, + CapPtr p, bool& need_post) { // TODO this needs to not double count stats @@ -444,7 +447,7 @@ namespace snmalloc if (likely(entry.get_remote() == public_state())) { - if (likely(dealloc_local_object_fast(entry, p.unsafe_ptr(), entropy))) + if (likely(dealloc_local_object_fast(entry, p.as_void(), entropy))) return; dealloc_local_object_slow(entry); @@ -569,7 +572,8 @@ namespace snmalloc return handle_message_queue_inner(action, args...); } - SNMALLOC_FAST_PATH void dealloc_local_object(void* p) + SNMALLOC_FAST_PATH void + dealloc_local_object(CapPtr p) { auto entry = SharedStateHandle::Pagemap::get_metaentry( backend_state_ptr(), snmalloc::address_cast(p)); @@ -580,7 +584,9 @@ namespace snmalloc } SNMALLOC_FAST_PATH static bool dealloc_local_object_fast( - const MetaEntry& entry, void* p, LocalEntropy& entropy) + const MetaEntry& entry, + CapPtr p, + LocalEntropy& entropy) { auto meta = entry.get_metaslab(); @@ -590,8 +596,7 @@ namespace snmalloc Metaslab::is_start_of_object(entry.get_sizeclass(), address_cast(p)), "Not deallocating start of an object"); - auto cp = FreeObject::QueuePtr( - reinterpret_cast*>(p)); + auto cp = p.as_static>(); auto& key = entropy.get_free_list_key(); @@ -604,7 +609,7 @@ namespace snmalloc template SNMALLOC_SLOW_PATH void* small_alloc( sizeclass_t sizeclass, - FreeListIter& + FreeListIter& fast_free_list) { size_t rsize = sizeclass_to_size(sizeclass); @@ -664,7 +669,7 @@ namespace snmalloc template SNMALLOC_SLOW_PATH void* small_alloc_slow( sizeclass_t sizeclass, - FreeListIter& + FreeListIter& fast_free_list, size_t rsize) { @@ -717,7 +722,7 @@ namespace snmalloc { SNMALLOC_ASSERT(attached_cache != nullptr); // TODO: Placeholder - auto domesticate = [](FreeObject::QueuePtr p) + auto domesticate = [](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { return p; }; if (destroy_queue) diff --git a/src/mem/freelist.h b/src/mem/freelist.h index 9dd8d9b06..fadf81e41 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -249,8 +249,8 @@ namespace snmalloc /** * Decode next. While traversing a queue, BView and BQueue here will often - * be equal (i.e., CBAllocExportWild) rather than dichotomous. However, - * we do occasionally decode an actual head pointer, so be polymorphic here. + * be equal (i.e., AllocUserWild) rather than dichotomous. However, we do + * occasionally decode an actual head pointer, so be polymorphic here. * * TODO: We'd like, in some sense, to more tightly couple or integrate this * into to the domestication process. We could introduce an additional diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index ac183f00d..74d33b9df 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -214,30 +214,29 @@ namespace snmalloc auto slowpath = [&]( sizeclass_t sizeclass, - FreeListIter* - fl) SNMALLOC_FAST_PATH_LAMBDA { - if (likely(core_alloc != nullptr)) - { - return core_alloc->handle_message_queue( - []( - CoreAlloc* core_alloc, - sizeclass_t sizeclass, - FreeListIter< - capptr::bounds::AllocFull, - capptr::bounds::AllocFull>* fl) { - return core_alloc->template small_alloc( - sizeclass, *fl); + FreeListIter* fl) + SNMALLOC_FAST_PATH_LAMBDA { + if (likely(core_alloc != nullptr)) + { + return core_alloc->handle_message_queue( + []( + CoreAlloc* core_alloc, + sizeclass_t sizeclass, + FreeListIter* + fl) { + return core_alloc->template small_alloc( + sizeclass, *fl); + }, + core_alloc, + sizeclass, + fl); + } + return lazy_init( + [&](CoreAlloc*, sizeclass_t sizeclass) { + return small_alloc(sizeclass_to_size(sizeclass)); }, - core_alloc, - sizeclass, - fl); - } - return lazy_init( - [&](CoreAlloc*, sizeclass_t sizeclass) { - return small_alloc(sizeclass_to_size(sizeclass)); - }, - sizeclass); - }; + sizeclass); + }; return local_cache.template alloc( size, slowpath); @@ -260,20 +259,18 @@ namespace snmalloc * In the second case we need to recheck if this is a remote deallocation, * as we might acquire the originating allocator. */ - SNMALLOC_SLOW_PATH void dealloc_remote_slow(void* p) + SNMALLOC_SLOW_PATH void dealloc_remote_slow(capptr::Alloc p) { if (core_alloc != nullptr) { #ifdef SNMALLOC_TRACING - std::cout << "Remote dealloc post" << p << " size " << alloc_size(p) - << std::endl; + std::cout << "Remote dealloc post" << p.unsafe_ptr() << " size " + << alloc_size(p.unsafe_ptr()) << std::endl; #endif MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry( core_alloc->backend_state_ptr(), address_cast(p)); local_cache.remote_dealloc_cache.template dealloc( - entry.get_remote()->trunc_id(), - capptr::AllocFull(p), - key_global); + entry.get_remote()->trunc_id(), p, key_global); post_remote_cache(); return; } @@ -281,8 +278,8 @@ namespace snmalloc // Recheck what kind of dealloc we should do incase, the allocator we get // from lazy_init is the originating allocator. lazy_init( - [&](CoreAlloc*, void* p) { - dealloc(p); // TODO don't double count statistics + [&](CoreAlloc*, CapPtr p) { + dealloc(p.unsafe_ptr()); // TODO don't double count statistics return nullptr; }, p); @@ -450,10 +447,10 @@ namespace snmalloc return alloc(size); } - SNMALLOC_FAST_PATH void dealloc(void* p) + SNMALLOC_FAST_PATH void dealloc(void* p_raw) { #ifdef SNMALLOC_PASS_THROUGH - external_alloc::free(p); + external_alloc::free(p_raw); #else // TODO: // Care is needed so that dealloc(nullptr) works before init @@ -461,6 +458,7 @@ namespace snmalloc // before init, that maps null to a remote_deallocator that will never be // in thread local state. + auto p = capptr_from_client(p_raw); const MetaEntry& entry = SharedStateHandle::Pagemap::get_metaentry( core_alloc->backend_state_ptr(), address_cast(p)); if (likely(local_cache.remote_allocator == entry.get_remote())) @@ -478,12 +476,10 @@ namespace snmalloc if (local_cache.remote_dealloc_cache.reserve_space(entry)) { local_cache.remote_dealloc_cache.template dealloc( - entry.get_remote()->trunc_id(), - capptr::AllocFull(p), - key_global); + entry.get_remote()->trunc_id(), p, key_global); # ifdef SNMALLOC_TRACING - std::cout << "Remote dealloc fast" << p << " size " << alloc_size(p) - << std::endl; + std::cout << "Remote dealloc fast" << p_raw << " size " + << alloc_size(p_raw) << std::endl; # endif return; } @@ -493,7 +489,7 @@ namespace snmalloc } // Large deallocation or null. - if (likely(p != nullptr)) + if (likely(p.unsafe_ptr() != nullptr)) { // Check this is managed by this pagemap. check_client(entry.get_sizeclass() != 0, "Not allocated by snmalloc."); @@ -511,7 +507,14 @@ namespace snmalloc # endif ChunkRecord* slab_record = reinterpret_cast(entry.get_metaslab()); - slab_record->chunk = capptr::Chunk(p); + /* + * StrictProvenance TODO: this is a subversive amplification. p is + * AllocWild-bounded, but we're coercing it to Chunk-bounded. We + * should, instead, not be storing ->chunk here, but should be keeping + * a CapPtr to this region internally even while it's + * allocated. + */ + slab_record->chunk = capptr::Chunk(p.unsafe_ptr()); check_init( []( CoreAlloc* core_alloc, diff --git a/src/mem/localcache.h b/src/mem/localcache.h index 8184141e2..d72648d67 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -13,20 +13,20 @@ namespace snmalloc using Stats = AllocStats; inline static SNMALLOC_FAST_PATH void* finish_alloc_no_zero( - FreeObject::HeadPtr p, + FreeObject::HeadPtr p, sizeclass_t sizeclass) { SNMALLOC_ASSERT(Metaslab::is_start_of_object(sizeclass, address_cast(p))); UNUSED(sizeclass); - auto r = capptr_reveal(capptr_to_user_address_control(p.as_void())); + auto r = capptr_reveal(p.as_void()); return r; } template inline static SNMALLOC_FAST_PATH void* finish_alloc( - FreeObject::HeadPtr p, + FreeObject::HeadPtr p, sizeclass_t sizeclass) { auto r = finish_alloc_no_zero(p, sizeclass); @@ -48,7 +48,7 @@ namespace snmalloc // Free list per small size class. These are used for // allocation on the fast path. This part of the code is inspired by // mimalloc. - FreeListIter + FreeListIter small_fast_free_lists[NUM_SIZECLASSES]; // This is the entropy for a particular thread. @@ -90,7 +90,8 @@ namespace snmalloc while (!small_fast_free_lists[i].empty()) { auto p = small_fast_free_lists[i].take(key); - dealloc(finish_alloc_no_zero(p, i)); + SNMALLOC_ASSERT(Metaslab::is_start_of_object(i, address_cast(p))); + dealloc(p.as_void()); } } diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 9a7165bb2..dca9dcae6 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -25,10 +25,10 @@ namespace snmalloc * Data-structure for building the free list for this slab. */ #ifdef SNMALLOC_CHECK_CLIENT - FreeListBuilder + FreeListBuilder free_queue; #else - FreeListBuilder + FreeListBuilder free_queue; #endif @@ -155,10 +155,10 @@ namespace snmalloc * available objects for this metaslab. */ static SNMALLOC_FAST_PATH - std::pair, bool> + std::pair, bool> alloc_free_list( Metaslab* meta, - FreeListIter& + FreeListIter& fast_free_list, LocalEntropy& entropy, sizeclass_t sizeclass) diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index 2be554f36..4b5ff1b9d 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -36,11 +36,11 @@ namespace snmalloc // Store the message queue on a separate cacheline. It is mutable data that // is read by other threads. alignas(CACHELINE_SIZE) - FreeObject::AtomicQueuePtr back{nullptr}; + FreeObject::AtomicQueuePtr back{nullptr}; // Store the two ends on different cache lines as access by different // threads. - alignas(CACHELINE_SIZE) - FreeObject::QueuePtr front{nullptr}; + alignas(CACHELINE_SIZE) FreeObject::QueuePtr front{ + nullptr}; constexpr RemoteAllocator() = default; @@ -50,7 +50,7 @@ namespace snmalloc SNMALLOC_ASSERT(front != nullptr); } - void init(FreeObject::QueuePtr stub) + void init(FreeObject::QueuePtr stub) { FreeObject::atomic_store_null(stub, key_global); front = stub; @@ -58,9 +58,9 @@ namespace snmalloc invariant(); } - FreeObject::QueuePtr destroy() + FreeObject::QueuePtr destroy() { - FreeObject::QueuePtr fnt = front; + FreeObject::QueuePtr fnt = front; back.store(nullptr, std::memory_order_relaxed); front = nullptr; return fnt; @@ -68,7 +68,7 @@ namespace snmalloc inline bool is_empty() { - FreeObject::QueuePtr bk = + FreeObject::QueuePtr bk = back.load(std::memory_order_relaxed); return bk == front; @@ -79,22 +79,22 @@ namespace snmalloc * last should be linked together through their next pointers. */ void enqueue( - FreeObject::QueuePtr first, - FreeObject::QueuePtr last, + FreeObject::QueuePtr first, + FreeObject::QueuePtr last, const FreeListKey& key) { invariant(); FreeObject::atomic_store_null(last, key); // exchange needs to be a release, so nullptr in next is visible. - FreeObject::QueuePtr prev = + FreeObject::QueuePtr prev = back.exchange(last, std::memory_order_release); // XXX prev is not known to be domesticated FreeObject::atomic_store_next(prev, first, key); } - FreeObject::QueuePtr peek() + FreeObject::QueuePtr peek() { return front; } @@ -103,16 +103,16 @@ namespace snmalloc * Returns the front message, or null if not possible to return a message. */ std::pair< - FreeObject::HeadPtr, + FreeObject::HeadPtr, bool> dequeue(const FreeListKey& key) { - auto domesticate = [](FreeObject::QueuePtr p) + auto domesticate = [](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { return p; }; invariant(); - FreeObject::HeadPtr - first = domesticate(front); - FreeObject::QueuePtr next = + FreeObject::HeadPtr first = + domesticate(front); + FreeObject::QueuePtr next = first->atomic_read_next(key, domesticate); if (next != nullptr) diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index 7bcc57a56..4f38ab90a 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -19,8 +19,8 @@ namespace snmalloc std::array< FreeListBuilder< false, - capptr::bounds::AllocFull, - capptr::bounds::AllocFull, + capptr::bounds::Alloc, + capptr::bounds::Alloc, false>, REMOTE_SLOTS> list; @@ -73,12 +73,12 @@ namespace snmalloc template SNMALLOC_FAST_PATH void dealloc( RemoteAllocator::alloc_id_t target_id, - capptr::AllocFull p, + capptr::Alloc p, const FreeListKey& key) { SNMALLOC_ASSERT(initialised); auto r = - p.template as_reinterpret>(); + p.template as_reinterpret>(); list[get_slot(target_id, 0)].add(r, key); } @@ -118,8 +118,7 @@ namespace snmalloc // Entries could map back onto the "resend" list, // so take copy of the head, mark the last element, // and clear the original list. - FreeListIter - resend; + FreeListIter resend; list[my_slot].close(resend, key); post_round++; From 1a608e6066c70ef5f2af0d0b048d45ec522efab9 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Mon, 16 Aug 2021 21:03:29 +0100 Subject: [PATCH 097/302] NFC: Introduce pointer domestication backend support `capptr_domesticate(Backend::LocalState*, CapPtr)` is the intended affordance for conversion to covert from a `CapPtr` with `B::wildness` `Wild` to a `CapPtr<>` with `B::with_wildness` and thence plumbed into the rest of the machinery. David added the SFINAE wrapper so that `Backend`-s now don't need to implement a domestication callback; instead, if the `Backend` does not provide a `capptr_domesticate` function, a default, which just does an explicit type cast, will be used instead. This is not yet hooked into the rest of the tree. Co-authored-by: David Chisnall --- src/backend/backend.h | 17 +++++++++++ src/backend/backend_concept.h | 19 ++++++++++++ src/backend/commonconfig.h | 51 ++++++++++++++++++++++++++++++++- src/backend/fixedglobalconfig.h | 28 ++++++++++++++++++ src/backend/pagemap.h | 9 ++++++ src/mem/localalloc.h | 10 +++++++ 6 files changed, 133 insertions(+), 1 deletion(-) diff --git a/src/backend/backend.h b/src/backend/backend.h index daf036393..36328f462 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -294,6 +294,23 @@ namespace snmalloc UNUSED(ls); concretePagemap.register_range(p, sz); } + + /** + * Return the bounds of the memory this back-end manages as a pair of + * addresses (start then end). This is available iff this is a + * fixed-range Backend. + */ + template + static SNMALLOC_FAST_PATH + std::enable_if_t> + get_bounds(LocalState* local_state) + { + static_assert( + fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); + + UNUSED(local_state); + return concretePagemap.get_bounds(); + } }; private: diff --git a/src/backend/backend_concept.h b/src/backend/backend_concept.h index 96c731900..e4cbc91e0 100644 --- a/src/backend/backend_concept.h +++ b/src/backend/backend_concept.h @@ -58,6 +58,24 @@ namespace snmalloc concept ConceptBackendMetaRange = ConceptBackendMeta && ConceptBackendMeta_Range; + /** + * The backend also defines domestication (that is, the difference between + * Tame and Wild CapPtr bounds). It exports the intended affordance for + * testing a Wild pointer and either returning nullptr or the original + * pointer, now Tame. + */ + template + concept ConceptBackendDomestication = + requires(typename Globals::LocalState* ls, + capptr::AllocWild ptr) + { + { Globals::capptr_domesticate(ls, ptr) } + -> ConceptSame>; + + { Globals::capptr_domesticate(ls, ptr.template as_static()) } + -> ConceptSame>; + }; + class CommonConfig; struct Flags; @@ -89,6 +107,7 @@ namespace snmalloc typename Globals::GlobalPoolState; { Globals::pool() } -> ConceptSame; + }; /** diff --git a/src/backend/commonconfig.h b/src/backend/commonconfig.h index 2e118799b..abc3333e4 100644 --- a/src/backend/commonconfig.h +++ b/src/backend/commonconfig.h @@ -1,7 +1,6 @@ #pragma once #include "../ds/defines.h" -#include "../mem/remotecache.h" namespace snmalloc { @@ -137,4 +136,54 @@ namespace snmalloc return true; } + namespace detail + { + /** + * SFINAE helper, calls capptr_domesticate in the backend if it exists. + */ + template< + SNMALLOC_CONCEPT(ConceptBackendDomestication) Backend, + typename T, + SNMALLOC_CONCEPT(capptr::ConceptBound) B> + SNMALLOC_FAST_PATH_INLINE auto + capptr_domesticate(typename Backend::LocalState* ls, CapPtr p, int) + -> decltype(Backend::capptr_domesticate(ls, p)) + { + return Backend::capptr_domesticate(ls, p); + } + + /** + * SFINAE helper. If the back end does not provide special handling for + * domestication then assume all wild pointers can be domesticated. + */ + template< + SNMALLOC_CONCEPT(ConceptBackendGlobals) Backend, + typename T, + SNMALLOC_CONCEPT(capptr::ConceptBound) B> + SNMALLOC_FAST_PATH_INLINE auto + capptr_domesticate(typename Backend::LocalState*, CapPtr p, long) + { + return CapPtr< + T, + typename B::template with_wildness>( + p.unsafe_ptr()); + } + } // namespace detail + + /** + * Wrapper that calls `Backend::capptr_domesticate` if and only if it is + * implemented. If it is not implemented then this assumes that any wild + * pointer can be domesticated. + */ + template< + SNMALLOC_CONCEPT(ConceptBackendGlobals) Backend, + typename T, + SNMALLOC_CONCEPT(capptr::ConceptBound) B> + SNMALLOC_FAST_PATH_INLINE auto + capptr_domesticate(typename Backend::LocalState* ls, CapPtr p) + { + return detail::capptr_domesticate(ls, p, 0); + } + } // namespace snmalloc +#include "../mem/remotecache.h" diff --git a/src/backend/fixedglobalconfig.h b/src/backend/fixedglobalconfig.h index 78647237f..761c371f3 100644 --- a/src/backend/fixedglobalconfig.h +++ b/src/backend/fixedglobalconfig.h @@ -51,5 +51,33 @@ namespace snmalloc { Backend::init(local_state, base, length); } + + /* Verify that a pointer points into the region managed by this config */ + template + static SNMALLOC_FAST_PATH CapPtr< + T, + typename B::template with_wildness> + capptr_domesticate(typename Backend::LocalState* ls, CapPtr p) + { + static_assert(B::wildness == capptr::dimension::Wildness::Wild); + + static const size_t sz = sizeof( + std::conditional, void>, void*, T>); + + UNUSED(ls); + auto address = address_cast(p); + auto bounds = Backend::Pagemap::get_bounds(nullptr); + if ( + (address < bounds.first) || (address > bounds.second) || + ((bounds.second - address) < sz)) + { + return nullptr; + } + + return CapPtr< + T, + typename B::template with_wildness>( + p.unsafe_ptr()); + } }; } diff --git a/src/backend/pagemap.h b/src/backend/pagemap.h index 72041624f..9c770f051 100644 --- a/src/backend/pagemap.h +++ b/src/backend/pagemap.h @@ -205,6 +205,15 @@ namespace snmalloc body_opt = new_body; } + template + std::enable_if_t> get_bounds() + { + static_assert( + has_bounds_ == has_bounds, "Don't set SFINAE template parameter!"); + + return {base, size}; + } + /** * Get the number of entries. */ diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 74d33b9df..2e252d6dc 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -569,6 +569,11 @@ namespace snmalloc #ifdef SNMALLOC_PASS_THROUGH return external_alloc::malloc_usable_size(const_cast(p_raw)); #else + // TODO What's the domestication policy here? At the moment we just probe + // the pagemap with the raw address, without checks. There could be + // implicit domestication through the `SharedStateHandle::Pagemap` or we + // could just leave well enough alone. + // Note that this should return 0 for nullptr. // Other than nullptr, we know the system will be initialised as it must // be called with something we have already allocated. @@ -600,6 +605,11 @@ namespace snmalloc void* external_pointer(void* p_raw) { #ifndef SNMALLOC_PASS_THROUGH + // TODO What's the domestication policy here? At the moment we just probe + // the pagemap with the raw address, without checks. There could be + // implicit domestication through the `SharedStateHandle::Pagemap` or we + // could just leave well enough alone. + // TODO bring back the CHERI bits. Wes to review if required. MetaEntry entry = SharedStateHandle::Pagemap::template get_metaentry( From a34b7a59739a22be51948a4f8a6b7c74c61736e8 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Mon, 16 Aug 2021 21:03:29 +0100 Subject: [PATCH 098/302] Make capptr_from_client return a Wild CapPtr Chase consequences in dealloc(). --- src/ds/ptrwrap.h | 8 ++++---- src/mem/localalloc.h | 37 +++++++++++++++++++++++++++---------- 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/src/ds/ptrwrap.h b/src/ds/ptrwrap.h index b08745e1a..6a0b024e1 100644 --- a/src/ds/ptrwrap.h +++ b/src/ds/ptrwrap.h @@ -362,13 +362,13 @@ namespace snmalloc } /** - * Given a void* from the client, it's fine to call it Alloc. Roughly - * dual to capptr_reveal(). + * Given a void* from the client, it's fine to call it AllocWild. + * Roughly dual to capptr_reveal(). */ - static inline SNMALLOC_FAST_PATH capptr::Alloc + static inline SNMALLOC_FAST_PATH capptr::AllocWild capptr_from_client(void* p) { - return capptr::Alloc(p); + return capptr::AllocWild(p); } /** diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 2e252d6dc..0aa70627a 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -458,13 +458,29 @@ namespace snmalloc // before init, that maps null to a remote_deallocator that will never be // in thread local state. - auto p = capptr_from_client(p_raw); + capptr::AllocWild p_wild = capptr_from_client(p_raw); + + /* + * p_tame may be nullptr, even if p_raw/p_wild are not, in the case where + * domestication fails. We exclusively use p_tame below so that such + * failures become no ops; in the nullptr path, which should be well off + * the fast path, we could be slightly more aggressive and test that p_raw + * is also nullptr and Pal::error() if not. (TODO) + * + * We do not rely on the bounds-checking ability of domestication here, + * and just check the address (and, on other architectures, perhaps + * well-formedness) of this pointer. The remainder of the logic will + * deal with the object's extent. + */ + capptr::Alloc p_tame = capptr_domesticate( + core_alloc->backend_state_ptr(), p_wild); + const MetaEntry& entry = SharedStateHandle::Pagemap::get_metaentry( - core_alloc->backend_state_ptr(), address_cast(p)); + core_alloc->backend_state_ptr(), address_cast(p_tame)); if (likely(local_cache.remote_allocator == entry.get_remote())) { if (likely(CoreAlloc::dealloc_local_object_fast( - entry, p, local_cache.entropy))) + entry, p_tame, local_cache.entropy))) return; core_alloc->dealloc_local_object_slow(entry); return; @@ -476,7 +492,7 @@ namespace snmalloc if (local_cache.remote_dealloc_cache.reserve_space(entry)) { local_cache.remote_dealloc_cache.template dealloc( - entry.get_remote()->trunc_id(), p, key_global); + entry.get_remote()->trunc_id(), p_tame, key_global); # ifdef SNMALLOC_TRACING std::cout << "Remote dealloc fast" << p_raw << " size " << alloc_size(p_raw) << std::endl; @@ -484,12 +500,12 @@ namespace snmalloc return; } - dealloc_remote_slow(p); + dealloc_remote_slow(p_tame); return; } // Large deallocation or null. - if (likely(p.unsafe_ptr() != nullptr)) + if (likely(p_tame != nullptr)) { // Check this is managed by this pagemap. check_client(entry.get_sizeclass() != 0, "Not allocated by snmalloc."); @@ -498,7 +514,8 @@ namespace snmalloc // Check for start of allocation. check_client( - pointer_align_down(p, size) == p, "Not start of an allocation."); + pointer_align_down(p_tame, size) == p_tame, + "Not start of an allocation."); size_t slab_sizeclass = large_size_to_chunk_sizeclass(size); # ifdef SNMALLOC_TRACING @@ -508,13 +525,13 @@ namespace snmalloc ChunkRecord* slab_record = reinterpret_cast(entry.get_metaslab()); /* - * StrictProvenance TODO: this is a subversive amplification. p is - * AllocWild-bounded, but we're coercing it to Chunk-bounded. We + * StrictProvenance TODO: this is a subversive amplification. p_tame is + * tame but Alloc-bounded, but we're coercing it to Chunk-bounded. We * should, instead, not be storing ->chunk here, but should be keeping * a CapPtr to this region internally even while it's * allocated. */ - slab_record->chunk = capptr::Chunk(p.unsafe_ptr()); + slab_record->chunk = capptr::Chunk(p_tame.unsafe_ptr()); check_init( []( CoreAlloc* core_alloc, From 77506765989bd4ce456e7d60844e2471b6569efd Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Mon, 13 Sep 2021 20:56:34 +0100 Subject: [PATCH 099/302] NFC: FreeListIter domestication plumbing Just an intermediate syntactic step to chase dependencies. All these introduced "domestication" callbacks are just the identity function, but they will let us thread the LocalAlloc's handle to the Backend state down to where it's needed. --- src/mem/corealloc.h | 26 ++++++++++++++++++-------- src/mem/freelist.h | 7 +++---- src/mem/localalloc.h | 5 ++++- src/mem/localcache.h | 16 +++++++++++----- src/mem/metaslab.h | 4 +++- src/mem/remotecache.h | 4 +++- 6 files changed, 42 insertions(+), 20 deletions(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 06858f84f..f66f50464 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -180,8 +180,11 @@ namespace snmalloc void* small_alloc_one(size_t size) { SNMALLOC_ASSERT(attached_cache != nullptr); + auto domesticate = [](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { return p; }; // Use attached cache, and fill it if it is empty. return attached_cache->template alloc( + domesticate, size, [&]( sizeclass_t sizeclass, @@ -278,7 +281,9 @@ namespace snmalloc FreeListIter fl; auto more = meta->free_queue.close(fl, key); UNUSED(more); - void* p = finish_alloc_no_zero(fl.take(key), sizeclass); + auto domesticate = [](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { return p; }; + void* p = finish_alloc_no_zero(fl.take(key, domesticate), sizeclass); #ifdef SNMALLOC_CHECK_CLIENT // Check free list is well-formed on platforms with @@ -286,7 +291,7 @@ namespace snmalloc size_t count = 1; // Already taken one above. while (!fl.empty()) { - fl.take(key); + fl.take(key, domesticate); count++; } // Check the list contains all the elements @@ -301,7 +306,7 @@ namespace snmalloc while (!fl.empty()) { - fl.take(key); + fl.take(key, domesticate); count++; } } @@ -634,8 +639,10 @@ namespace snmalloc if (meta->needed() == 0) alloc_classes[sizeclass].unused--; - auto [p, still_active] = - Metaslab::alloc_free_list(meta, fast_free_list, entropy, sizeclass); + auto domesticate = [](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { return p; }; + auto [p, still_active] = Metaslab::alloc_free_list( + domesticate, meta, fast_free_list, entropy, sizeclass); if (still_active) { @@ -701,8 +708,10 @@ namespace snmalloc // Build a free list for the slab alloc_new_list(slab, meta, rsize, slab_size, entropy); - auto [p, still_active] = - Metaslab::alloc_free_list(meta, fast_free_list, entropy, sizeclass); + auto domesticate = [](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { return p; }; + auto [p, still_active] = Metaslab::alloc_free_list( + domesticate, meta, fast_free_list, entropy, sizeclass); if (still_active) { @@ -750,7 +759,8 @@ namespace snmalloc auto posted = attached_cache->flush( - backend_state_ptr(), [&](auto p) { dealloc_local_object(p); }); + backend_state_ptr(), + [&](capptr::Alloc p) { dealloc_local_object(p); }); // We may now have unused slabs, return to the global allocator. for (sizeclass_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++) diff --git a/src/mem/freelist.h b/src/mem/freelist.h index fadf81e41..8d6ad07c6 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -416,12 +416,11 @@ namespace snmalloc /** * Moves the iterator on, and returns the current value. */ - FreeObject::HeadPtr take(const FreeListKey& key) + template + FreeObject::HeadPtr + take(const FreeListKey& key, Domesticator domesticate) { auto c = curr; - // TODO: Placeholder - auto domesticate = [](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { return p; }; auto next = curr->read_next(key, domesticate); Aal::prefetch(next.unsafe_ptr()); diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 0aa70627a..9d1c5f2ef 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -211,6 +211,9 @@ namespace snmalloc SNMALLOC_FAST_PATH void* small_alloc(size_t size) { // SNMALLOC_ASSUME(size <= sizeclass_to_size(NUM_SIZECLASSES)); + + auto domesticate = [](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { return p; }; auto slowpath = [&]( sizeclass_t sizeclass, @@ -239,7 +242,7 @@ namespace snmalloc }; return local_cache.template alloc( - size, slowpath); + domesticate, size, slowpath); } /** diff --git a/src/mem/localcache.h b/src/mem/localcache.h index d72648d67..e146208ba 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -82,6 +82,8 @@ namespace snmalloc typename SharedStateHandle::LocalState* local_state, DeallocFun dealloc) { auto& key = entropy.get_free_list_key(); + auto domesticate = [](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { return p; }; for (size_t i = 0; i < NUM_SIZECLASSES; i++) { @@ -89,7 +91,7 @@ namespace snmalloc // call. while (!small_fast_free_lists[i].empty()) { - auto p = small_fast_free_lists[i].take(key); + auto p = small_fast_free_lists[i].take(key, domesticate); SNMALLOC_ASSERT(Metaslab::is_start_of_object(i, address_cast(p))); dealloc(p.as_void()); } @@ -99,18 +101,22 @@ namespace snmalloc local_state, remote_allocator->trunc_id(), key_global); } - template - SNMALLOC_FAST_PATH void* alloc(size_t size, Slowpath slowpath) + template< + ZeroMem zero_mem, + typename SharedStateHandle, + typename Slowpath, + typename Domesticator> + SNMALLOC_FAST_PATH void* + alloc(Domesticator domesticate, size_t size, Slowpath slowpath) { auto& key = entropy.get_free_list_key(); - sizeclass_t sizeclass = size_to_sizeclass(size); stats.alloc_request(size); stats.sizeclass_alloc(sizeclass); auto& fl = small_fast_free_lists[sizeclass]; if (likely(!fl.empty())) { - auto p = fl.take(key); + auto p = fl.take(key, domesticate); return finish_alloc(p, sizeclass); } return slowpath(sizeclass, &fl); diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index dca9dcae6..015dff179 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -154,9 +154,11 @@ namespace snmalloc * component, but with randomisation, it may only return part of the * available objects for this metaslab. */ + template static SNMALLOC_FAST_PATH std::pair, bool> alloc_free_list( + Domesticator domesticate, Metaslab* meta, FreeListIter& fast_free_list, @@ -167,7 +169,7 @@ namespace snmalloc std::remove_reference_t tmp_fl; auto remaining = meta->free_queue.close(tmp_fl, key); - auto p = tmp_fl.take(key); + auto p = tmp_fl.take(key, domesticate); fast_free_list = tmp_fl; #ifdef SNMALLOC_CHECK_CLIENT diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index 4f38ab90a..039d83d1a 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -92,6 +92,8 @@ namespace snmalloc SNMALLOC_ASSERT(initialised); size_t post_round = 0; bool sent_something = false; + auto domesticate = [](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { return p; }; while (true) { @@ -127,7 +129,7 @@ namespace snmalloc { // Use the next N bits to spread out remote deallocs in our own // slot. - auto r = resend.take(key); + auto r = resend.take(key, domesticate); MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry( local_state, address_cast(r)); auto i = entry.get_remote()->trunc_id(); From d4c120dfe5af7cbfc9b1c05de244f3238e3a0b32 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Wed, 15 Sep 2021 00:19:39 +0100 Subject: [PATCH 100/302] Free queues hold Wild pointers --- src/mem/corealloc.h | 51 ++++++++++++++++-------- src/mem/localalloc.h | 95 +++++++++++++++++++++++--------------------- src/mem/localcache.h | 21 ++++++---- src/mem/metaslab.h | 23 ++++++----- 4 files changed, 109 insertions(+), 81 deletions(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index f66f50464..c882912ec 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -180,15 +180,19 @@ namespace snmalloc void* small_alloc_one(size_t size) { SNMALLOC_ASSERT(attached_cache != nullptr); - auto domesticate = [](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { return p; }; + auto domesticate = + [this](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate( + backend_state_ptr(), p); + }; // Use attached cache, and fill it if it is empty. return attached_cache->template alloc( domesticate, size, [&]( sizeclass_t sizeclass, - FreeListIter* fl) { + FreeListIter* fl) { return small_alloc(sizeclass, *fl); }); } @@ -252,7 +256,8 @@ namespace snmalloc do { b.add( - FreeObject::make( + // Here begins our treatment of the heap as containing Wild pointers + FreeObject::make( capptr_to_user_address_control(curr_ptr.as_void())), key, entropy); @@ -263,7 +268,8 @@ namespace snmalloc do { b.add( - FreeObject::make( + // Here begins our treatment of the heap as containing Wild pointers + FreeObject::make( capptr_to_user_address_control( Aal::capptr_bound( p.as_void(), rsize))), @@ -278,11 +284,15 @@ namespace snmalloc ChunkRecord* clear_slab(Metaslab* meta, sizeclass_t sizeclass) { auto& key = entropy.get_free_list_key(); - FreeListIter fl; + FreeListIter fl; auto more = meta->free_queue.close(fl, key); UNUSED(more); - auto domesticate = [](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { return p; }; + auto local_state = backend_state_ptr(); + auto domesticate = + [local_state](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(local_state, p); + }; void* p = finish_alloc_no_zero(fl.take(key, domesticate), sizeclass); #ifdef SNMALLOC_CHECK_CLIENT @@ -601,7 +611,7 @@ namespace snmalloc Metaslab::is_start_of_object(entry.get_sizeclass(), address_cast(p)), "Not deallocating start of an object"); - auto cp = p.as_static>(); + auto cp = p.as_static>(); auto& key = entropy.get_free_list_key(); @@ -614,7 +624,7 @@ namespace snmalloc template SNMALLOC_SLOW_PATH void* small_alloc( sizeclass_t sizeclass, - FreeListIter& + FreeListIter& fast_free_list) { size_t rsize = sizeclass_to_size(sizeclass); @@ -639,8 +649,12 @@ namespace snmalloc if (meta->needed() == 0) alloc_classes[sizeclass].unused--; - auto domesticate = [](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { return p; }; + auto domesticate = + [this](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate( + backend_state_ptr(), p); + }; auto [p, still_active] = Metaslab::alloc_free_list( domesticate, meta, fast_free_list, entropy, sizeclass); @@ -676,7 +690,7 @@ namespace snmalloc template SNMALLOC_SLOW_PATH void* small_alloc_slow( sizeclass_t sizeclass, - FreeListIter& + FreeListIter& fast_free_list, size_t rsize) { @@ -708,8 +722,12 @@ namespace snmalloc // Build a free list for the slab alloc_new_list(slab, meta, rsize, slab_size, entropy); - auto domesticate = [](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { return p; }; + auto domesticate = + [this](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate( + backend_state_ptr(), p); + }; auto [p, still_active] = Metaslab::alloc_free_list( domesticate, meta, fast_free_list, entropy, sizeclass); @@ -745,7 +763,8 @@ namespace snmalloc auto& entry = SharedStateHandle::Pagemap::get_metaentry( backend_state_ptr(), snmalloc::address_cast(p)); handle_dealloc_remote(entry, p.as_void(), need_post); - // XXX n is not known to be domesticated + // TODO p = SharedStateHandle::capptr_domesticate(backend_state_ptr(), + // n); p = n; } } diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 9d1c5f2ef..83c78c610 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -211,35 +211,38 @@ namespace snmalloc SNMALLOC_FAST_PATH void* small_alloc(size_t size) { // SNMALLOC_ASSUME(size <= sizeclass_to_size(NUM_SIZECLASSES)); - - auto domesticate = [](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { return p; }; + auto domesticate = + [this](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate( + core_alloc->backend_state_ptr(), p); + }; auto slowpath = [&]( sizeclass_t sizeclass, - FreeListIter* fl) - SNMALLOC_FAST_PATH_LAMBDA { - if (likely(core_alloc != nullptr)) - { - return core_alloc->handle_message_queue( - []( - CoreAlloc* core_alloc, - sizeclass_t sizeclass, - FreeListIter* - fl) { - return core_alloc->template small_alloc( - sizeclass, *fl); - }, - core_alloc, - sizeclass, - fl); - } - return lazy_init( - [&](CoreAlloc*, sizeclass_t sizeclass) { - return small_alloc(sizeclass_to_size(sizeclass)); + FreeListIter* + fl) SNMALLOC_FAST_PATH_LAMBDA { + if (likely(core_alloc != nullptr)) + { + return core_alloc->handle_message_queue( + []( + CoreAlloc* core_alloc, + sizeclass_t sizeclass, + FreeListIter* + fl) { + return core_alloc->template small_alloc( + sizeclass, *fl); }, - sizeclass); - }; + core_alloc, + sizeclass, + fl); + } + return lazy_init( + [&](CoreAlloc*, sizeclass_t sizeclass) { + return small_alloc(sizeclass_to_size(sizeclass)); + }, + sizeclass); + }; return local_cache.template alloc( domesticate, size, slowpath); @@ -278,8 +281,8 @@ namespace snmalloc return; } - // Recheck what kind of dealloc we should do incase, the allocator we get - // from lazy_init is the originating allocator. + // Recheck what kind of dealloc we should do incase, the allocator we + // get from lazy_init is the originating allocator. lazy_init( [&](CoreAlloc*, CapPtr p) { dealloc(p.unsafe_ptr()); // TODO don't double count statistics @@ -327,8 +330,8 @@ namespace snmalloc {} /** - * Call `SharedStateHandle::ensure_init()` if it is implemented, do nothing - * otherwise. + * Call `SharedStateHandle::ensure_init()` if it is implemented, do + * nothing otherwise. */ SNMALLOC_FAST_PATH void ensure_init() @@ -458,17 +461,17 @@ namespace snmalloc // TODO: // Care is needed so that dealloc(nullptr) works before init // The backend allocator must ensure that a minimal page map exists - // before init, that maps null to a remote_deallocator that will never be - // in thread local state. + // before init, that maps null to a remote_deallocator that will never + // be in thread local state. capptr::AllocWild p_wild = capptr_from_client(p_raw); /* - * p_tame may be nullptr, even if p_raw/p_wild are not, in the case where - * domestication fails. We exclusively use p_tame below so that such - * failures become no ops; in the nullptr path, which should be well off - * the fast path, we could be slightly more aggressive and test that p_raw - * is also nullptr and Pal::error() if not. (TODO) + * p_tame may be nullptr, even if p_raw/p_wild are not, in the case + * where domestication fails. We exclusively use p_tame below so that + * such failures become no ops; in the nullptr path, which should be + * well off the fast path, we could be slightly more aggressive and test + * that p_raw is also nullptr and Pal::error() if not. (TODO) * * We do not rely on the bounds-checking ability of domestication here, * and just check the address (and, on other architectures, perhaps @@ -589,17 +592,17 @@ namespace snmalloc #ifdef SNMALLOC_PASS_THROUGH return external_alloc::malloc_usable_size(const_cast(p_raw)); #else - // TODO What's the domestication policy here? At the moment we just probe - // the pagemap with the raw address, without checks. There could be - // implicit domestication through the `SharedStateHandle::Pagemap` or we - // could just leave well enough alone. + // TODO What's the domestication policy here? At the moment we just + // probe the pagemap with the raw address, without checks. There could + // be implicit domestication through the `SharedStateHandle::Pagemap` or + // we could just leave well enough alone. // Note that this should return 0 for nullptr. // Other than nullptr, we know the system will be initialised as it must // be called with something we have already allocated. // To handle this case we require the uninitialised pagemap contain an - // entry for the first chunk of memory, that states it represents a large - // object, so we can pull the check for null off the fast path. + // entry for the first chunk of memory, that states it represents a + // large object, so we can pull the check for null off the fast path. MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry( core_alloc->backend_state_ptr(), address_cast(p_raw)); @@ -625,10 +628,10 @@ namespace snmalloc void* external_pointer(void* p_raw) { #ifndef SNMALLOC_PASS_THROUGH - // TODO What's the domestication policy here? At the moment we just probe - // the pagemap with the raw address, without checks. There could be - // implicit domestication through the `SharedStateHandle::Pagemap` or we - // could just leave well enough alone. + // TODO What's the domestication policy here? At the moment we just + // probe the pagemap with the raw address, without checks. There could + // be implicit domestication through the `SharedStateHandle::Pagemap` or + // we could just leave well enough alone. // TODO bring back the CHERI bits. Wes to review if required. MetaEntry entry = diff --git a/src/mem/localcache.h b/src/mem/localcache.h index e146208ba..0f54dc771 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -12,9 +12,9 @@ namespace snmalloc { using Stats = AllocStats; + template inline static SNMALLOC_FAST_PATH void* finish_alloc_no_zero( - FreeObject::HeadPtr p, - sizeclass_t sizeclass) + FreeObject::HeadPtr p, sizeclass_t sizeclass) { SNMALLOC_ASSERT(Metaslab::is_start_of_object(sizeclass, address_cast(p))); UNUSED(sizeclass); @@ -24,10 +24,12 @@ namespace snmalloc return r; } - template + template< + ZeroMem zero_mem, + typename SharedStateHandle, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> inline static SNMALLOC_FAST_PATH void* finish_alloc( - FreeObject::HeadPtr p, - sizeclass_t sizeclass) + FreeObject::HeadPtr p, sizeclass_t sizeclass) { auto r = finish_alloc_no_zero(p, sizeclass); @@ -48,7 +50,7 @@ namespace snmalloc // Free list per small size class. These are used for // allocation on the fast path. This part of the code is inspired by // mimalloc. - FreeListIter + FreeListIter small_fast_free_lists[NUM_SIZECLASSES]; // This is the entropy for a particular thread. @@ -82,8 +84,11 @@ namespace snmalloc typename SharedStateHandle::LocalState* local_state, DeallocFun dealloc) { auto& key = entropy.get_free_list_key(); - auto domesticate = [](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { return p; }; + auto domesticate = + [local_state](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(local_state, p); + }; for (size_t i = 0; i < NUM_SIZECLASSES; i++) { diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 015dff179..4be0499d5 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -25,10 +25,10 @@ namespace snmalloc * Data-structure for building the free list for this slab. */ #ifdef SNMALLOC_CHECK_CLIENT - FreeListBuilder + FreeListBuilder free_queue; #else - FreeListBuilder + FreeListBuilder free_queue; #endif @@ -155,15 +155,16 @@ namespace snmalloc * available objects for this metaslab. */ template - static SNMALLOC_FAST_PATH - std::pair, bool> - alloc_free_list( - Domesticator domesticate, - Metaslab* meta, - FreeListIter& - fast_free_list, - LocalEntropy& entropy, - sizeclass_t sizeclass) + static SNMALLOC_FAST_PATH std::pair< + FreeObject::HeadPtr, + bool> + alloc_free_list( + Domesticator domesticate, + Metaslab* meta, + FreeListIter& + fast_free_list, + LocalEntropy& entropy, + sizeclass_t sizeclass) { auto& key = entropy.get_free_list_key(); From 94ab856ff5378c6b600c1304cb326ce3f1651df8 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Wed, 15 Sep 2021 00:05:16 +0100 Subject: [PATCH 101/302] NFC: remote queue domestication plumbing These are, at present, just identity functions in the right places. --- src/mem/corealloc.h | 4 +++- src/mem/remoteallocator.h | 5 ++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index c882912ec..77363f6ab 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -421,13 +421,15 @@ namespace snmalloc handle_message_queue_inner(Action action, Args... args) { bool need_post = false; + auto domesticate = [](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { return p; }; for (size_t i = 0; i < REMOTE_BATCH; i++) { auto p = message_queue().peek(); auto& entry = SharedStateHandle::Pagemap::get_metaentry( backend_state_ptr(), snmalloc::address_cast(p)); - auto r = message_queue().dequeue(key_global); + auto r = message_queue().dequeue(key_global, domesticate); if (unlikely(!r.second)) break; diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index 4b5ff1b9d..bce63529f 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -102,13 +102,12 @@ namespace snmalloc /** * Returns the front message, or null if not possible to return a message. */ + template std::pair< FreeObject::HeadPtr, bool> - dequeue(const FreeListKey& key) + dequeue(const FreeListKey& key, Domesticator domesticate) { - auto domesticate = [](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { return p; }; invariant(); FreeObject::HeadPtr first = domesticate(front); From 501c14661fd375d2fc7e52ab5078c5a4ba0ec178 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Tue, 28 Sep 2021 15:47:15 +0100 Subject: [PATCH 102/302] Remote queues hold Wild pointers --- src/mem/corealloc.h | 36 ++++++++++++++---------- src/mem/remoteallocator.h | 59 ++++++++++++++++++++++++--------------- src/mem/remotecache.h | 15 ++++++---- 3 files changed, 66 insertions(+), 44 deletions(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 77363f6ab..8d84c91f8 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -163,7 +163,7 @@ namespace snmalloc // Using an actual allocation removes a conditional from a critical path. auto dummy = capptr::Alloc(small_alloc_one(MIN_ALLOC_SIZE)) - .template as_static>(); + .template as_static>(); if (dummy == nullptr) { error("Critical error: Out-of-memory during initialisation."); @@ -421,13 +421,17 @@ namespace snmalloc handle_message_queue_inner(Action action, Args... args) { bool need_post = false; - auto domesticate = [](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { return p; }; + auto local_state = backend_state_ptr(); + auto domesticate = + [local_state](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(local_state, p); + }; for (size_t i = 0; i < REMOTE_BATCH; i++) { auto p = message_queue().peek(); auto& entry = SharedStateHandle::Pagemap::get_metaentry( - backend_state_ptr(), snmalloc::address_cast(p)); + local_state, snmalloc::address_cast(p)); auto r = message_queue().dequeue(key_global, domesticate); @@ -750,24 +754,26 @@ namespace snmalloc bool flush(bool destroy_queue = false) { SNMALLOC_ASSERT(attached_cache != nullptr); - // TODO: Placeholder - auto domesticate = [](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { return p; }; + auto local_state = backend_state_ptr(); + auto domesticate = + [local_state](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(local_state, p); + }; if (destroy_queue) { - auto p = message_queue().destroy(); + auto p_wild = message_queue().destroy(); + auto p_tame = domesticate(p_wild); - while (p != nullptr) + while (p_tame != nullptr) { bool need_post = true; // Always going to post, so ignore. - auto n = p->atomic_read_next(key_global, domesticate); + auto n_tame = p_tame->atomic_read_next(key_global, domesticate); auto& entry = SharedStateHandle::Pagemap::get_metaentry( - backend_state_ptr(), snmalloc::address_cast(p)); - handle_dealloc_remote(entry, p.as_void(), need_post); - // TODO p = SharedStateHandle::capptr_domesticate(backend_state_ptr(), - // n); - p = n; + backend_state_ptr(), snmalloc::address_cast(p_tame)); + handle_dealloc_remote(entry, p_tame.as_void(), need_post); + p_tame = n_tame; } } else diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index bce63529f..c658c2eea 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -36,11 +36,11 @@ namespace snmalloc // Store the message queue on a separate cacheline. It is mutable data that // is read by other threads. alignas(CACHELINE_SIZE) - FreeObject::AtomicQueuePtr back{nullptr}; + FreeObject::AtomicQueuePtr back{nullptr}; // Store the two ends on different cache lines as access by different // threads. - alignas(CACHELINE_SIZE) FreeObject::QueuePtr front{ - nullptr}; + alignas(CACHELINE_SIZE) + FreeObject::QueuePtr front{nullptr}; constexpr RemoteAllocator() = default; @@ -50,17 +50,19 @@ namespace snmalloc SNMALLOC_ASSERT(front != nullptr); } - void init(FreeObject::QueuePtr stub) + void + init(FreeObject::HeadPtr + stub) { FreeObject::atomic_store_null(stub, key_global); - front = stub; - back.store(stub, std::memory_order_relaxed); + front = capptr_rewild(stub); + back.store(front, std::memory_order_relaxed); invariant(); } - FreeObject::QueuePtr destroy() + FreeObject::QueuePtr destroy() { - FreeObject::QueuePtr fnt = front; + FreeObject::QueuePtr fnt = front; back.store(nullptr, std::memory_order_relaxed); front = nullptr; return fnt; @@ -68,7 +70,7 @@ namespace snmalloc inline bool is_empty() { - FreeObject::QueuePtr bk = + FreeObject::QueuePtr bk = back.load(std::memory_order_relaxed); return bk == front; @@ -78,23 +80,26 @@ namespace snmalloc * Pushes a list of messages to the queue. Each message from first to * last should be linked together through their next pointers. */ + template void enqueue( - FreeObject::QueuePtr first, - FreeObject::QueuePtr last, - const FreeListKey& key) + FreeObject::HeadPtr + first, + FreeObject::HeadPtr + last, + const FreeListKey& key, + Domesticator domesticate) { invariant(); FreeObject::atomic_store_null(last, key); // exchange needs to be a release, so nullptr in next is visible. - FreeObject::QueuePtr prev = - back.exchange(last, std::memory_order_release); + FreeObject::QueuePtr prev = + back.exchange(capptr_rewild(last), std::memory_order_release); - // XXX prev is not known to be domesticated - FreeObject::atomic_store_next(prev, first, key); + FreeObject::atomic_store_next(domesticate(prev), first, key); } - FreeObject::QueuePtr peek() + FreeObject::QueuePtr peek() { return front; } @@ -104,19 +109,27 @@ namespace snmalloc */ template std::pair< - FreeObject::HeadPtr, + FreeObject::HeadPtr, bool> dequeue(const FreeListKey& key, Domesticator domesticate) { invariant(); - FreeObject::HeadPtr first = - domesticate(front); - FreeObject::QueuePtr next = - first->atomic_read_next(key, domesticate); + FreeObject::HeadPtr + first = domesticate(front); + FreeObject::HeadPtr + next = first->atomic_read_next(key, domesticate); if (next != nullptr) { - front = next; + /* + * We've domesticate_queue-d next so that we can read through it, but + * we're storing it back into client-accessible memory in + * !QueueHeadsAreTame builds, so go ahead and consider it Wild again. + * On QueueHeadsAreTame builds, the subsequent domesticate_head call + * above will also be a type-level sleight of hand, but we can still + * justify it by the domesticate_queue that happened in this dequeue(). + */ + front = capptr_rewild(next); invariant(); return {first, true}; } diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index 039d83d1a..0d20d8969 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -20,7 +20,7 @@ namespace snmalloc FreeListBuilder< false, capptr::bounds::Alloc, - capptr::bounds::Alloc, + capptr::bounds::AllocWild, false>, REMOTE_SLOTS> list; @@ -78,7 +78,7 @@ namespace snmalloc { SNMALLOC_ASSERT(initialised); auto r = - p.template as_reinterpret>(); + p.template as_reinterpret>(); list[get_slot(target_id, 0)].add(r, key); } @@ -92,8 +92,11 @@ namespace snmalloc SNMALLOC_ASSERT(initialised); size_t post_round = 0; bool sent_something = false; - auto domesticate = [](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { return p; }; + auto domesticate = + [local_state](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(local_state, p); + }; while (true) { @@ -109,7 +112,7 @@ namespace snmalloc auto [first, last] = list[i].extract_segment(key); MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry( local_state, address_cast(first)); - entry.get_remote()->enqueue(first, last, key); + entry.get_remote()->enqueue(first, last, key, domesticate); sent_something = true; } } @@ -120,7 +123,7 @@ namespace snmalloc // Entries could map back onto the "resend" list, // so take copy of the head, mark the last element, // and clear the original list. - FreeListIter resend; + FreeListIter resend; list[my_slot].close(resend, key); post_round++; From 7deb3b61da1cd8f4ae50cc12623e22696bd1317e Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Wed, 6 Oct 2021 14:01:44 +0100 Subject: [PATCH 103/302] Fix builds on MSVC 2016 Without this it complains that LocalCache's constructor can't be constexpr because small_fast_free_lists isn't initialized. It's not clear to me why it didn't mind before, and nobody else seems to mind now, but this shouldn't break anyone else, either. --- src/mem/localcache.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mem/localcache.h b/src/mem/localcache.h index 0f54dc771..4d3d55a49 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -51,7 +51,7 @@ namespace snmalloc // allocation on the fast path. This part of the code is inspired by // mimalloc. FreeListIter - small_fast_free_lists[NUM_SIZECLASSES]; + small_fast_free_lists[NUM_SIZECLASSES] = {}; // This is the entropy for a particular thread. LocalEntropy entropy; From bc365e0abb9c798f3bab3ede3d4d11e32039c713 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Fri, 8 Oct 2021 13:57:48 +0100 Subject: [PATCH 104/302] Default template args for FreeObject & friends Now that explicit annotations have gotten us through the refactoring, it's time for the scaffolding to disappear. src/mem/freelist.h is left generic for any future machinations, but `FreeObject::T<>`, the several `FreeObject::...Ptr<>`s, `FreeListIter<>`, and `FreeListBuilder<>` are given default parameters and all uses are shortened to use defaults where possible. --- src/mem/corealloc.h | 73 ++++++++++---------------- src/mem/freelist.h | 105 +++++++++++++++++++++----------------- src/mem/localalloc.h | 57 ++++++++++----------- src/mem/localcache.h | 24 ++++----- src/mem/metaslab.h | 13 ++--- src/mem/remoteallocator.h | 37 +++++--------- src/mem/remotecache.h | 21 +++----- 7 files changed, 143 insertions(+), 187 deletions(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 8d84c91f8..611827ca0 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -161,9 +161,8 @@ namespace snmalloc { // Manufacture an allocation to prime the queue // Using an actual allocation removes a conditional from a critical path. - auto dummy = - capptr::Alloc(small_alloc_one(MIN_ALLOC_SIZE)) - .template as_static>(); + auto dummy = capptr::Alloc(small_alloc_one(MIN_ALLOC_SIZE)) + .template as_static>(); if (dummy == nullptr) { error("Critical error: Out-of-memory during initialisation."); @@ -181,18 +180,12 @@ namespace snmalloc { SNMALLOC_ASSERT(attached_cache != nullptr); auto domesticate = - [this](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { - return capptr_domesticate( - backend_state_ptr(), p); - }; + [this](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(backend_state_ptr(), p); + }; // Use attached cache, and fill it if it is empty. return attached_cache->template alloc( - domesticate, - size, - [&]( - sizeclass_t sizeclass, - FreeListIter* fl) { + domesticate, size, [&](sizeclass_t sizeclass, FreeListIter<>* fl) { return small_alloc(sizeclass, *fl); }); } @@ -284,15 +277,14 @@ namespace snmalloc ChunkRecord* clear_slab(Metaslab* meta, sizeclass_t sizeclass) { auto& key = entropy.get_free_list_key(); - FreeListIter fl; + FreeListIter<> fl; auto more = meta->free_queue.close(fl, key); UNUSED(more); auto local_state = backend_state_ptr(); auto domesticate = - [local_state](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { - return capptr_domesticate(local_state, p); - }; + [local_state](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(local_state, p); + }; void* p = finish_alloc_no_zero(fl.take(key, domesticate), sizeclass); #ifdef SNMALLOC_CHECK_CLIENT @@ -423,10 +415,9 @@ namespace snmalloc bool need_post = false; auto local_state = backend_state_ptr(); auto domesticate = - [local_state](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { - return capptr_domesticate(local_state, p); - }; + [local_state](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(local_state, p); + }; for (size_t i = 0; i < REMOTE_BATCH; i++) { auto p = message_queue().peek(); @@ -617,7 +608,7 @@ namespace snmalloc Metaslab::is_start_of_object(entry.get_sizeclass(), address_cast(p)), "Not deallocating start of an object"); - auto cp = p.as_static>(); + auto cp = p.as_static>(); auto& key = entropy.get_free_list_key(); @@ -628,10 +619,8 @@ namespace snmalloc } template - SNMALLOC_SLOW_PATH void* small_alloc( - sizeclass_t sizeclass, - FreeListIter& - fast_free_list) + SNMALLOC_SLOW_PATH void* + small_alloc(sizeclass_t sizeclass, FreeListIter<>& fast_free_list) { size_t rsize = sizeclass_to_size(sizeclass); @@ -655,12 +644,10 @@ namespace snmalloc if (meta->needed() == 0) alloc_classes[sizeclass].unused--; - auto domesticate = - [this](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { - return capptr_domesticate( - backend_state_ptr(), p); - }; + auto domesticate = [this]( + FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(backend_state_ptr(), p); + }; auto [p, still_active] = Metaslab::alloc_free_list( domesticate, meta, fast_free_list, entropy, sizeclass); @@ -695,10 +682,7 @@ namespace snmalloc template SNMALLOC_SLOW_PATH void* small_alloc_slow( - sizeclass_t sizeclass, - FreeListIter& - fast_free_list, - size_t rsize) + sizeclass_t sizeclass, FreeListIter<>& fast_free_list, size_t rsize) { // No existing free list get a new slab. size_t slab_size = sizeclass_to_slab_size(sizeclass); @@ -729,11 +713,9 @@ namespace snmalloc alloc_new_list(slab, meta, rsize, slab_size, entropy); auto domesticate = - [this](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { - return capptr_domesticate( - backend_state_ptr(), p); - }; + [this](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(backend_state_ptr(), p); + }; auto [p, still_active] = Metaslab::alloc_free_list( domesticate, meta, fast_free_list, entropy, sizeclass); @@ -756,10 +738,9 @@ namespace snmalloc SNMALLOC_ASSERT(attached_cache != nullptr); auto local_state = backend_state_ptr(); auto domesticate = - [local_state](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { - return capptr_domesticate(local_state, p); - }; + [local_state](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(local_state, p); + }; if (destroy_queue) { diff --git a/src/mem/freelist.h b/src/mem/freelist.h index 8d6ad07c6..a002c00d8 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -55,22 +55,31 @@ namespace snmalloc class FreeObject { public: - template + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue = capptr::bounds::AllocWild> class T; /** * This "inductive step" type -- a queue-annotated pointer to a FreeObject * containing a queue-annotated pointer -- shows up all over the place. - * Give it a shorter name (FreeObject::QueuePtr) for convenience. + * Give it a shorter name (FreeObject::BQueuePtr) for convenience. */ template - using QueuePtr = CapPtr, BQueue>; + using BQueuePtr = CapPtr, BQueue>; /** - * As with QueuePtr, but atomic. + * In particular, external code almost always uses AllocWild as the queue + * annotation. Make their lives easier. + */ + using QueuePtr = BQueuePtr; + + /** + * As with BQueuePtr, but atomic. */ template - using AtomicQueuePtr = AtomicCapPtr, BQueue>; + using BAtomicQueuePtr = AtomicCapPtr, BQueue>; + + using AtomicQueuePtr = BAtomicQueuePtr; /** * This is the "base case" of that induction. While we can't get rid of the @@ -81,7 +90,9 @@ namespace snmalloc template< SNMALLOC_CONCEPT(capptr::ConceptBound) BView, SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> - using HeadPtr = CapPtr, BView>; + using BHeadPtr = CapPtr, BView>; + + using HeadPtr = BHeadPtr; /** * As with HeadPtr, but atomic. @@ -115,9 +126,9 @@ namespace snmalloc union { - QueuePtr next_object; + BQueuePtr next_object; // TODO: Should really use C++20 atomic_ref rather than a union. - AtomicQueuePtr atomic_next_object; + BAtomicQueuePtr atomic_next_object; }; #ifdef SNMALLOC_CHECK_CLIENT // Encoded representation of a back pointer. @@ -131,7 +142,7 @@ namespace snmalloc SNMALLOC_CONCEPT(capptr::ConceptBound) BView = typename BQueue:: template with_wildness, typename Domesticator> - HeadPtr + BHeadPtr atomic_read_next(const FreeListKey& key, Domesticator domesticate) { auto n_wild = FreeObject::decode_next( @@ -156,7 +167,7 @@ namespace snmalloc SNMALLOC_CONCEPT(capptr::ConceptBound) BView = typename BQueue:: template with_wildness, typename Domesticator> - HeadPtr + BHeadPtr read_next(const FreeListKey& key, Domesticator domesticate) { return domesticate(FreeObject::decode_next( @@ -179,7 +190,7 @@ namespace snmalloc template< SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue, SNMALLOC_CONCEPT(capptr::ConceptBound) BView> - static HeadPtr make(CapPtr p) + static BHeadPtr make(CapPtr p) { return p.template as_static>(); } @@ -241,10 +252,10 @@ namespace snmalloc template< SNMALLOC_CONCEPT(capptr::ConceptBound) BView, SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> - inline static QueuePtr encode_next( - address_t curr, HeadPtr next, const FreeListKey& key) + inline static BQueuePtr encode_next( + address_t curr, BHeadPtr next, const FreeListKey& key) { - return QueuePtr(code_next(curr, next.unsafe_ptr(), key)); + return BQueuePtr(code_next(curr, next.unsafe_ptr(), key)); } /** @@ -264,10 +275,10 @@ namespace snmalloc template< SNMALLOC_CONCEPT(capptr::ConceptBound) BView, SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> - inline static HeadPtr decode_next( - address_t curr, HeadPtr next, const FreeListKey& key) + inline static BHeadPtr decode_next( + address_t curr, BHeadPtr next, const FreeListKey& key) { - return HeadPtr(code_next(curr, next.unsafe_ptr(), key)); + return BHeadPtr(code_next(curr, next.unsafe_ptr(), key)); } template< @@ -299,9 +310,9 @@ namespace snmalloc template< SNMALLOC_CONCEPT(capptr::ConceptBound) BView, SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> - static QueuePtr* store_next( - QueuePtr* curr, - HeadPtr next, + static BQueuePtr* store_next( + BQueuePtr* curr, + BHeadPtr next, const FreeListKey& key) { assert_view_queue_bounds(); @@ -317,9 +328,9 @@ namespace snmalloc } template - static void store_null(QueuePtr* curr, const FreeListKey& key) + static void store_null(BQueuePtr* curr, const FreeListKey& key) { - *curr = encode_next(address_cast(curr), QueuePtr(nullptr), key); + *curr = encode_next(address_cast(curr), BQueuePtr(nullptr), key); } /** @@ -331,8 +342,8 @@ namespace snmalloc SNMALLOC_CONCEPT(capptr::ConceptBound) BView, SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> static void atomic_store_next( - HeadPtr curr, - HeadPtr next, + BHeadPtr curr, + BHeadPtr next, const FreeListKey& key) { static_assert(BView::wildness == capptr::dimension::Wildness::Tame); @@ -354,13 +365,13 @@ namespace snmalloc SNMALLOC_CONCEPT(capptr::ConceptBound) BView, SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> static void - atomic_store_null(HeadPtr curr, const FreeListKey& key) + atomic_store_null(BHeadPtr curr, const FreeListKey& key) { static_assert(BView::wildness == capptr::dimension::Wildness::Tame); curr->atomic_next_object.store( encode_next( - address_cast(&curr->next_object), QueuePtr(nullptr), key), + address_cast(&curr->next_object), BQueuePtr(nullptr), key), std::memory_order_relaxed); } }; @@ -375,18 +386,18 @@ namespace snmalloc * Checks signing of pointers */ template< - SNMALLOC_CONCEPT(capptr::ConceptBound) BView, - SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + SNMALLOC_CONCEPT(capptr::ConceptBound) BView = capptr::bounds::Alloc, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue = capptr::bounds::AllocWild> class FreeListIter { - FreeObject::HeadPtr curr{nullptr}; + FreeObject::BHeadPtr curr{nullptr}; #ifdef SNMALLOC_CHECK_CLIENT address_t prev{0}; #endif public: constexpr FreeListIter( - FreeObject::HeadPtr head, address_t prev_value) + FreeObject::BHeadPtr head, address_t prev_value) : curr(head) { #ifdef SNMALLOC_CHECK_CLIENT @@ -408,7 +419,7 @@ namespace snmalloc /** * Returns current head without affecting the iterator. */ - FreeObject::HeadPtr peek() + FreeObject::BHeadPtr peek() { return curr; } @@ -417,7 +428,7 @@ namespace snmalloc * Moves the iterator on, and returns the current value. */ template - FreeObject::HeadPtr + FreeObject::BHeadPtr take(const FreeListKey& key, Domesticator domesticate) { auto c = curr; @@ -455,9 +466,9 @@ namespace snmalloc */ template< bool RANDOM, - SNMALLOC_CONCEPT(capptr::ConceptBound) BView, - SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue, - bool INIT = true> + bool INIT = true, + SNMALLOC_CONCEPT(capptr::ConceptBound) BView = capptr::bounds::Alloc, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue = capptr::bounds::AllocWild> class FreeListBuilder { static constexpr size_t LENGTH = RANDOM ? 2 : 1; @@ -466,7 +477,7 @@ namespace snmalloc * We use native pointers below so that we don't run afoul of strict * aliasing rules. head is a FreeObject::HeadPtr -- that * is, a known-domesticated pointer to a queue of wild pointers -- and - * it's usually the case that end is a FreeObject::QueuePtr* -- + * it's usually the case that end is a FreeObject::BQueuePtr* -- * that is, a known-domesticated pointer to a wild pointer to a queue of * wild pointers. However, in order to do branchless inserts, we set end * = &head, which breaks strict aliasing rules with the types as given. @@ -481,19 +492,19 @@ namespace snmalloc // This enables branch free enqueuing. std::array end{nullptr}; - FreeObject::QueuePtr* cast_end(uint32_t ix) + FreeObject::BQueuePtr* cast_end(uint32_t ix) { - return reinterpret_cast*>(end[ix]); + return reinterpret_cast*>(end[ix]); } - void set_end(uint32_t ix, FreeObject::QueuePtr* p) + void set_end(uint32_t ix, FreeObject::BQueuePtr* p) { end[ix] = reinterpret_cast(p); } - FreeObject::HeadPtr cast_head(uint32_t ix) + FreeObject::BHeadPtr cast_head(uint32_t ix) { - return FreeObject::HeadPtr( + return FreeObject::BHeadPtr( static_cast*>(head[ix])); } @@ -527,7 +538,7 @@ namespace snmalloc * Adds an element to the builder */ void add( - FreeObject::HeadPtr n, + FreeObject::BHeadPtr n, const FreeListKey& key, LocalEntropy& entropy) { @@ -554,7 +565,7 @@ namespace snmalloc */ template std::enable_if_t - add(FreeObject::HeadPtr n, const FreeListKey& key) + add(FreeObject::BHeadPtr n, const FreeListKey& key) { static_assert(RANDOM_ == RANDOM, "Don't set template parameter"); set_end(0, FreeObject::store_next(cast_end(0), n, key)); @@ -578,7 +589,7 @@ namespace snmalloc * and is thus subject to encoding if the next_object pointers * encoded. */ - FreeObject::HeadPtr + FreeObject::BHeadPtr read_head(uint32_t index, const FreeListKey& key) { return FreeObject::decode_next( @@ -652,8 +663,8 @@ namespace snmalloc std::enable_if_t< !RANDOM_, std::pair< - FreeObject::HeadPtr, - FreeObject::HeadPtr>> + FreeObject::BHeadPtr, + FreeObject::BHeadPtr>> extract_segment(const FreeListKey& key) { static_assert(RANDOM_ == RANDOM, "Don't set SFINAE parameter!"); @@ -664,7 +675,7 @@ namespace snmalloc // this is doing a CONTAINING_RECORD like cast to get back // to the actual object. This isn't true if the builder is // empty, but you are not allowed to call this in the empty case. - auto last = FreeObject::HeadPtr( + auto last = FreeObject::BHeadPtr( FreeObject::from_next_ptr(cast_end(0))); init(); return {first, last}; diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 83c78c610..23a69876a 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -211,38 +211,33 @@ namespace snmalloc SNMALLOC_FAST_PATH void* small_alloc(size_t size) { // SNMALLOC_ASSUME(size <= sizeclass_to_size(NUM_SIZECLASSES)); - auto domesticate = - [this](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { - return capptr_domesticate( - core_alloc->backend_state_ptr(), p); - }; - auto slowpath = - [&]( - sizeclass_t sizeclass, - FreeListIter* - fl) SNMALLOC_FAST_PATH_LAMBDA { - if (likely(core_alloc != nullptr)) - { - return core_alloc->handle_message_queue( - []( - CoreAlloc* core_alloc, - sizeclass_t sizeclass, - FreeListIter* - fl) { - return core_alloc->template small_alloc( - sizeclass, *fl); - }, - core_alloc, - sizeclass, - fl); - } - return lazy_init( - [&](CoreAlloc*, sizeclass_t sizeclass) { - return small_alloc(sizeclass_to_size(sizeclass)); + auto domesticate = [this](FreeObject::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate( + core_alloc->backend_state_ptr(), p); + }; + auto slowpath = [&]( + sizeclass_t sizeclass, + FreeListIter<>* fl) SNMALLOC_FAST_PATH_LAMBDA { + if (likely(core_alloc != nullptr)) + { + return core_alloc->handle_message_queue( + []( + CoreAlloc* core_alloc, + sizeclass_t sizeclass, + FreeListIter<>* fl) { + return core_alloc->template small_alloc(sizeclass, *fl); }, - sizeclass); - }; + core_alloc, + sizeclass, + fl); + } + return lazy_init( + [&](CoreAlloc*, sizeclass_t sizeclass) { + return small_alloc(sizeclass_to_size(sizeclass)); + }, + sizeclass); + }; return local_cache.template alloc( domesticate, size, slowpath); diff --git a/src/mem/localcache.h b/src/mem/localcache.h index 4d3d55a49..593fc3212 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -12,9 +12,8 @@ namespace snmalloc { using Stats = AllocStats; - template - inline static SNMALLOC_FAST_PATH void* finish_alloc_no_zero( - FreeObject::HeadPtr p, sizeclass_t sizeclass) + inline static SNMALLOC_FAST_PATH void* + finish_alloc_no_zero(FreeObject::HeadPtr p, sizeclass_t sizeclass) { SNMALLOC_ASSERT(Metaslab::is_start_of_object(sizeclass, address_cast(p))); UNUSED(sizeclass); @@ -24,12 +23,9 @@ namespace snmalloc return r; } - template< - ZeroMem zero_mem, - typename SharedStateHandle, - SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> - inline static SNMALLOC_FAST_PATH void* finish_alloc( - FreeObject::HeadPtr p, sizeclass_t sizeclass) + template + inline static SNMALLOC_FAST_PATH void* + finish_alloc(FreeObject::HeadPtr p, sizeclass_t sizeclass) { auto r = finish_alloc_no_zero(p, sizeclass); @@ -50,8 +46,7 @@ namespace snmalloc // Free list per small size class. These are used for // allocation on the fast path. This part of the code is inspired by // mimalloc. - FreeListIter - small_fast_free_lists[NUM_SIZECLASSES] = {}; + FreeListIter<> small_fast_free_lists[NUM_SIZECLASSES] = {}; // This is the entropy for a particular thread. LocalEntropy entropy; @@ -85,10 +80,9 @@ namespace snmalloc { auto& key = entropy.get_free_list_key(); auto domesticate = - [local_state](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { - return capptr_domesticate(local_state, p); - }; + [local_state](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(local_state, p); + }; for (size_t i = 0; i < NUM_SIZECLASSES; i++) { diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 4be0499d5..3b31fa4fa 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -25,11 +25,9 @@ namespace snmalloc * Data-structure for building the free list for this slab. */ #ifdef SNMALLOC_CHECK_CLIENT - FreeListBuilder - free_queue; + FreeListBuilder free_queue; #else - FreeListBuilder - free_queue; + FreeListBuilder free_queue; #endif /** @@ -155,14 +153,11 @@ namespace snmalloc * available objects for this metaslab. */ template - static SNMALLOC_FAST_PATH std::pair< - FreeObject::HeadPtr, - bool> + static SNMALLOC_FAST_PATH std::pair alloc_free_list( Domesticator domesticate, Metaslab* meta, - FreeListIter& - fast_free_list, + FreeListIter<>& fast_free_list, LocalEntropy& entropy, sizeclass_t sizeclass) { diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index c658c2eea..6e3b0a354 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -35,12 +35,10 @@ namespace snmalloc // Store the message queue on a separate cacheline. It is mutable data that // is read by other threads. - alignas(CACHELINE_SIZE) - FreeObject::AtomicQueuePtr back{nullptr}; + alignas(CACHELINE_SIZE) FreeObject::AtomicQueuePtr back{nullptr}; // Store the two ends on different cache lines as access by different // threads. - alignas(CACHELINE_SIZE) - FreeObject::QueuePtr front{nullptr}; + alignas(CACHELINE_SIZE) FreeObject::QueuePtr front{nullptr}; constexpr RemoteAllocator() = default; @@ -50,9 +48,7 @@ namespace snmalloc SNMALLOC_ASSERT(front != nullptr); } - void - init(FreeObject::HeadPtr - stub) + void init(FreeObject::HeadPtr stub) { FreeObject::atomic_store_null(stub, key_global); front = capptr_rewild(stub); @@ -60,9 +56,9 @@ namespace snmalloc invariant(); } - FreeObject::QueuePtr destroy() + FreeObject::QueuePtr destroy() { - FreeObject::QueuePtr fnt = front; + FreeObject::QueuePtr fnt = front; back.store(nullptr, std::memory_order_relaxed); front = nullptr; return fnt; @@ -70,8 +66,7 @@ namespace snmalloc inline bool is_empty() { - FreeObject::QueuePtr bk = - back.load(std::memory_order_relaxed); + FreeObject::QueuePtr bk = back.load(std::memory_order_relaxed); return bk == front; } @@ -82,10 +77,8 @@ namespace snmalloc */ template void enqueue( - FreeObject::HeadPtr - first, - FreeObject::HeadPtr - last, + FreeObject::HeadPtr first, + FreeObject::HeadPtr last, const FreeListKey& key, Domesticator domesticate) { @@ -93,13 +86,13 @@ namespace snmalloc FreeObject::atomic_store_null(last, key); // exchange needs to be a release, so nullptr in next is visible. - FreeObject::QueuePtr prev = + FreeObject::QueuePtr prev = back.exchange(capptr_rewild(last), std::memory_order_release); FreeObject::atomic_store_next(domesticate(prev), first, key); } - FreeObject::QueuePtr peek() + FreeObject::QueuePtr peek() { return front; } @@ -108,16 +101,12 @@ namespace snmalloc * Returns the front message, or null if not possible to return a message. */ template - std::pair< - FreeObject::HeadPtr, - bool> + std::pair dequeue(const FreeListKey& key, Domesticator domesticate) { invariant(); - FreeObject::HeadPtr - first = domesticate(front); - FreeObject::HeadPtr - next = first->atomic_read_next(key, domesticate); + FreeObject::HeadPtr first = domesticate(front); + FreeObject::HeadPtr next = first->atomic_read_next(key, domesticate); if (next != nullptr) { diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index 0d20d8969..5a739aa0e 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -16,14 +16,7 @@ namespace snmalloc */ struct RemoteDeallocCache { - std::array< - FreeListBuilder< - false, - capptr::bounds::Alloc, - capptr::bounds::AllocWild, - false>, - REMOTE_SLOTS> - list; + std::array, REMOTE_SLOTS> list; /** * The total amount of memory we are waiting for before we will dispatch @@ -77,8 +70,7 @@ namespace snmalloc const FreeListKey& key) { SNMALLOC_ASSERT(initialised); - auto r = - p.template as_reinterpret>(); + auto r = p.template as_reinterpret>(); list[get_slot(target_id, 0)].add(r, key); } @@ -93,10 +85,9 @@ namespace snmalloc size_t post_round = 0; bool sent_something = false; auto domesticate = - [local_state](FreeObject::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { - return capptr_domesticate(local_state, p); - }; + [local_state](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(local_state, p); + }; while (true) { @@ -123,7 +114,7 @@ namespace snmalloc // Entries could map back onto the "resend" list, // so take copy of the head, mark the last element, // and clear the original list. - FreeListIter resend; + FreeListIter<> resend; list[my_slot].close(resend, key); post_round++; From 96155db64041171c43f34d5798d1f57612249f9e Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Fri, 8 Oct 2021 16:45:00 +0100 Subject: [PATCH 105/302] Collect freelist things in a namespace Motivated by renaming `FreeObject::{Head,Queue,AtomicQueue}Ptr` to `freelist::...Ptr`, in fact go further, moving `FreeObject` itself to `freelist::Object` and `FreeListBuilder` to `freelist::Builder` and `FreeListIter` to `freelist::Iter` --- src/mem/corealloc.h | 28 +- src/mem/freelist.h | 1078 +++++++++++++++++++------------------ src/mem/localalloc.h | 6 +- src/mem/localcache.h | 10 +- src/mem/metaslab.h | 8 +- src/mem/remoteallocator.h | 32 +- src/mem/remotecache.h | 8 +- 7 files changed, 595 insertions(+), 575 deletions(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 611827ca0..0188e91d4 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -162,7 +162,7 @@ namespace snmalloc // Manufacture an allocation to prime the queue // Using an actual allocation removes a conditional from a critical path. auto dummy = capptr::Alloc(small_alloc_one(MIN_ALLOC_SIZE)) - .template as_static>(); + .template as_static>(); if (dummy == nullptr) { error("Critical error: Out-of-memory during initialisation."); @@ -180,12 +180,12 @@ namespace snmalloc { SNMALLOC_ASSERT(attached_cache != nullptr); auto domesticate = - [this](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + [this](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { return capptr_domesticate(backend_state_ptr(), p); }; // Use attached cache, and fill it if it is empty. return attached_cache->template alloc( - domesticate, size, [&](sizeclass_t sizeclass, FreeListIter<>* fl) { + domesticate, size, [&](sizeclass_t sizeclass, freelist::Iter<>* fl) { return small_alloc(sizeclass, *fl); }); } @@ -250,7 +250,7 @@ namespace snmalloc { b.add( // Here begins our treatment of the heap as containing Wild pointers - FreeObject::make( + freelist::Object::make( capptr_to_user_address_control(curr_ptr.as_void())), key, entropy); @@ -262,7 +262,7 @@ namespace snmalloc { b.add( // Here begins our treatment of the heap as containing Wild pointers - FreeObject::make( + freelist::Object::make( capptr_to_user_address_control( Aal::capptr_bound( p.as_void(), rsize))), @@ -277,12 +277,12 @@ namespace snmalloc ChunkRecord* clear_slab(Metaslab* meta, sizeclass_t sizeclass) { auto& key = entropy.get_free_list_key(); - FreeListIter<> fl; + freelist::Iter<> fl; auto more = meta->free_queue.close(fl, key); UNUSED(more); auto local_state = backend_state_ptr(); auto domesticate = - [local_state](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + [local_state](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { return capptr_domesticate(local_state, p); }; void* p = finish_alloc_no_zero(fl.take(key, domesticate), sizeclass); @@ -415,7 +415,7 @@ namespace snmalloc bool need_post = false; auto local_state = backend_state_ptr(); auto domesticate = - [local_state](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + [local_state](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { return capptr_domesticate(local_state, p); }; for (size_t i = 0; i < REMOTE_BATCH; i++) @@ -608,7 +608,7 @@ namespace snmalloc Metaslab::is_start_of_object(entry.get_sizeclass(), address_cast(p)), "Not deallocating start of an object"); - auto cp = p.as_static>(); + auto cp = p.as_static>(); auto& key = entropy.get_free_list_key(); @@ -620,7 +620,7 @@ namespace snmalloc template SNMALLOC_SLOW_PATH void* - small_alloc(sizeclass_t sizeclass, FreeListIter<>& fast_free_list) + small_alloc(sizeclass_t sizeclass, freelist::Iter<>& fast_free_list) { size_t rsize = sizeclass_to_size(sizeclass); @@ -645,7 +645,7 @@ namespace snmalloc alloc_classes[sizeclass].unused--; auto domesticate = [this]( - FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { return capptr_domesticate(backend_state_ptr(), p); }; auto [p, still_active] = Metaslab::alloc_free_list( @@ -682,7 +682,7 @@ namespace snmalloc template SNMALLOC_SLOW_PATH void* small_alloc_slow( - sizeclass_t sizeclass, FreeListIter<>& fast_free_list, size_t rsize) + sizeclass_t sizeclass, freelist::Iter<>& fast_free_list, size_t rsize) { // No existing free list get a new slab. size_t slab_size = sizeclass_to_slab_size(sizeclass); @@ -713,7 +713,7 @@ namespace snmalloc alloc_new_list(slab, meta, rsize, slab_size, entropy); auto domesticate = - [this](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + [this](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { return capptr_domesticate(backend_state_ptr(), p); }; auto [p, still_active] = Metaslab::alloc_free_list( @@ -738,7 +738,7 @@ namespace snmalloc SNMALLOC_ASSERT(attached_cache != nullptr); auto local_state = backend_state_ptr(); auto domesticate = - [local_state](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + [local_state](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { return capptr_domesticate(local_state, p); }; diff --git a/src/mem/freelist.h b/src/mem/freelist.h index a002c00d8..b58978ab7 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -9,7 +9,7 @@ * * The corruption detection works as follows * - * FreeObject + * free Object * ----------------------------- * | next | prev_encoded | ... | * ----------------------------- @@ -52,633 +52,653 @@ namespace snmalloc return (c + key.key1) * (n + key.key2); } - class FreeObject + namespace freelist { - public: - template< - SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue = capptr::bounds::AllocWild> - class T; + class Object + { + public: + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue = + capptr::bounds::AllocWild> + class T; - /** - * This "inductive step" type -- a queue-annotated pointer to a FreeObject - * containing a queue-annotated pointer -- shows up all over the place. - * Give it a shorter name (FreeObject::BQueuePtr) for convenience. - */ - template - using BQueuePtr = CapPtr, BQueue>; + /** + * This "inductive step" type -- a queue-annotated pointer to a free + * Object containing a queue-annotated pointer -- shows up all over the + * place. Give it a shorter name (Object::BQueuePtr) for + * convenience. + */ + template + using BQueuePtr = CapPtr, BQueue>; - /** - * In particular, external code almost always uses AllocWild as the queue - * annotation. Make their lives easier. - */ - using QueuePtr = BQueuePtr; + /** + * As with BQueuePtr, but atomic. + */ + template + using BAtomicQueuePtr = AtomicCapPtr, BQueue>; - /** - * As with BQueuePtr, but atomic. - */ - template - using BAtomicQueuePtr = AtomicCapPtr, BQueue>; + /** + * This is the "base case" of that induction. While we can't get rid of + * the two different type parameters (in general), we can at least get rid + * of a bit of the clutter. "freelist::Object::HeadPtr" + * looks a little nicer than "CapPtr, BView>". + */ + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + using BHeadPtr = CapPtr, BView>; - using AtomicQueuePtr = BAtomicQueuePtr; + /** + * As with BHeadPtr, but atomic. + */ + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + using BAtomicHeadPtr = AtomicCapPtr, BView>; - /** - * This is the "base case" of that induction. While we can't get rid of the - * two different type parameters (in general), we can at least get rid of a - * bit of the clutter. "FreeObject::HeadPtr" looks a little - * nicer than "CapPtr, BView>". - */ - template< - SNMALLOC_CONCEPT(capptr::ConceptBound) BView, - SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> - using BHeadPtr = CapPtr, BView>; + /** + * Free objects within each slab point directly to the next. + * There is an optional second field that is effectively a + * back pointer in a doubly linked list, however, it is encoded + * to prevent corruption. + * + * This is an inner class to avoid the need to specify BQueue when calling + * static methods. + * + * Raw C++ pointers to this type are *assumed to be domesticated*. In + * some cases we still explicitly annotate domesticated free Object*-s as + * CapPtr<>, but more often CapPtr,B> will have B = A. + * + * TODO: Consider putting prev_encoded at the end of the object, would + * require size to be threaded through, but would provide more OOB + * detection. + */ + template + class T + { + friend class Object; - using HeadPtr = BHeadPtr; + union + { + BQueuePtr next_object; + // TODO: Should really use C++20 atomic_ref rather than a union. + BAtomicQueuePtr atomic_next_object; + }; +#ifdef SNMALLOC_CHECK_CLIENT + // Encoded representation of a back pointer. + // Hard to fake, and provides consistency on + // the next pointers. + address_t prev_encoded; +#endif - /** - * As with HeadPtr, but atomic. - */ - template< - SNMALLOC_CONCEPT(capptr::ConceptBound) BView, - SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> - using AtomicHeadPtr = AtomicCapPtr, BView>; + public: + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView = typename BQueue:: + template with_wildness, + typename Domesticator> + BHeadPtr + atomic_read_next(const FreeListKey& key, Domesticator domesticate) + { + auto n_wild = Object::decode_next( + address_cast(&this->next_object), + this->atomic_next_object.load(std::memory_order_acquire), + key); + auto n_tame = domesticate(n_wild); +#ifdef SNMALLOC_CHECK_CLIENT + if (n_tame != nullptr) + { + n_tame->check_prev( + signed_prev(address_cast(this), address_cast(n_tame), key)); + } +#endif + return n_tame; + } - /** - * Free objects within each slab point directly to the next. - * There is an optional second field that is effectively a - * back pointer in a doubly linked list, however, it is encoded - * to prevent corruption. - * - * This is an inner class to avoid the need to specify BQueue when calling - * static methods. - * - * Raw C++ pointers to this type are *assumed to be domesticated*. In some - * cases we still explicitly annotate domesticated FreeObject*-s as - * CapPtr<>, but more often CapPtr,B> will have B = A. - * - * TODO: Consider putting prev_encoded at the end of the object, would - * require size to be threaded through, but would provide more OOB - * detection. - */ - template - class T - { - friend class FreeObject; + /** + * Read the next pointer + */ + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView = typename BQueue:: + template with_wildness, + typename Domesticator> + BHeadPtr + read_next(const FreeListKey& key, Domesticator domesticate) + { + return domesticate(Object::decode_next( + address_cast(&this->next_object), this->next_object, key)); + } - union - { - BQueuePtr next_object; - // TODO: Should really use C++20 atomic_ref rather than a union. - BAtomicQueuePtr atomic_next_object; + /** + * Check the signature of this free Object + */ + void check_prev(address_t signed_prev) + { + UNUSED(signed_prev); + check_client( + signed_prev == this->prev_encoded, + "Heap corruption - free list corrupted!"); + } }; -#ifdef SNMALLOC_CHECK_CLIENT - // Encoded representation of a back pointer. - // Hard to fake, and provides consistency on - // the next pointers. - address_t prev_encoded; -#endif - public: + // Note the inverted template argument order, since BView is inferable. template< - SNMALLOC_CONCEPT(capptr::ConceptBound) BView = typename BQueue:: - template with_wildness, - typename Domesticator> - BHeadPtr - atomic_read_next(const FreeListKey& key, Domesticator domesticate) + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue, + SNMALLOC_CONCEPT(capptr::ConceptBound) BView> + static BHeadPtr make(CapPtr p) { - auto n_wild = FreeObject::decode_next( - address_cast(&this->next_object), - this->atomic_next_object.load(std::memory_order_acquire), - key); - auto n_tame = domesticate(n_wild); -#ifdef SNMALLOC_CHECK_CLIENT - if (n_tame != nullptr) + return p.template as_static>(); + } + + /** + * A container-of operation to convert &f->next_object to f + */ + template + static Object::T* + from_next_ptr(CapPtr, BQueue>* ptr) + { + static_assert(offsetof(Object::T, next_object) == 0); + return reinterpret_cast*>(ptr); + } + + private: + /** + * Involutive encryption with raw pointers + */ + template + inline static Object::T* + code_next(address_t curr, Object::T* next, const FreeListKey& key) + { + // Note we can consider other encoding schemes here. + // * XORing curr and next. This doesn't require any key material + // * XORing (curr * key). This makes it harder to guess the underlying + // key, as each location effectively has its own key. + // Curr is not used in the current encoding scheme. + UNUSED(curr); + + if constexpr (CHECK_CLIENT && !aal_supports) { - n_tame->check_prev( - signed_prev(address_cast(this), address_cast(n_tame), key)); + return reinterpret_cast*>( + reinterpret_cast(next) ^ key.key_next); + } + else + { + UNUSED(key); + return next; } -#endif - return n_tame; } + public: /** - * Read the next pointer + * Encode next. We perform two convenient little bits of type-level + * sleight of hand here: + * + * 1) We convert the provided HeadPtr to a QueuePtr, forgetting BView in + * the result; all the callers write the result through a pointer to a + * QueuePtr, though, strictly, the result itself is no less domesticated + * than the input (even if it is obfuscated). + * + * 2) Speaking of obfuscation, we continue to use a CapPtr<> type even + * though the result is likely not safe to dereference, being an + * obfuscated bundle of bits (on non-CHERI architectures, anyway). That's + * additional motivation to consider the result BQueue-bounded, as that + * is likely (but not necessarily) Wild. */ template< - SNMALLOC_CONCEPT(capptr::ConceptBound) BView = typename BQueue:: - template with_wildness, - typename Domesticator> - BHeadPtr - read_next(const FreeListKey& key, Domesticator domesticate) + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + inline static BQueuePtr encode_next( + address_t curr, BHeadPtr next, const FreeListKey& key) { - return domesticate(FreeObject::decode_next( - address_cast(&this->next_object), this->next_object, key)); + return BQueuePtr(code_next(curr, next.unsafe_ptr(), key)); } /** - * Check the signature of this FreeObject + * Decode next. While traversing a queue, BView and BQueue here will + * often be equal (i.e., AllocUserWild) rather than dichotomous. However, + * we do occasionally decode an actual head pointer, so be polymorphic + * here. + * + * TODO: We'd like, in some sense, to more tightly couple or integrate + * this into to the domestication process. We could introduce an + * additional state in the capptr_bounds::wild taxonomy (e.g, Obfuscated) + * so that the Domesticator-s below have to call through this function to + * get the Wild pointer they can then make Tame. It's not yet entirely + * clear what that would look like and whether/how the encode_next side of + * things should be exposed. For the moment, obfuscation is left + * encapsulated within Object and we do not capture any of it statically. */ - void check_prev(address_t signed_prev) + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + inline static BHeadPtr decode_next( + address_t curr, BHeadPtr next, const FreeListKey& key) { - UNUSED(signed_prev); - check_client( - signed_prev == this->prev_encoded, - "Heap corruption - free list corrupted!"); + return BHeadPtr(code_next(curr, next.unsafe_ptr(), key)); } - }; - // Note the inverted template argument order, since BView is inferable. - template< - SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue, - SNMALLOC_CONCEPT(capptr::ConceptBound) BView> - static BHeadPtr make(CapPtr p) - { - return p.template as_static>(); - } + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + static void assert_view_queue_bounds() + { + static_assert( + BView::wildness == capptr::dimension::Wildness::Tame, + "Free Object View must be domesticated, justifying raw pointers"); + + static_assert( + std::is_same_v< + typename BQueue::template with_wildness< + capptr::dimension::Wildness::Tame>, + BView>, + "Free Object Queue bounds must match View bounds (but may be Wild)"); + } - /** - * A container-of operation to convert &f->next_object to f - */ - template - static FreeObject::T* - from_next_ptr(CapPtr, BQueue>* ptr) - { - static_assert(offsetof(FreeObject::T, next_object) == 0); - return reinterpret_cast*>(ptr); - } + /** + * Assign next_object and update its prev_encoded if + * SNMALLOC_CHECK_CLIENT. Static so that it can be used on reference to a + * free Object. + * + * Returns a pointer to the next_object field of the next parameter as an + * optimization for repeated snoc operations (in which + * next->next_object is nullptr). + */ + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + static BQueuePtr* store_next( + BQueuePtr* curr, + BHeadPtr next, + const FreeListKey& key) + { + assert_view_queue_bounds(); - private: - /** - * Involutive encryption with raw pointers - */ - template - inline static FreeObject::T* code_next( - address_t curr, FreeObject::T* next, const FreeListKey& key) - { - // Note we can consider other encoding schemes here. - // * XORing curr and next. This doesn't require any key material - // * XORing (curr * key). This makes it harder to guess the underlying - // key, as each location effectively has its own key. - // Curr is not used in the current encoding scheme. - UNUSED(curr); - - if constexpr (CHECK_CLIENT && !aal_supports) +#ifdef SNMALLOC_CHECK_CLIENT + next->prev_encoded = + signed_prev(address_cast(curr), address_cast(next), key); +#else + UNUSED(key); +#endif + *curr = encode_next(address_cast(curr), next, key); + return &(next->next_object); + } + + template + static void store_null(BQueuePtr* curr, const FreeListKey& key) { - return reinterpret_cast*>( - reinterpret_cast(next) ^ key.key_next); + *curr = + encode_next(address_cast(curr), BQueuePtr(nullptr), key); } - else + + /** + * Assign next_object and update its prev_encoded if SNMALLOC_CHECK_CLIENT + * + * Uses the atomic view of next, so can be used in the message queues. + */ + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + static void atomic_store_next( + BHeadPtr curr, + BHeadPtr next, + const FreeListKey& key) { + static_assert(BView::wildness == capptr::dimension::Wildness::Tame); + +#ifdef SNMALLOC_CHECK_CLIENT + next->prev_encoded = + signed_prev(address_cast(curr), address_cast(next), key); +#else UNUSED(key); - return next; +#endif + // Signature needs to be visible before item is linked in + // so requires release semantics. + curr->atomic_next_object.store( + encode_next(address_cast(&curr->next_object), next, key), + std::memory_order_release); + } + + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BView, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> + static void + atomic_store_null(BHeadPtr curr, const FreeListKey& key) + { + static_assert(BView::wildness == capptr::dimension::Wildness::Tame); + + curr->atomic_next_object.store( + encode_next( + address_cast(&curr->next_object), BQueuePtr(nullptr), key), + std::memory_order_relaxed); } - } + }; + + static_assert( + sizeof(Object) <= MIN_ALLOC_SIZE, + "Needs to be able to fit in smallest allocation."); - public: /** - * Encode next. We perform two convenient little bits of type-level - * sleight of hand here: - * - * 1) We convert the provided HeadPtr to a QueuePtr, forgetting BView in - * the result; all the callers write the result through a pointer to a - * QueuePtr, though, strictly, the result itself is no less domesticated - * than the input (even if it is obfuscated). - * - * 2) Speaking of obfuscation, we continue to use a CapPtr<> type even - * though the result is likely not safe to dereference, being an - * obfuscated bundle of bits (on non-CHERI architectures, anyway). That's - * additional motivation to consider the result BQueue-bounded, as that is - * likely (but not necessarily) Wild. + * External code almost always uses Alloc and AllocWild for its free lists. + * Give them a convenient alias. */ - template< - SNMALLOC_CONCEPT(capptr::ConceptBound) BView, - SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> - inline static BQueuePtr encode_next( - address_t curr, BHeadPtr next, const FreeListKey& key) - { - return BQueuePtr(code_next(curr, next.unsafe_ptr(), key)); - } + using HeadPtr = + Object::BHeadPtr; /** - * Decode next. While traversing a queue, BView and BQueue here will often - * be equal (i.e., AllocUserWild) rather than dichotomous. However, we do - * occasionally decode an actual head pointer, so be polymorphic here. - * - * TODO: We'd like, in some sense, to more tightly couple or integrate this - * into to the domestication process. We could introduce an additional - * state in the capptr_bounds::wild taxonomy (e.g, Obfuscated) so that the - * Domesticator-s below have to call through this function to get the Wild - * pointer they can then make Tame. It's not yet entirely clear what that - * would look like and whether/how the encode_next side of things should be - * exposed. For the moment, obfuscation is left encapsulated within - * FreeObject and we do not capture any of it statically. + * Like HeadPtr, but atomic */ - template< - SNMALLOC_CONCEPT(capptr::ConceptBound) BView, - SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> - inline static BHeadPtr decode_next( - address_t curr, BHeadPtr next, const FreeListKey& key) - { - return BHeadPtr(code_next(curr, next.unsafe_ptr(), key)); - } - - template< - SNMALLOC_CONCEPT(capptr::ConceptBound) BView, - SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> - static void assert_view_queue_bounds() - { - static_assert( - BView::wildness == capptr::dimension::Wildness::Tame, - "FreeObject View must be domesticated, justifying raw pointers"); - - static_assert( - std::is_same_v< - typename BQueue::template with_wildness< - capptr::dimension::Wildness::Tame>, - BView>, - "FreeObject Queue bounds must match View bounds (but may be Wild)"); - } + using AtomicHeadPtr = + Object::BAtomicHeadPtr; /** - * Assign next_object and update its prev_encoded if - * SNMALLOC_CHECK_CLIENT. Static so that it can be used on reference to a - * FreeObject. - * - * Returns a pointer to the next_object field of the next parameter as an - * optimization for repeated snoc operations (in which - * next->next_object is nullptr). + * External code's inductive cases almost always use AllocWild. */ - template< - SNMALLOC_CONCEPT(capptr::ConceptBound) BView, - SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> - static BQueuePtr* store_next( - BQueuePtr* curr, - BHeadPtr next, - const FreeListKey& key) - { - assert_view_queue_bounds(); - -#ifdef SNMALLOC_CHECK_CLIENT - next->prev_encoded = - signed_prev(address_cast(curr), address_cast(next), key); -#else - UNUSED(key); -#endif - *curr = encode_next(address_cast(curr), next, key); - return &(next->next_object); - } + using QueuePtr = Object::BQueuePtr; - template - static void store_null(BQueuePtr* curr, const FreeListKey& key) - { - *curr = encode_next(address_cast(curr), BQueuePtr(nullptr), key); - } + /** + * Like QueuePtr, but atomic + */ + using AtomicQueuePtr = Object::BAtomicQueuePtr; /** - * Assign next_object and update its prev_encoded if SNMALLOC_CHECK_CLIENT + * Used to iterate a free list in object space. * - * Uses the atomic view of next, so can be used in the message queues. + * Checks signing of pointers */ template< - SNMALLOC_CONCEPT(capptr::ConceptBound) BView, - SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> - static void atomic_store_next( - BHeadPtr curr, - BHeadPtr next, - const FreeListKey& key) - { - static_assert(BView::wildness == capptr::dimension::Wildness::Tame); - -#ifdef SNMALLOC_CHECK_CLIENT - next->prev_encoded = - signed_prev(address_cast(curr), address_cast(next), key); -#else - UNUSED(key); -#endif - // Signature needs to be visible before item is linked in - // so requires release semantics. - curr->atomic_next_object.store( - encode_next(address_cast(&curr->next_object), next, key), - std::memory_order_release); - } - - template< - SNMALLOC_CONCEPT(capptr::ConceptBound) BView, - SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue> - static void - atomic_store_null(BHeadPtr curr, const FreeListKey& key) + SNMALLOC_CONCEPT(capptr::ConceptBound) BView = capptr::bounds::Alloc, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue = capptr::bounds::AllocWild> + class Iter { - static_assert(BView::wildness == capptr::dimension::Wildness::Tame); - - curr->atomic_next_object.store( - encode_next( - address_cast(&curr->next_object), BQueuePtr(nullptr), key), - std::memory_order_relaxed); - } - }; - - static_assert( - sizeof(FreeObject) <= MIN_ALLOC_SIZE, - "Needs to be able to fit in smallest allocation."); - - /** - * Used to iterate a free list in object space. - * - * Checks signing of pointers - */ - template< - SNMALLOC_CONCEPT(capptr::ConceptBound) BView = capptr::bounds::Alloc, - SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue = capptr::bounds::AllocWild> - class FreeListIter - { - FreeObject::BHeadPtr curr{nullptr}; + Object::BHeadPtr curr{nullptr}; #ifdef SNMALLOC_CHECK_CLIENT - address_t prev{0}; + address_t prev{0}; #endif - public: - constexpr FreeListIter( - FreeObject::BHeadPtr head, address_t prev_value) - : curr(head) - { + public: + constexpr Iter(Object::BHeadPtr head, address_t prev_value) + : curr(head) + { #ifdef SNMALLOC_CHECK_CLIENT - prev = prev_value; + prev = prev_value; #endif - UNUSED(prev_value); - } + UNUSED(prev_value); + } - constexpr FreeListIter() = default; + constexpr Iter() = default; - /** - * Checks if there are any more values to iterate. - */ - bool empty() - { - return curr == nullptr; - } + /** + * Checks if there are any more values to iterate. + */ + bool empty() + { + return curr == nullptr; + } - /** - * Returns current head without affecting the iterator. - */ - FreeObject::BHeadPtr peek() - { - return curr; - } + /** + * Returns current head without affecting the iterator. + */ + Object::BHeadPtr peek() + { + return curr; + } - /** - * Moves the iterator on, and returns the current value. - */ - template - FreeObject::BHeadPtr - take(const FreeListKey& key, Domesticator domesticate) - { - auto c = curr; - auto next = curr->read_next(key, domesticate); + /** + * Moves the iterator on, and returns the current value. + */ + template + Object::BHeadPtr + take(const FreeListKey& key, Domesticator domesticate) + { + auto c = curr; + auto next = curr->read_next(key, domesticate); - Aal::prefetch(next.unsafe_ptr()); - curr = next; + Aal::prefetch(next.unsafe_ptr()); + curr = next; #ifdef SNMALLOC_CHECK_CLIENT - c->check_prev(prev); - prev = signed_prev(address_cast(c), address_cast(next), key); + c->check_prev(prev); + prev = signed_prev(address_cast(c), address_cast(next), key); #else - UNUSED(key); + UNUSED(key); #endif - return c; - } - }; + return c; + } + }; - /** - * Used to build a free list in object space. - * - * Adds signing of pointers in the SNMALLOC_CHECK_CLIENT mode - * - * If RANDOM is enabled, the builder uses two queues, and - * "randomly" decides to add to one of the two queues. This - * means that we will maintain a randomisation of the order - * between allocations. - * - * The fields are paired up to give better codegen as then they are offset - * by a power of 2, and the bit extract from the interleaving seed can - * be shifted to calculate the relevant offset to index the fields. - * - * If RANDOM is set to false, then the code does not perform any - * randomisation. - */ - template< - bool RANDOM, - bool INIT = true, - SNMALLOC_CONCEPT(capptr::ConceptBound) BView = capptr::bounds::Alloc, - SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue = capptr::bounds::AllocWild> - class FreeListBuilder - { - static constexpr size_t LENGTH = RANDOM ? 2 : 1; - - /* - * We use native pointers below so that we don't run afoul of strict - * aliasing rules. head is a FreeObject::HeadPtr -- that - * is, a known-domesticated pointer to a queue of wild pointers -- and - * it's usually the case that end is a FreeObject::BQueuePtr* -- - * that is, a known-domesticated pointer to a wild pointer to a queue of - * wild pointers. However, in order to do branchless inserts, we set end - * = &head, which breaks strict aliasing rules with the types as given. - * Fortunately, these are private members and so we can use native - * pointers and just expose a more strongly typed interface. + /** + * Used to build a free list in object space. + * + * Adds signing of pointers in the SNMALLOC_CHECK_CLIENT mode + * + * If RANDOM is enabled, the builder uses two queues, and + * "randomly" decides to add to one of the two queues. This + * means that we will maintain a randomisation of the order + * between allocations. + * + * The fields are paired up to give better codegen as then they are offset + * by a power of 2, and the bit extract from the interleaving seed can + * be shifted to calculate the relevant offset to index the fields. + * + * If RANDOM is set to false, then the code does not perform any + * randomisation. */ - - // Pointer to the first element. - std::array head{nullptr}; - // Pointer to the reference to the last element. - // In the empty case end[i] == &head[i] - // This enables branch free enqueuing. - std::array end{nullptr}; - - FreeObject::BQueuePtr* cast_end(uint32_t ix) + template< + bool RANDOM, + bool INIT = true, + SNMALLOC_CONCEPT(capptr::ConceptBound) BView = capptr::bounds::Alloc, + SNMALLOC_CONCEPT(capptr::ConceptBound) BQueue = capptr::bounds::AllocWild> + class Builder { - return reinterpret_cast*>(end[ix]); - } + static constexpr size_t LENGTH = RANDOM ? 2 : 1; + + /* + * We use native pointers below so that we don't run afoul of strict + * aliasing rules. head is a Object::HeadPtr -- that is, a + * known-domesticated pointer to a queue of wild pointers -- and it's + * usually the case that end is a Object::BQueuePtr* -- that is, a + * known-domesticated pointer to a wild pointer to a queue of wild + * pointers. However, in order to do branchless inserts, we set end = + * &head, which breaks strict aliasing rules with the types as given. + * Fortunately, these are private members and so we can use native + * pointers and just expose a more strongly typed interface. + */ - void set_end(uint32_t ix, FreeObject::BQueuePtr* p) - { - end[ix] = reinterpret_cast(p); - } + // Pointer to the first element. + std::array head{nullptr}; + // Pointer to the reference to the last element. + // In the empty case end[i] == &head[i] + // This enables branch free enqueuing. + std::array end{nullptr}; - FreeObject::BHeadPtr cast_head(uint32_t ix) - { - return FreeObject::BHeadPtr( - static_cast*>(head[ix])); - } + Object::BQueuePtr* cast_end(uint32_t ix) + { + return reinterpret_cast*>(end[ix]); + } - std::array length{}; + void set_end(uint32_t ix, Object::BQueuePtr* p) + { + end[ix] = reinterpret_cast(p); + } - public: - constexpr FreeListBuilder() - { - if (INIT) + Object::BHeadPtr cast_head(uint32_t ix) { - init(); + return Object::BHeadPtr( + static_cast*>(head[ix])); } - } - /** - * Checks if the builder contains any elements. - */ - bool empty() - { - for (size_t i = 0; i < LENGTH; i++) + std::array length{}; + + public: + constexpr Builder() { - if (end[i] != &head[i]) + if (INIT) { - return false; + init(); } } - return true; - } - /** - * Adds an element to the builder - */ - void add( - FreeObject::BHeadPtr n, - const FreeListKey& key, - LocalEntropy& entropy) - { - uint32_t index; - if constexpr (RANDOM) - index = entropy.next_bit(); - else - index = 0; - - set_end(index, FreeObject::store_next(cast_end(index), n, key)); - if constexpr (RANDOM) + /** + * Checks if the builder contains any elements. + */ + bool empty() { - length[index]++; + for (size_t i = 0; i < LENGTH; i++) + { + if (end[i] != &head[i]) + { + return false; + } + } + return true; } - } - /** - * Adds an element to the builder, if we are guaranteed that - * RANDOM is false. This is useful in certain construction - * cases that do not need to introduce randomness, such as - * during the initialisation construction of a free list, which - * uses its own algorithm, or during building remote deallocation - * lists, which will be randomised at the other end. - */ - template - std::enable_if_t - add(FreeObject::BHeadPtr n, const FreeListKey& key) - { - static_assert(RANDOM_ == RANDOM, "Don't set template parameter"); - set_end(0, FreeObject::store_next(cast_end(0), n, key)); - } - - /** - * Makes a terminator to a free list. - */ - SNMALLOC_FAST_PATH void - terminate_list(uint32_t index, const FreeListKey& key) - { - FreeObject::store_null(cast_end(index), key); - } - - /** - * Read head removing potential encoding - * - * Although, head does not require meta-data protection - * as it is not stored in an object allocation. For uniformity - * it is treated like the next_object field in a FreeObject - * and is thus subject to encoding if the next_object pointers - * encoded. - */ - FreeObject::BHeadPtr - read_head(uint32_t index, const FreeListKey& key) - { - return FreeObject::decode_next( - address_cast(&head[index]), cast_head(index), key); - } - - address_t get_fake_signed_prev(uint32_t index, const FreeListKey& key) - { - return signed_prev( - address_cast(&head[index]), address_cast(read_head(index, key)), key); - } - - /** - * Close a free list, and set the iterator parameter - * to iterate it. - * - * In the RANDOM case, it may return only part of the freelist. - * - * The return value is how many entries are still contained in the builder. - */ - SNMALLOC_FAST_PATH uint16_t - close(FreeListIter& fl, const FreeListKey& key) - { - uint32_t i; - if constexpr (RANDOM) + /** + * Adds an element to the builder + */ + void add( + Object::BHeadPtr n, + const FreeListKey& key, + LocalEntropy& entropy) { - SNMALLOC_ASSERT(end[1] != &head[0]); - SNMALLOC_ASSERT(end[0] != &head[1]); - - // Select longest list. - i = length[0] > length[1] ? 0 : 1; + uint32_t index; + if constexpr (RANDOM) + index = entropy.next_bit(); + else + index = 0; + + set_end(index, Object::store_next(cast_end(index), n, key)); + if constexpr (RANDOM) + { + length[index]++; + } } - else + + /** + * Adds an element to the builder, if we are guaranteed that + * RANDOM is false. This is useful in certain construction + * cases that do not need to introduce randomness, such as + * during the initialisation construction of a free list, which + * uses its own algorithm, or during building remote deallocation + * lists, which will be randomised at the other end. + */ + template + std::enable_if_t + add(Object::BHeadPtr n, const FreeListKey& key) { - i = 0; + static_assert(RANDOM_ == RANDOM, "Don't set template parameter"); + set_end(0, Object::store_next(cast_end(0), n, key)); } - terminate_list(i, key); - - fl = {read_head(i, key), get_fake_signed_prev(i, key)}; - - end[i] = &head[i]; + /** + * Makes a terminator to a free list. + */ + SNMALLOC_FAST_PATH void + terminate_list(uint32_t index, const FreeListKey& key) + { + Object::store_null(cast_end(index), key); + } - if constexpr (RANDOM) + /** + * Read head removing potential encoding + * + * Although, head does not require meta-data protection + * as it is not stored in an object allocation. For uniformity + * it is treated like the next_object field in a free Object + * and is thus subject to encoding if the next_object pointers + * encoded. + */ + Object::BHeadPtr + read_head(uint32_t index, const FreeListKey& key) { - length[i] = 0; - return length[1 - i]; + return Object::decode_next( + address_cast(&head[index]), cast_head(index), key); } - else + + address_t get_fake_signed_prev(uint32_t index, const FreeListKey& key) { - return 0; + return signed_prev( + address_cast(&head[index]), address_cast(read_head(index, key)), key); } - } - /** - * Set the builder to a not building state. - */ - constexpr void init() - { - for (size_t i = 0; i < LENGTH; i++) + /** + * Close a free list, and set the iterator parameter + * to iterate it. + * + * In the RANDOM case, it may return only part of the freelist. + * + * The return value is how many entries are still contained in the + * builder. + */ + SNMALLOC_FAST_PATH uint16_t + close(Iter& fl, const FreeListKey& key) { + uint32_t i; + if constexpr (RANDOM) + { + SNMALLOC_ASSERT(end[1] != &head[0]); + SNMALLOC_ASSERT(end[0] != &head[1]); + + // Select longest list. + i = length[0] > length[1] ? 0 : 1; + } + else + { + i = 0; + } + + terminate_list(i, key); + + fl = {read_head(i, key), get_fake_signed_prev(i, key)}; + end[i] = &head[i]; - if (RANDOM) + + if constexpr (RANDOM) { length[i] = 0; + return length[1 - i]; + } + else + { + return 0; } } - } - - template - std::enable_if_t< - !RANDOM_, - std::pair< - FreeObject::BHeadPtr, - FreeObject::BHeadPtr>> - extract_segment(const FreeListKey& key) - { - static_assert(RANDOM_ == RANDOM, "Don't set SFINAE parameter!"); - SNMALLOC_ASSERT(!empty()); - - auto first = read_head(0, key); - // end[0] is pointing to the first field in the object, - // this is doing a CONTAINING_RECORD like cast to get back - // to the actual object. This isn't true if the builder is - // empty, but you are not allowed to call this in the empty case. - auto last = FreeObject::BHeadPtr( - FreeObject::from_next_ptr(cast_end(0))); - init(); - return {first, last}; - } - }; + + /** + * Set the builder to a not building state. + */ + constexpr void init() + { + for (size_t i = 0; i < LENGTH; i++) + { + end[i] = &head[i]; + if (RANDOM) + { + length[i] = 0; + } + } + } + + template + std::enable_if_t< + !RANDOM_, + std::pair< + Object::BHeadPtr, + Object::BHeadPtr>> + extract_segment(const FreeListKey& key) + { + static_assert(RANDOM_ == RANDOM, "Don't set SFINAE parameter!"); + SNMALLOC_ASSERT(!empty()); + + auto first = read_head(0, key); + // end[0] is pointing to the first field in the object, + // this is doing a CONTAINING_RECORD like cast to get back + // to the actual object. This isn't true if the builder is + // empty, but you are not allowed to call this in the empty case. + auto last = + Object::BHeadPtr(Object::from_next_ptr(cast_end(0))); + init(); + return {first, last}; + } + }; + } // namespace freelist } // namespace snmalloc diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 23a69876a..b90cca2d4 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -211,21 +211,21 @@ namespace snmalloc SNMALLOC_FAST_PATH void* small_alloc(size_t size) { // SNMALLOC_ASSUME(size <= sizeclass_to_size(NUM_SIZECLASSES)); - auto domesticate = [this](FreeObject::QueuePtr p) + auto domesticate = [this](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { return capptr_domesticate( core_alloc->backend_state_ptr(), p); }; auto slowpath = [&]( sizeclass_t sizeclass, - FreeListIter<>* fl) SNMALLOC_FAST_PATH_LAMBDA { + freelist::Iter<>* fl) SNMALLOC_FAST_PATH_LAMBDA { if (likely(core_alloc != nullptr)) { return core_alloc->handle_message_queue( []( CoreAlloc* core_alloc, sizeclass_t sizeclass, - FreeListIter<>* fl) { + freelist::Iter<>* fl) { return core_alloc->template small_alloc(sizeclass, *fl); }, core_alloc, diff --git a/src/mem/localcache.h b/src/mem/localcache.h index 593fc3212..1ee55a866 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -13,7 +13,7 @@ namespace snmalloc using Stats = AllocStats; inline static SNMALLOC_FAST_PATH void* - finish_alloc_no_zero(FreeObject::HeadPtr p, sizeclass_t sizeclass) + finish_alloc_no_zero(freelist::HeadPtr p, sizeclass_t sizeclass) { SNMALLOC_ASSERT(Metaslab::is_start_of_object(sizeclass, address_cast(p))); UNUSED(sizeclass); @@ -25,14 +25,14 @@ namespace snmalloc template inline static SNMALLOC_FAST_PATH void* - finish_alloc(FreeObject::HeadPtr p, sizeclass_t sizeclass) + finish_alloc(freelist::HeadPtr p, sizeclass_t sizeclass) { auto r = finish_alloc_no_zero(p, sizeclass); if constexpr (zero_mem == YesZero) SharedStateHandle::Pal::zero(r, sizeclass_to_size(sizeclass)); - // TODO: Should this be zeroing the FreeObject state, in the non-zeroing + // TODO: Should this be zeroing the free Object state, in the non-zeroing // case? return r; @@ -46,7 +46,7 @@ namespace snmalloc // Free list per small size class. These are used for // allocation on the fast path. This part of the code is inspired by // mimalloc. - FreeListIter<> small_fast_free_lists[NUM_SIZECLASSES] = {}; + freelist::Iter<> small_fast_free_lists[NUM_SIZECLASSES] = {}; // This is the entropy for a particular thread. LocalEntropy entropy; @@ -80,7 +80,7 @@ namespace snmalloc { auto& key = entropy.get_free_list_key(); auto domesticate = - [local_state](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + [local_state](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { return capptr_domesticate(local_state, p); }; diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 3b31fa4fa..614dd00a4 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -25,9 +25,9 @@ namespace snmalloc * Data-structure for building the free list for this slab. */ #ifdef SNMALLOC_CHECK_CLIENT - FreeListBuilder free_queue; + freelist::Builder free_queue; #else - FreeListBuilder free_queue; + freelist::Builder free_queue; #endif /** @@ -153,11 +153,11 @@ namespace snmalloc * available objects for this metaslab. */ template - static SNMALLOC_FAST_PATH std::pair + static SNMALLOC_FAST_PATH std::pair alloc_free_list( Domesticator domesticate, Metaslab* meta, - FreeListIter<>& fast_free_list, + freelist::Iter<>& fast_free_list, LocalEntropy& entropy, sizeclass_t sizeclass) { diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index 6e3b0a354..f23f75bdd 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -35,10 +35,10 @@ namespace snmalloc // Store the message queue on a separate cacheline. It is mutable data that // is read by other threads. - alignas(CACHELINE_SIZE) FreeObject::AtomicQueuePtr back{nullptr}; + alignas(CACHELINE_SIZE) freelist::AtomicQueuePtr back{nullptr}; // Store the two ends on different cache lines as access by different // threads. - alignas(CACHELINE_SIZE) FreeObject::QueuePtr front{nullptr}; + alignas(CACHELINE_SIZE) freelist::QueuePtr front{nullptr}; constexpr RemoteAllocator() = default; @@ -48,17 +48,17 @@ namespace snmalloc SNMALLOC_ASSERT(front != nullptr); } - void init(FreeObject::HeadPtr stub) + void init(freelist::HeadPtr stub) { - FreeObject::atomic_store_null(stub, key_global); + freelist::Object::atomic_store_null(stub, key_global); front = capptr_rewild(stub); back.store(front, std::memory_order_relaxed); invariant(); } - FreeObject::QueuePtr destroy() + freelist::QueuePtr destroy() { - FreeObject::QueuePtr fnt = front; + freelist::QueuePtr fnt = front; back.store(nullptr, std::memory_order_relaxed); front = nullptr; return fnt; @@ -66,7 +66,7 @@ namespace snmalloc inline bool is_empty() { - FreeObject::QueuePtr bk = back.load(std::memory_order_relaxed); + freelist::QueuePtr bk = back.load(std::memory_order_relaxed); return bk == front; } @@ -77,22 +77,22 @@ namespace snmalloc */ template void enqueue( - FreeObject::HeadPtr first, - FreeObject::HeadPtr last, + freelist::HeadPtr first, + freelist::HeadPtr last, const FreeListKey& key, Domesticator domesticate) { invariant(); - FreeObject::atomic_store_null(last, key); + freelist::Object::atomic_store_null(last, key); // exchange needs to be a release, so nullptr in next is visible. - FreeObject::QueuePtr prev = + freelist::QueuePtr prev = back.exchange(capptr_rewild(last), std::memory_order_release); - FreeObject::atomic_store_next(domesticate(prev), first, key); + freelist::Object::atomic_store_next(domesticate(prev), first, key); } - FreeObject::QueuePtr peek() + freelist::QueuePtr peek() { return front; } @@ -101,12 +101,12 @@ namespace snmalloc * Returns the front message, or null if not possible to return a message. */ template - std::pair + std::pair dequeue(const FreeListKey& key, Domesticator domesticate) { invariant(); - FreeObject::HeadPtr first = domesticate(front); - FreeObject::HeadPtr next = first->atomic_read_next(key, domesticate); + freelist::HeadPtr first = domesticate(front); + freelist::HeadPtr next = first->atomic_read_next(key, domesticate); if (next != nullptr) { diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index 5a739aa0e..af48527fa 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -16,7 +16,7 @@ namespace snmalloc */ struct RemoteDeallocCache { - std::array, REMOTE_SLOTS> list; + std::array, REMOTE_SLOTS> list; /** * The total amount of memory we are waiting for before we will dispatch @@ -70,7 +70,7 @@ namespace snmalloc const FreeListKey& key) { SNMALLOC_ASSERT(initialised); - auto r = p.template as_reinterpret>(); + auto r = p.template as_reinterpret>(); list[get_slot(target_id, 0)].add(r, key); } @@ -85,7 +85,7 @@ namespace snmalloc size_t post_round = 0; bool sent_something = false; auto domesticate = - [local_state](FreeObject::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + [local_state](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { return capptr_domesticate(local_state, p); }; @@ -114,7 +114,7 @@ namespace snmalloc // Entries could map back onto the "resend" list, // so take copy of the head, mark the last element, // and clear the original list. - FreeListIter<> resend; + freelist::Iter<> resend; list[my_slot].close(resend, key); post_round++; From 97950a9fca7dd1b3a5def7fa40f8204ca8628f15 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Tue, 12 Oct 2021 02:11:57 +0100 Subject: [PATCH 106/302] Optionally consider RemoteAllocator heads Tame --- src/backend/commonconfig.h | 9 +++++++ src/mem/corealloc.h | 19 +++++++++++++- src/mem/remoteallocator.h | 51 ++++++++++++++++++++++++++++++++------ src/mem/remotecache.h | 12 ++++++++- 4 files changed, 81 insertions(+), 10 deletions(-) diff --git a/src/backend/commonconfig.h b/src/backend/commonconfig.h index abc3333e4..befb2e488 100644 --- a/src/backend/commonconfig.h +++ b/src/backend/commonconfig.h @@ -77,6 +77,15 @@ namespace snmalloc * for allocating core allocators. */ bool LocalAllocSupportsLazyInit = true; + + /** + * Are the front and back pointers to the message queue in a RemoteAllocator + * considered to be capptr_bounds::Wildness::Tame (as opposed to Wild)? + * That is, is it presumed that clients or other potentialadversaries cannot + * access the front and back pointers themselves, even if they can access + * the queue nodes themselves (which are always considered Wild)? + */ + bool QueueHeadsAreTame = true; }; /** diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 0188e91d4..4ac256c39 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -424,7 +424,24 @@ namespace snmalloc auto& entry = SharedStateHandle::Pagemap::get_metaentry( local_state, snmalloc::address_cast(p)); - auto r = message_queue().dequeue(key_global, domesticate); + std::pair r; + if constexpr (SharedStateHandle::Options.QueueHeadsAreTame) + { + /* + * The front of the queue has already been validated; just change the + * annotating type. + */ + auto domesticate_first = [](freelist::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { + return freelist::HeadPtr(p.unsafe_ptr()); + }; + r = + message_queue().dequeue(key_global, domesticate_first, domesticate); + } + else + { + r = message_queue().dequeue(key_global, domesticate, domesticate); + } if (unlikely(!r.second)) break; diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index f23f75bdd..6d6655b3c 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -29,6 +29,33 @@ namespace snmalloc */ inline static FreeListKey key_global(0xdeadbeef, 0xbeefdead, 0xdeadbeef); + /** + * + * A RemoteAllocator is the message queue of freed objects. It exposes a MPSC + * append-only atomic queue that uses one xchg per append. + * + * The internal pointers are considered QueuePtr-s to support deployment + * scenarios in which the RemoteAllocator itself is exposed to the client. + * This is excessively paranoid in the common case that the RemoteAllocator-s + * are as "hard" for the client to reach as the Pagemap, which we trust to + * store not just Tame CapPtr<>s but raw C++ pointers. + * + * While we could try to condition the types used here on a flag in the + * backend's `struct Flags Options` value, we instead expose two domesticator + * callbacks at the interface and are careful to use one for the front and + * back values and the other for pointers read from the queue itself. That's + * not ideal, but it lets the client condition its behavior appropriately and + * prevents us from accidentally following either of these pointers in generic + * code. + * + * `domesticate_head` is used for the pointer used to reach the of the queue, + * while `domesticate_queue` is used to traverse the first link in the queue + * itself. In the case that the RemoteAllocator is not easily accessible to + * the client, `domesticate_head` can just be a type coersion, and + * `domesticate_queue` should perform actual validation. If the + * RemoteAllocator is exposed to the client, both Domesticators should perform + * validation. + */ struct alignas(REMOTE_MIN_ALIGN) RemoteAllocator { using alloc_id_t = address_t; @@ -74,13 +101,16 @@ namespace snmalloc /** * Pushes a list of messages to the queue. Each message from first to * last should be linked together through their next pointers. + * + * The Domesticator here is used only on pointers read from the head. See + * the commentary on the class. */ - template + template void enqueue( freelist::HeadPtr first, freelist::HeadPtr last, const FreeListKey& key, - Domesticator domesticate) + Domesticator_head domesticate_head) { invariant(); freelist::Object::atomic_store_null(last, key); @@ -89,7 +119,7 @@ namespace snmalloc freelist::QueuePtr prev = back.exchange(capptr_rewild(last), std::memory_order_release); - freelist::Object::atomic_store_next(domesticate(prev), first, key); + freelist::Object::atomic_store_next(domesticate_head(prev), first, key); } freelist::QueuePtr peek() @@ -99,14 +129,19 @@ namespace snmalloc /** * Returns the front message, or null if not possible to return a message. + * + * Takes a domestication callback for each of "pointers read from head" and + * "pointers read from queue". See the commentary on the class. */ - template - std::pair - dequeue(const FreeListKey& key, Domesticator domesticate) + template + std::pair dequeue( + const FreeListKey& key, + Domesticator_head domesticate_head, + Domesticator_queue domesticate_queue) { invariant(); - freelist::HeadPtr first = domesticate(front); - freelist::HeadPtr next = first->atomic_read_next(key, domesticate); + freelist::HeadPtr first = domesticate_head(front); + freelist::HeadPtr next = first->atomic_read_next(key, domesticate_queue); if (next != nullptr) { diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index af48527fa..46cb4288a 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -103,7 +103,17 @@ namespace snmalloc auto [first, last] = list[i].extract_segment(key); MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry( local_state, address_cast(first)); - entry.get_remote()->enqueue(first, last, key, domesticate); + if constexpr (SharedStateHandle::Options.QueueHeadsAreTame) + { + auto domesticate_nop = [](freelist::QueuePtr p) { + return freelist::HeadPtr(p.unsafe_ptr()); + }; + entry.get_remote()->enqueue(first, last, key, domesticate_nop); + } + else + { + entry.get_remote()->enqueue(first, last, key, domesticate); + } sent_something = true; } } From 2316a5c574120ea4756b14a6833867abdc5f25ef Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Tue, 12 Oct 2021 22:24:10 +0100 Subject: [PATCH 107/302] Add minimal domestication test Instantiate two allocators and arrange for a message to get passed between them by exploiting the existing slow-paths' handling of message queues. Count and CHECK the number of domestication calls during this message passing. For a little more excitement, pave over the forward pointer in the freelist::Object::T that is the message and have the domestication callback patch the original value back; should we somehow fail to invoke the domestication callback on that address, this will induce a crash (WHP, on both CHECK_CLIENT and unchecked builds). --- src/test/func/domestication/domestication.cc | 173 +++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 src/test/func/domestication/domestication.cc diff --git a/src/test/func/domestication/domestication.cc b/src/test/func/domestication/domestication.cc new file mode 100644 index 000000000..ec6e45d9e --- /dev/null +++ b/src/test/func/domestication/domestication.cc @@ -0,0 +1,173 @@ +#include + +#ifdef SNMALLOC_PASS_THROUGH +// This test does not make sense in pass-through +int main() +{ + return 0; +} +#else + +// # define SNMALLOC_TRACING + +# include +# include + +// Specify type of allocator +# define SNMALLOC_PROVIDE_OWN_CONFIG +namespace snmalloc +{ + class CustomGlobals : public BackendAllocator + { + public: + using GlobalPoolState = PoolState>; + + private: + using Backend = BackendAllocator; + SNMALLOC_REQUIRE_CONSTINIT + inline static ChunkAllocatorState slab_allocator_state; + + SNMALLOC_REQUIRE_CONSTINIT + inline static GlobalPoolState alloc_pool; + + public: + /* + * C++, even as late as C++20, has some really quite strict limitations on + * designated initializers. However, as of C++17, we can have constexpr + * lambdas and so can use more of the power of the statement fragment of + * C++, and not just its initializer fragment, to initialize a non-prefix + * subset of the flags (in any order, at that). + */ + static constexpr Flags Options = []() constexpr + { + Flags opts = {}; + opts.QueueHeadsAreTame = false; + return opts; + } + (); + + static ChunkAllocatorState& + get_slab_allocator_state(Backend::LocalState* = nullptr) + { + return slab_allocator_state; + } + + static GlobalPoolState& pool() + { + return alloc_pool; + } + + static void register_clean_up() + { + snmalloc::register_clean_up(); + } + + static inline bool domesticate_trace; + static inline size_t domesticate_count; + static inline uintptr_t* domesticate_patch_location; + static inline uintptr_t domesticate_patch_value; + + /* Verify that a pointer points into the region managed by this config */ + template + static SNMALLOC_FAST_PATH CapPtr< + T, + typename B::template with_wildness> + capptr_domesticate(typename Backend::LocalState*, CapPtr p) + { + domesticate_count++; + + if (domesticate_trace) + { + std::cout << "Domesticating " << p.unsafe_ptr() +# if __has_builtin(__builtin_return_address) + << " from " << __builtin_return_address(0) +# endif + << std::endl; + } + + if ( + domesticate_patch_location != nullptr && + p.template as_reinterpret().unsafe_ptr() == + domesticate_patch_location) + { + std::cout << "Patching over corruption" << std::endl; + *domesticate_patch_location = domesticate_patch_value; + snmalloc::CustomGlobals::domesticate_patch_location = nullptr; + } + + return CapPtr< + T, + typename B::template with_wildness>( + p.unsafe_ptr()); + } + }; + + using Alloc = LocalAllocator; +} + +# define SNMALLOC_NAME_MANGLE(a) test_##a +# include "../../../override/malloc.cc" + +int main() +{ + snmalloc::CustomGlobals::init(); // init pagemap + snmalloc::CustomGlobals::domesticate_count = 0; + + LocalEntropy entropy; + entropy.init(); + key_global = FreeListKey(entropy.get_free_list_key()); + + auto alloc1 = new Alloc(); + + auto p = alloc1->alloc(48); + std::cout << "Allocated p " << p << std::endl; + + // Put that free object on alloc1's remote queue + auto alloc2 = new Alloc(); + alloc2->dealloc(p); + alloc2->flush(); + + // Clobber the linkage but not the back pointer + snmalloc::CustomGlobals::domesticate_patch_location = + static_cast(p); + snmalloc::CustomGlobals::domesticate_patch_value = + *static_cast(p); + memset(p, 0xA5, sizeof(void*)); + + snmalloc::CustomGlobals::domesticate_trace = true; + snmalloc::CustomGlobals::domesticate_count = 0; + + auto q = alloc1->alloc(56); + std::cout << "Allocated q " << q << std::endl; + + snmalloc::CustomGlobals::domesticate_trace = false; + + /* + * Expected domestication calls in the above message passing: + * + * - On !QueueHeadsAreTame builds only, RemoteAllocator::dequeue + * domesticating the front pointer (to the initial stub) + * + * - RemoteAllocator::dequeue domesticating the stub's next pointer (p) + * + * - On !QueueHeadsAreTame builds only, RemoteAllocator::dequeue + * domesticating the front pointer (to p, this time) + * + * - RemoteAllocator::dequeue domesticating nullptr (p is the last message) + * + * - Metaslab::alloc_free_list, domesticating the successor object in the + * newly minted freelist::Iter (i.e., the thing that would be allocated + * after q). + */ + static constexpr size_t expected_count = + snmalloc::CustomGlobals::Options.QueueHeadsAreTame ? 3 : 5; + SNMALLOC_CHECK(snmalloc::CustomGlobals::domesticate_count == expected_count); + + // Prevent the allocators from going out of scope during the above test + alloc1->flush(); + alloc2->flush(); + + return 0; +} + +#endif From ed10717dde3c2674f00aef1043b3891ffc6075c3 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Wed, 13 Oct 2021 12:52:01 +0100 Subject: [PATCH 108/302] RemoteAllocator: dequeue as destructive iterator This avoids repeated double-tapping domestication of the same pointer in !QueueHeadsAreTame builds, by keeping the current "front" pointer to the queue in trusted locations (stack, register) rather than storing it back to possibly client-accessible memory. --- src/mem/corealloc.h | 57 ++++++++++---------- src/mem/remoteallocator.h | 57 +++++++++++++------- src/test/func/domestication/domestication.cc | 5 +- 3 files changed, 67 insertions(+), 52 deletions(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 4ac256c39..7ec0a31cf 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -418,37 +418,36 @@ namespace snmalloc [local_state](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { return capptr_domesticate(local_state, p); }; - for (size_t i = 0; i < REMOTE_BATCH; i++) - { - auto p = message_queue().peek(); - auto& entry = SharedStateHandle::Pagemap::get_metaentry( - local_state, snmalloc::address_cast(p)); - - std::pair r; - if constexpr (SharedStateHandle::Options.QueueHeadsAreTame) - { - /* - * The front of the queue has already been validated; just change the - * annotating type. - */ - auto domesticate_first = [](freelist::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { - return freelist::HeadPtr(p.unsafe_ptr()); - }; - r = - message_queue().dequeue(key_global, domesticate_first, domesticate); - } - else - { - r = message_queue().dequeue(key_global, domesticate, domesticate); - } - - if (unlikely(!r.second)) - break; + size_t i = 0; + auto cb = [this, local_state, &need_post, &i](freelist::HeadPtr msg) + SNMALLOC_FAST_PATH_LAMBDA { #ifdef SNMALLOC_TRACING - std::cout << "Handling remote" << std::endl; + std::cout << "Handling remote" << std::endl; #endif - handle_dealloc_remote(entry, r.first.as_void(), need_post); + + auto& entry = SharedStateHandle::Pagemap::get_metaentry( + local_state, snmalloc::address_cast(msg)); + + handle_dealloc_remote(entry, msg.as_void(), need_post); + + return (i++ < REMOTE_BATCH); + }; + + if constexpr (SharedStateHandle::Options.QueueHeadsAreTame) + { + /* + * The front of the queue has already been validated; just change the + * annotating type. + */ + auto domesticate_first = [](freelist::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { + return freelist::HeadPtr(p.unsafe_ptr()); + }; + message_queue().dequeue(key_global, domesticate_first, domesticate, cb); + } + else + { + message_queue().dequeue(key_global, domesticate, domesticate, cb); } if (need_post) diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index 6d6655b3c..d316d3c32 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -128,37 +128,56 @@ namespace snmalloc } /** - * Returns the front message, or null if not possible to return a message. + * Destructively iterate the queue. Each queue element is removed and fed + * to the callback in turn. The callback may return false to stop iteration + * early (but must have processed the element it was given!). * * Takes a domestication callback for each of "pointers read from head" and * "pointers read from queue". See the commentary on the class. */ - template - std::pair dequeue( + template< + typename Domesticator_head, + typename Domesticator_queue, + typename Cb> + void dequeue( const FreeListKey& key, Domesticator_head domesticate_head, - Domesticator_queue domesticate_queue) + Domesticator_queue domesticate_queue, + Cb cb) { invariant(); - freelist::HeadPtr first = domesticate_head(front); - freelist::HeadPtr next = first->atomic_read_next(key, domesticate_queue); + freelist::HeadPtr curr = domesticate_head(front); + freelist::HeadPtr next = curr->atomic_read_next(key, domesticate_queue); - if (next != nullptr) + while (next != nullptr) { - /* - * We've domesticate_queue-d next so that we can read through it, but - * we're storing it back into client-accessible memory in - * !QueueHeadsAreTame builds, so go ahead and consider it Wild again. - * On QueueHeadsAreTame builds, the subsequent domesticate_head call - * above will also be a type-level sleight of hand, but we can still - * justify it by the domesticate_queue that happened in this dequeue(). - */ - front = capptr_rewild(next); - invariant(); - return {first, true}; + if (!cb(curr)) + { + /* + * We've domesticate_queue-d next so that we can read through it, but + * we're storing it back into client-accessible memory in + * !QueueHeadsAreTame builds, so go ahead and consider it Wild again. + * On QueueHeadsAreTame builds, the subsequent domesticate_head call + * above will also be a type-level sleight of hand, but we can still + * justify it by the domesticate_queue that happened in this + * dequeue(). + */ + front = capptr_rewild(next); + invariant(); + return; + } + + curr = next; + next = next->atomic_read_next(key, domesticate_queue); } - return {nullptr, false}; + /* + * Here, we've hit the end of the queue: next is nullptr and curr has not + * been handed to the callback. The same considerations about Wildness + * above hold here. + */ + front = capptr_rewild(curr); + invariant(); } alloc_id_t trunc_id() diff --git a/src/test/func/domestication/domestication.cc b/src/test/func/domestication/domestication.cc index ec6e45d9e..26bc096d1 100644 --- a/src/test/func/domestication/domestication.cc +++ b/src/test/func/domestication/domestication.cc @@ -150,9 +150,6 @@ int main() * * - RemoteAllocator::dequeue domesticating the stub's next pointer (p) * - * - On !QueueHeadsAreTame builds only, RemoteAllocator::dequeue - * domesticating the front pointer (to p, this time) - * * - RemoteAllocator::dequeue domesticating nullptr (p is the last message) * * - Metaslab::alloc_free_list, domesticating the successor object in the @@ -160,7 +157,7 @@ int main() * after q). */ static constexpr size_t expected_count = - snmalloc::CustomGlobals::Options.QueueHeadsAreTame ? 3 : 5; + snmalloc::CustomGlobals::Options.QueueHeadsAreTame ? 3 : 4; SNMALLOC_CHECK(snmalloc::CustomGlobals::domesticate_count == expected_count); // Prevent the allocators from going out of scope during the above test From 93bb037c643f6f9ce9d00fa3c56217339a0d3ee2 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 18 Oct 2021 12:04:09 +0100 Subject: [PATCH 109/302] Fixed bounds calculation for domestication Bug: bounds was base,size but code used it as base, end. --- src/backend/fixedglobalconfig.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/backend/fixedglobalconfig.h b/src/backend/fixedglobalconfig.h index 761c371f3..f03bc9946 100644 --- a/src/backend/fixedglobalconfig.h +++ b/src/backend/fixedglobalconfig.h @@ -66,10 +66,8 @@ namespace snmalloc UNUSED(ls); auto address = address_cast(p); - auto bounds = Backend::Pagemap::get_bounds(nullptr); - if ( - (address < bounds.first) || (address > bounds.second) || - ((bounds.second - address) < sz)) + auto [base, length] = Backend::Pagemap::get_bounds(nullptr); + if ((address - base > (length - sz)) || (length < sz)) { return nullptr; } From 6db9a2f0e2934c15158705996d5fcae8bb96bf30 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 18 Oct 2021 12:06:26 +0100 Subject: [PATCH 110/302] Add validate to freelist::Builder The free list builder in a checked build will only validate entries when they are removed. This commit adds a validate method, so they can be checked during teardown. This means that programs that leak memory will still fail if the free list has become corrupt. --- src/mem/corealloc.h | 21 ++++++++++++++++++++- src/mem/freelist.h | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 7ec0a31cf..0105df0f5 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -330,12 +330,31 @@ namespace snmalloc return chunk_record; } + template SNMALLOC_SLOW_PATH void dealloc_local_slabs(sizeclass_t sizeclass) { // Return unused slabs of sizeclass_t back to global allocator alloc_classes[sizeclass].queue.filter([this, sizeclass](Metaslab* meta) { + auto domesticate = + [this](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + auto res = + capptr_domesticate(backend_state_ptr(), p); +#ifdef SNMALLOC_TRACING + if (res.unsafe_ptr() != p.unsafe_ptr()) + printf( + "Domesticated %p to %p!\n", p.unsafe_ptr(), res.unsafe_ptr()); +#endif + return res; + }; + if (meta->needed() != 0) + { + if (check_slabs) + { + meta->free_queue.validate(entropy.get_free_list_key(), domesticate); + } return false; + } alloc_classes[sizeclass].length--; alloc_classes[sizeclass].unused--; @@ -789,7 +808,7 @@ namespace snmalloc // We may now have unused slabs, return to the global allocator. for (sizeclass_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++) { - dealloc_local_slabs(sizeclass); + dealloc_local_slabs(sizeclass); } return posted; diff --git a/src/mem/freelist.h b/src/mem/freelist.h index b58978ab7..7b6e3ca5e 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -116,6 +116,13 @@ namespace snmalloc template class T { + template< + bool, + bool, + SNMALLOC_CONCEPT(capptr::ConceptBound), + SNMALLOC_CONCEPT(capptr::ConceptBound)> + friend class Builder; + friend class Object; union @@ -699,6 +706,40 @@ namespace snmalloc init(); return {first, last}; } + + template + SNMALLOC_FAST_PATH void + validate(const FreeListKey& key, Domesticator domesticate) + { +#ifdef SNMALLOC_CHECK_CLIENT + for (uint32_t i = 0; i < LENGTH; i++) + { + if (&head[i] == end[i]) + { + SNMALLOC_ASSERT(length[i] == 0); + continue; + } + + size_t count = 1; + auto curr = read_head(i, key); + auto prev = get_fake_signed_prev(i, key); + while (true) + { + curr->check_prev(prev); + if (address_cast(&(curr->next_object)) == address_cast(end[i])) + break; + count++; + auto next = curr->read_next(key, domesticate); + prev = signed_prev(address_cast(curr), address_cast(next), key); + curr = next; + } + SNMALLOC_ASSERT(count == length[i]); + } +#else + UNUSED(key); + UNUSED(domesticate); +#endif + } }; } // namespace freelist } // namespace snmalloc From 1de28eead7f66a84ae39df2461a24749b7f8ae31 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Mon, 18 Oct 2021 16:41:03 +0100 Subject: [PATCH 111/302] test/func/malloc fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `check_result()` `abort()` on `null` and non-`nullptr` result. Otherwise it just prints and doesn't end the test - Don't call `realloc(, 0)`; this has never been consistent and the current C2x draft (see §7.22.3.5 and N2464) finally just declares it to be undefined behavior. POSIX (2017) tolerates our current behavior of freeing the passed-in pointer and returning a new object. --- src/test/func/malloc/malloc.cc | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index dfd92e2fa..0c426a781 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -20,9 +20,8 @@ void check_result(size_t size, size_t align, void* p, int err, bool null) if (p != nullptr) { printf("Expected null, and got non-null return!\n"); - failed = true; + abort(); } - our_free(p); return; } @@ -183,7 +182,6 @@ int main(int argc, char** argv) { const size_t size = sizeclass_to_size(sc); test_realloc(our_malloc(size), size, SUCCESS, false); - test_realloc(our_malloc(size), 0, SUCCESS, true); test_realloc(nullptr, size, SUCCESS, false); test_realloc(our_malloc(size), ((size_t)-1) / 2, ENOMEM, true); for (sizeclass_t sc2 = 0; sc2 < NUM_SIZECLASSES; sc2++) @@ -198,7 +196,6 @@ int main(int argc, char** argv) { const size_t size = bits::one_at_bit(sc); test_realloc(our_malloc(size), size, SUCCESS, false); - test_realloc(our_malloc(size), 0, SUCCESS, true); test_realloc(nullptr, size, SUCCESS, false); test_realloc(our_malloc(size), ((size_t)-1) / 2, ENOMEM, true); for (sizeclass_t sc2 = 0; sc2 < (MAX_SIZECLASS_BITS + 4); sc2++) From dba795ac6fec339119babb83e34073d7c1bb343d Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Sun, 17 Oct 2021 18:37:38 +0100 Subject: [PATCH 112/302] NFC: move some capptr utility functions to that namespace We'll want user_address_control_type in some particular PALs, so it can't live in pal.h. While here, make the spelling be capptr::is_spatial_refinement. --- src/aal/aal.h | 2 +- src/ds/ptrwrap.h | 62 ++++++++++++++++++++++++++++-------------------- src/pal/pal.h | 16 +++---------- 3 files changed, 40 insertions(+), 40 deletions(-) diff --git a/src/aal/aal.h b/src/aal/aal.h index 7e2af3755..821d8a3ed 100644 --- a/src/aal/aal.h +++ b/src/aal/aal.h @@ -197,7 +197,7 @@ namespace snmalloc { // Impose constraints on bounds annotations. static_assert(BIn::spatial >= capptr::dimension::Spatial::Chunk); - static_assert(capptr_is_spatial_refinement()); + static_assert(capptr::is_spatial_refinement()); UNUSED(size); return CapPtr(a.template as_static().unsafe_capptr); diff --git a/src/ds/ptrwrap.h b/src/ds/ptrwrap.h index 6a0b024e1..9ee2cf20a 100644 --- a/src/ds/ptrwrap.h +++ b/src/ds/ptrwrap.h @@ -48,7 +48,7 @@ namespace snmalloc /** * On some platforms (e.g., CHERI), pointers can be checked to see whether * they authorize control of the address space. See the PAL's - * capptr_export(). + * capptr_to_user_address_control(). */ enum class AddressSpaceControl { @@ -188,37 +188,47 @@ namespace snmalloc */ using AllocWild = Alloc::with_wildness; } // namespace bounds - } // namespace capptr - /** - * Determine whether BI is a spatial refinement of BO. - * Chunk and ChunkD are considered eqivalent here. - */ - template< - SNMALLOC_CONCEPT(capptr::ConceptBound) BI, - SNMALLOC_CONCEPT(capptr::ConceptBound) BO> - SNMALLOC_CONSTEVAL bool capptr_is_spatial_refinement() - { - if (BI::address_space_control != BO::address_space_control) - { - return false; - } + /** + * Compute the AddressSpaceControl::User variant of a capptr::bound + * annotation. This is used by the PAL's capptr_to_user_address_control + * function to compute its return value's annotation. + */ + template + using user_address_control_type = + typename B::template with_address_space_control< + dimension::AddressSpaceControl::User>; - if (BI::wildness != BO::wildness) + /** + * Determine whether BI is a spatial refinement of BO. + * Chunk and ChunkD are considered eqivalent here. + */ + template< + SNMALLOC_CONCEPT(capptr::ConceptBound) BI, + SNMALLOC_CONCEPT(capptr::ConceptBound) BO> + SNMALLOC_CONSTEVAL bool is_spatial_refinement() { - return false; - } + if (BI::address_space_control != BO::address_space_control) + { + return false; + } - switch (BI::spatial) - { - using namespace capptr::dimension; - case Spatial::Chunk: - return true; + if (BI::wildness != BO::wildness) + { + return false; + } + + switch (BI::spatial) + { + using namespace capptr::dimension; + case Spatial::Chunk: + return true; - case Spatial::Alloc: - return BO::spatial == Spatial::Alloc; + case Spatial::Alloc: + return BO::spatial == Spatial::Alloc; + } } - } + } // namespace capptr /** * A pointer annotated with a "phantom type parameter" carrying a static diff --git a/src/pal/pal.h b/src/pal/pal.h index a2ad11556..6e2ef2a71 100644 --- a/src/pal/pal.h +++ b/src/pal/pal.h @@ -69,16 +69,6 @@ namespace snmalloc // Used to keep Superslab metadata committed. static constexpr size_t OS_PAGE_SIZE = Pal::page_size; - /** - * Compute the AddressSpaceControl::User variant of a capptr::bound - * annotation. This is used by the PAL's capptr_export function to compute - * its return value's annotation. - */ - template - using capptr_user_address_control_type = - typename B::template with_address_space_control< - capptr::dimension::AddressSpaceControl::User>; - /** * Perform platform-specific adjustment of return pointers. * @@ -92,10 +82,10 @@ namespace snmalloc SNMALLOC_CONCEPT(capptr::ConceptBound) B> static inline typename std::enable_if_t< !aal_supports, - CapPtr>> + CapPtr>> capptr_to_user_address_control(CapPtr p) { - return CapPtr>(p.unsafe_capptr); + return CapPtr>(p.unsafe_capptr); } template< @@ -105,7 +95,7 @@ namespace snmalloc SNMALLOC_CONCEPT(capptr::ConceptBound) B> static SNMALLOC_FAST_PATH typename std::enable_if_t< aal_supports, - CapPtr>> + CapPtr>> capptr_to_user_address_control(CapPtr p) { return PAL::capptr_to_user_address_control(p); From c2ce9c118d9deba96cb552904f8ae6a0087fca86 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Sun, 17 Oct 2021 17:23:39 +0100 Subject: [PATCH 113/302] NFC: Racing stripes on Metaslab accessors Even on debug builds, these little things should be inlined --- src/mem/metaslab.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 614dd00a4..7e61daefd 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -203,10 +203,12 @@ namespace snmalloc * the second argument of this must always be the return value from * `get_remote_and_sizeclass`. */ + SNMALLOC_FAST_PATH MetaEntry(Metaslab* meta, uintptr_t remote_and_sizeclass) : meta(meta), remote_and_sizeclass(remote_and_sizeclass) {} + SNMALLOC_FAST_PATH MetaEntry(Metaslab* meta, RemoteAllocator* remote, sizeclass_t sizeclass) : meta(meta) { @@ -215,7 +217,7 @@ namespace snmalloc pointer_offset(reinterpret_cast(remote), sizeclass); } - [[nodiscard]] Metaslab* get_metaslab() const + [[nodiscard]] SNMALLOC_FAST_PATH Metaslab* get_metaslab() const { return meta; } @@ -226,18 +228,18 @@ namespace snmalloc * only safe use for this is to pass it to the two-argument constructor of * this class. */ - uintptr_t get_remote_and_sizeclass() + [[nodiscard]] SNMALLOC_FAST_PATH uintptr_t get_remote_and_sizeclass() { return remote_and_sizeclass; } - [[nodiscard]] RemoteAllocator* get_remote() const + [[nodiscard]] SNMALLOC_FAST_PATH RemoteAllocator* get_remote() const { return reinterpret_cast( pointer_align_down(remote_and_sizeclass)); } - [[nodiscard]] sizeclass_t get_sizeclass() const + [[nodiscard]] SNMALLOC_FAST_PATH sizeclass_t get_sizeclass() const { return remote_and_sizeclass & (alignof(RemoteAllocator) - 1); } From 3462c53983d7609f043ff5df2c957ef09f057ed3 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Fri, 20 Aug 2021 13:10:31 +0100 Subject: [PATCH 114/302] NFC: Remove spurious forward declarations There is no such thing as "struct Slab" any more. We use alignof(RemoteAllocator) below, so we already require the complete type definition at this point. --- src/mem/metaslab.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 7e61daefd..9ff584cd9 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -9,8 +9,6 @@ namespace snmalloc { - class Slab; - // The Metaslab represent the status of a single slab. // This can be either a short or a standard slab. class alignas(CACHELINE_SIZE) Metaslab @@ -184,8 +182,6 @@ namespace snmalloc } }; - struct RemoteAllocator; - /** * Entry stored in the pagemap. */ From c54d2b527d12e8bf17e3f61b908289aaa02bb9dc Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Tue, 19 Oct 2021 13:33:10 +0100 Subject: [PATCH 115/302] NFC: CapPtr in external_pointer; drop capptr_rebound capptr_rebound was only ever going to be used for external_pointer, which now operates entirely using pointer_offset. So instead, just make external_pointer use capptr::AllocWild, capptr_from_client, and a new capptr_reveal_wild. --- src/aal/aal.h | 15 --------------- src/aal/aal_concept.h | 6 ------ src/ds/ptrwrap.h | 10 ++++++++++ src/mem/localalloc.h | 24 +++++++++++++----------- 4 files changed, 23 insertions(+), 32 deletions(-) diff --git a/src/aal/aal.h b/src/aal/aal.h index 821d8a3ed..45669e527 100644 --- a/src/aal/aal.h +++ b/src/aal/aal.h @@ -202,21 +202,6 @@ namespace snmalloc UNUSED(size); return CapPtr(a.template as_static().unsafe_capptr); } - - /** - * For architectures which do not enforce StrictProvenance, there's nothing - * to be done, so just return the pointer unmodified with new annotation. - */ - template< - typename T, - SNMALLOC_CONCEPT(capptr::ConceptBound) BOut, - SNMALLOC_CONCEPT(capptr::ConceptBound) BIn> - static SNMALLOC_FAST_PATH CapPtr - capptr_rebound(CapPtr a, CapPtr r) noexcept - { - UNUSED(a); - return CapPtr(r.unsafe_capptr); - } }; } // namespace snmalloc diff --git a/src/aal/aal_concept.h b/src/aal/aal_concept.h index 2a14a94b4..15eed19bc 100644 --- a/src/aal/aal_concept.h +++ b/src/aal/aal_concept.h @@ -53,12 +53,6 @@ namespace snmalloc { AAL::template capptr_bound(auth, sz) } noexcept -> ConceptSame>; - - /** - * Construct a copy of auth with its target set to that of ret. - */ - { AAL::capptr_rebound(auth, ret) } noexcept - -> ConceptSame>; }; template diff --git a/src/ds/ptrwrap.h b/src/ds/ptrwrap.h index 9ee2cf20a..cec1874ac 100644 --- a/src/ds/ptrwrap.h +++ b/src/ds/ptrwrap.h @@ -371,6 +371,16 @@ namespace snmalloc return p.unsafe_capptr; } + /** + * Like capptr_reveal, but sometimes we do mean to reveal wild pointers + * (specifically in external_pointer, where we're revealing something + * architecturally derived from a user pointer). + */ + inline SNMALLOC_FAST_PATH void* capptr_reveal_wild(capptr::AllocWild p) + { + return p.unsafe_capptr; + } + /** * Given a void* from the client, it's fine to call it AllocWild. * Roughly dual to capptr_reveal(). diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index b90cca2d4..7c48b2a35 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -628,26 +628,28 @@ namespace snmalloc // be implicit domestication through the `SharedStateHandle::Pagemap` or // we could just leave well enough alone. - // TODO bring back the CHERI bits. Wes to review if required. + capptr::AllocWild p = capptr_from_client(p_raw); + MetaEntry entry = SharedStateHandle::Pagemap::template get_metaentry( - core_alloc->backend_state_ptr(), address_cast(p_raw)); + core_alloc->backend_state_ptr(), address_cast(p)); auto sizeclass = entry.get_sizeclass(); if (likely(entry.get_remote() != SharedStateHandle::fake_large_remote)) { auto rsize = sizeclass_to_size(sizeclass); - auto offset = - address_cast(p_raw) & (sizeclass_to_slab_size(sizeclass) - 1); + auto offset = address_cast(p) & (sizeclass_to_slab_size(sizeclass) - 1); auto start_offset = round_by_sizeclass(sizeclass, offset); if constexpr (location == Start) { UNUSED(rsize); - return pointer_offset(p_raw, start_offset - offset); + return capptr_reveal_wild(pointer_offset(p, start_offset - offset)); } else if constexpr (location == End) - return pointer_offset(p_raw, rsize + start_offset - offset - 1); + return capptr_reveal_wild( + pointer_offset(p, rsize + start_offset - offset - 1)); else - return pointer_offset(p_raw, rsize + start_offset - offset); + return capptr_reveal_wild( + pointer_offset(p, rsize + start_offset - offset)); } // Sizeclass zero of a large allocation is used for not managed by us. @@ -655,13 +657,13 @@ namespace snmalloc { // This is a large allocation, find start by masking. auto rsize = bits::one_at_bit(sizeclass); - auto start = pointer_align_down(p_raw, rsize); + auto start = pointer_align_down(p, rsize); if constexpr (location == Start) - return start; + return capptr_reveal_wild(start); else if constexpr (location == End) - return pointer_offset(start, rsize - 1); + return capptr_reveal_wild(pointer_offset(start, rsize - 1)); else - return pointer_offset(start, rsize); + return capptr_reveal_wild(pointer_offset(start, rsize)); } #else UNUSED(p_raw); From 52a4b0c8d01eb1594794a6ce977202855d52985e Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Wed, 20 Oct 2021 02:43:34 +0100 Subject: [PATCH 116/302] NFC: Make unsafe_capptr private, use unsafe_ptr() --- src/aal/aal.h | 2 +- src/ds/ptrwrap.h | 18 ++++++++++-------- src/pal/pal.h | 4 ++-- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/aal/aal.h b/src/aal/aal.h index 45669e527..5408723bc 100644 --- a/src/aal/aal.h +++ b/src/aal/aal.h @@ -200,7 +200,7 @@ namespace snmalloc static_assert(capptr::is_spatial_refinement()); UNUSED(size); - return CapPtr(a.template as_static().unsafe_capptr); + return CapPtr(a.template as_static().unsafe_ptr()); } }; } // namespace snmalloc diff --git a/src/ds/ptrwrap.h b/src/ds/ptrwrap.h index cec1874ac..8b91a01bb 100644 --- a/src/ds/ptrwrap.h +++ b/src/ds/ptrwrap.h @@ -235,10 +235,11 @@ namespace snmalloc * summary of its StrictProvenance metadata. */ template - struct CapPtr + class CapPtr { T* unsafe_capptr; + public: /** * nullptr is implicitly constructable at any bounds type */ @@ -358,7 +359,7 @@ namespace snmalloc inline SNMALLOC_FAST_PATH capptr::Alloc capptr_chunk_is_alloc(capptr::ChunkUser p) { - return capptr::Alloc(p.unsafe_capptr); + return capptr::Alloc(p.unsafe_ptr()); } /** @@ -368,7 +369,7 @@ namespace snmalloc */ inline SNMALLOC_FAST_PATH void* capptr_reveal(capptr::Alloc p) { - return p.unsafe_capptr; + return p.unsafe_ptr(); } /** @@ -378,7 +379,7 @@ namespace snmalloc */ inline SNMALLOC_FAST_PATH void* capptr_reveal_wild(capptr::AllocWild p) { - return p.unsafe_capptr; + return p.unsafe_ptr(); } /** @@ -403,7 +404,7 @@ namespace snmalloc return CapPtr< T, typename B::template with_wildness>( - p.unsafe_capptr); + p.unsafe_ptr()); } /** @@ -416,10 +417,11 @@ namespace snmalloc * will expose or consume only CapPtr with the same bounds annotation. */ template - struct AtomicCapPtr + class AtomicCapPtr { std::atomic unsafe_capptr; + public: /** * nullptr is constructable at any bounds type */ @@ -458,7 +460,7 @@ namespace snmalloc CapPtr desired, std::memory_order order = std::memory_order_seq_cst) noexcept { - this->unsafe_capptr.store(desired.unsafe_capptr, order); + this->unsafe_capptr.store(desired.unsafe_ptr(), order); } SNMALLOC_FAST_PATH CapPtr exchange( @@ -466,7 +468,7 @@ namespace snmalloc std::memory_order order = std::memory_order_seq_cst) noexcept { return CapPtr( - this->unsafe_capptr.exchange(desired.unsafe_capptr, order)); + this->unsafe_capptr.exchange(desired.unsafe_ptr(), order)); } SNMALLOC_FAST_PATH bool operator==(const AtomicCapPtr& rhs) const diff --git a/src/pal/pal.h b/src/pal/pal.h index 6e2ef2a71..096df25bf 100644 --- a/src/pal/pal.h +++ b/src/pal/pal.h @@ -85,7 +85,7 @@ namespace snmalloc CapPtr>> capptr_to_user_address_control(CapPtr p) { - return CapPtr>(p.unsafe_capptr); + return CapPtr>(p.unsafe_ptr()); } template< @@ -118,7 +118,7 @@ namespace snmalloc { static_assert( !page_aligned || B::spatial >= capptr::dimension::Spatial::Chunk); - PAL::template zero(p.unsafe_capptr, sz); + PAL::template zero(p.unsafe_ptr(), sz); } static_assert( From 554db3997dba26aeafbb859946e24d2da6a49c6b Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Sun, 17 Oct 2021 18:03:39 +0100 Subject: [PATCH 117/302] localalloc::dealloc tweak size math for large objects Avoid computing bits::next_pow2_bits(1 << n). Even if the compiler can see through enough of the algebra, it's surely more direct to just use n. While here, slightly expand documentation about what's going on with the "sizeclass" encoded into MetaEntry-s. --- src/backend/address_space_core.h | 5 ++++- src/mem/localalloc.h | 16 +++++++++++++--- src/mem/metaslab.h | 12 ++++++++++++ src/mem/sizeclasstable.h | 10 ++++++++++ 4 files changed, 39 insertions(+), 4 deletions(-) diff --git a/src/backend/address_space_core.h b/src/backend/address_space_core.h index d3088c0fe..03308c002 100644 --- a/src/backend/address_space_core.h +++ b/src/backend/address_space_core.h @@ -77,13 +77,16 @@ namespace snmalloc { if (align_bits >= MIN_CHUNK_BITS) { - // The pagemap stores MetaEntrys, abuse the metaslab field to be the + // The pagemap stores `MetaEntry`s; abuse the metaslab field to be the // next block in the stack of blocks. // // The pagemap entries here have nullptr (i.e., fake_large_remote) as // their remote, and so other accesses to the pagemap (by // external_pointer, for example) will not attempt to follow this // "Metaslab" pointer. + // + // dealloc() can reject attempts to free such MetaEntry-s due to the + // zero sizeclass. MetaEntry t(reinterpret_cast(next.unsafe_ptr()), nullptr, 0); Pagemap::set_metaentry(local_state, address_cast(base), 1, t); return; diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 7c48b2a35..38ee42b22 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -508,21 +508,31 @@ namespace snmalloc // Large deallocation or null. if (likely(p_tame != nullptr)) { + size_t entry_sizeclass = entry.get_sizeclass(); + // Check this is managed by this pagemap. - check_client(entry.get_sizeclass() != 0, "Not allocated by snmalloc."); + // + // TODO: Should this be tested even in the !CHECK_CLIENT case? Things + // go fairly pear-shaped, with the ASM's ranges[] getting cross-linked + // with a ChunkAllocator's chunk_stack[0], which seems bad. + check_client(entry_sizeclass != 0, "Not allocated by snmalloc."); - size_t size = bits::one_at_bit(entry.get_sizeclass()); + size_t size = bits::one_at_bit(entry_sizeclass); + size_t slab_sizeclass = + metaentry_chunk_sizeclass_to_slab_sizeclass(entry_sizeclass); // Check for start of allocation. check_client( pointer_align_down(p_tame, size) == p_tame, "Not start of an allocation."); - size_t slab_sizeclass = large_size_to_chunk_sizeclass(size); # ifdef SNMALLOC_TRACING std::cout << "Large deallocation: " << size << " chunk sizeclass: " << slab_sizeclass << std::endl; +# else + UNUSED(size); # endif + ChunkRecord* slab_record = reinterpret_cast(entry.get_metaslab()); /* diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 9ff584cd9..ce9d6105c 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -188,6 +188,18 @@ namespace snmalloc class MetaEntry { Metaslab* meta{nullptr}; + + /** + * A bit-packed pointer to the owning allocator (if any), and the sizeclass + * of this chunk. The sizeclass here is itself a union between two cases: + * + * * log_2(size), at least MIN_CHUNK_BITS, for large allocations. + * + * * a value in [0, NUM_SIZECLASSES] for small allocations. These may be + * directly passed to the sizeclass (not slab_sizeclass) functions of + * sizeclasstable.h + * + */ uintptr_t remote_and_sizeclass{0}; public: diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index c406ac1f2..ebd091fa6 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -165,6 +165,16 @@ namespace snmalloc return bits::one_at_bit(MIN_CHUNK_BITS + sizeclass); } + /** + * For large allocations, the metaentry stores the raw log_2 of the size, + * which must be shifted into the index space of slab_sizeclass-es. + */ + inline static size_t + metaentry_chunk_sizeclass_to_slab_sizeclass(sizeclass_t sizeclass) + { + return sizeclass - MIN_CHUNK_BITS; + } + inline constexpr static uint16_t sizeclass_to_slab_object_count(sizeclass_t sizeclass) { From 7f6378f5bfb04e2e5be8387057072216dcde5cd6 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Mon, 18 Oct 2021 21:22:42 +0100 Subject: [PATCH 118/302] test/func/domestication: tweak for robustness - Grab a larger second allocation on the first allocator to dodge the sizeclass of the prior alloc on that allocator *and* any implicit, bootstrapping slabs that get opened (e.g., for remote queue message stubs). - De-FAST_PATH the domestication function. No need to always inline it, here. - Document things a little better --- src/test/func/domestication/domestication.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/test/func/domestication/domestication.cc b/src/test/func/domestication/domestication.cc index 26bc096d1..a1607de47 100644 --- a/src/test/func/domestication/domestication.cc +++ b/src/test/func/domestication/domestication.cc @@ -69,7 +69,7 @@ namespace snmalloc /* Verify that a pointer points into the region managed by this config */ template - static SNMALLOC_FAST_PATH CapPtr< + static CapPtr< T, typename B::template with_wildness> capptr_domesticate(typename Backend::LocalState*, CapPtr p) @@ -119,6 +119,8 @@ int main() auto alloc1 = new Alloc(); + // Allocate from alloc1; the size doesn't matter a whole lot, it just needs to + // be a small object and so definitely owned by this allocator rather. auto p = alloc1->alloc(48); std::cout << "Allocated p " << p << std::endl; @@ -137,7 +139,9 @@ int main() snmalloc::CustomGlobals::domesticate_trace = true; snmalloc::CustomGlobals::domesticate_count = 0; - auto q = alloc1->alloc(56); + // Open a new slab, so that slow path will pick up the message queue. That + // means this should be a sizeclass we've not used before, even internally. + auto q = alloc1->alloc(512); std::cout << "Allocated q " << q << std::endl; snmalloc::CustomGlobals::domesticate_trace = false; From 31b8d99ca61b900d6215404b4f7a0cf245bbd684 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Mon, 18 Oct 2021 21:49:09 +0100 Subject: [PATCH 119/302] test/func/memcpy: adjust test sizes --- src/test/func/memcpy/func-memcpy.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/test/func/memcpy/func-memcpy.cc b/src/test/func/memcpy/func-memcpy.cc index 1727d3825..202b303d1 100644 --- a/src/test/func/memcpy/func-memcpy.cc +++ b/src/test/func/memcpy/func-memcpy.cc @@ -135,8 +135,13 @@ int main() // Skip the checks that expect bounds checks to fail when we are not the // malloc implementation. # if !defined(SNMALLOC_PASS_THROUGH) - // Some sizes to check for out-of-bounds access - std::initializer_list sizes = {16, 1024, 2 * 1024 * 1024}; + // Some sizes to check for out-of-bounds access. As we are only able to + // catch overflows past the end of the sizeclass-padded allocation, make + // sure we don't try to test on smaller allocations. + std::initializer_list sizes = {MIN_ALLOC_SIZE, 1024, 2 * 1024 * 1024}; + static_assert( + MIN_ALLOC_SIZE < 1024, + "Can't detect overflow except at sizeclass boundaries"); for (auto sz : sizes) { // Check in bounds From 5bb556cb685f31ce8d8d6acf39ce3a70aa6c99ef Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Wed, 20 Oct 2021 02:49:32 +0100 Subject: [PATCH 120/302] Convert alloc paths to capptr::Alloc The use of void* had let an overzealous unsafe_ptr() leak a pointer with address space control to the client (in LocalAllocator::alloc_not_small, specifically). Correct this to call capptr_chunk_is_alloc() (to capture our intent) and capptr_to_user_address_control() (to do the bounding) and defer the conversion to void* until the very periphery of the allocator, using capptr_reveal() (again, to capture intent). --- src/mem/corealloc.h | 15 ++++++++------- src/mem/localalloc.h | 11 +++++------ src/mem/localcache.h | 13 ++++++------- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 0105df0f5..87d4434c7 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -176,7 +176,7 @@ namespace snmalloc * - Allocating stub in the message queue * Note this is not performance critical as very infrequently called. */ - void* small_alloc_one(size_t size) + capptr::Alloc small_alloc_one(size_t size) { SNMALLOC_ASSERT(attached_cache != nullptr); auto domesticate = @@ -285,7 +285,8 @@ namespace snmalloc [local_state](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { return capptr_domesticate(local_state, p); }; - void* p = finish_alloc_no_zero(fl.take(key, domesticate), sizeclass); + capptr::Alloc p = + finish_alloc_no_zero(fl.take(key, domesticate), sizeclass); #ifdef SNMALLOC_CHECK_CLIENT // Check free list is well-formed on platforms with @@ -321,11 +322,11 @@ namespace snmalloc auto start_of_slab = pointer_align_down( p, snmalloc::sizeclass_to_slab_size(sizeclass)); // TODO Add bounds correctly here - chunk_record->chunk = capptr::Chunk(start_of_slab); + chunk_record->chunk = capptr::Chunk(start_of_slab.unsafe_ptr()); #ifdef SNMALLOC_TRACING - std::cout << "Slab " << start_of_slab << " is unused, Object sizeclass " - << sizeclass << std::endl; + std::cout << "Slab " << start_of_slab.unsafe_ptr() + << " is unused, Object sizeclass " << sizeclass << std::endl; #endif return chunk_record; } @@ -654,7 +655,7 @@ namespace snmalloc } template - SNMALLOC_SLOW_PATH void* + SNMALLOC_SLOW_PATH capptr::Alloc small_alloc(sizeclass_t sizeclass, freelist::Iter<>& fast_free_list) { size_t rsize = sizeclass_to_size(sizeclass); @@ -716,7 +717,7 @@ namespace snmalloc } template - SNMALLOC_SLOW_PATH void* small_alloc_slow( + SNMALLOC_SLOW_PATH capptr::Alloc small_alloc_slow( sizeclass_t sizeclass, freelist::Iter<>& fast_free_list, size_t rsize) { // No existing free list get a new slab. diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 38ee42b22..a39eb04e4 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -167,7 +167,7 @@ namespace snmalloc * passed to the core allocator. */ template - SNMALLOC_SLOW_PATH void* alloc_not_small(size_t size) + SNMALLOC_SLOW_PATH capptr::Alloc alloc_not_small(size_t size) { if (size == 0) { @@ -203,12 +203,12 @@ namespace snmalloc chunk.unsafe_ptr(), size); } - return chunk.unsafe_ptr(); + return capptr_chunk_is_alloc(capptr_to_user_address_control(chunk)); }); } template - SNMALLOC_FAST_PATH void* small_alloc(size_t size) + SNMALLOC_FAST_PATH capptr::Alloc small_alloc(size_t size) { // SNMALLOC_ASSUME(size <= sizeclass_to_size(NUM_SIZECLASSES)); auto domesticate = [this](freelist::QueuePtr p) @@ -430,11 +430,10 @@ namespace snmalloc { // Small allocations are more likely. Improve // branch prediction by placing this case first. - return small_alloc(size); + return capptr_reveal(small_alloc(size)); } - // TODO capptr_reveal? - return alloc_not_small(size); + return capptr_reveal(alloc_not_small(size)); #endif } diff --git a/src/mem/localcache.h b/src/mem/localcache.h index 1ee55a866..b7c2be79a 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -12,25 +12,24 @@ namespace snmalloc { using Stats = AllocStats; - inline static SNMALLOC_FAST_PATH void* + inline static SNMALLOC_FAST_PATH capptr::Alloc finish_alloc_no_zero(freelist::HeadPtr p, sizeclass_t sizeclass) { SNMALLOC_ASSERT(Metaslab::is_start_of_object(sizeclass, address_cast(p))); UNUSED(sizeclass); - auto r = capptr_reveal(p.as_void()); - - return r; + return p.as_void(); } template - inline static SNMALLOC_FAST_PATH void* + inline static SNMALLOC_FAST_PATH capptr::Alloc finish_alloc(freelist::HeadPtr p, sizeclass_t sizeclass) { auto r = finish_alloc_no_zero(p, sizeclass); if constexpr (zero_mem == YesZero) - SharedStateHandle::Pal::zero(r, sizeclass_to_size(sizeclass)); + SharedStateHandle::Pal::zero( + r.unsafe_ptr(), sizeclass_to_size(sizeclass)); // TODO: Should this be zeroing the free Object state, in the non-zeroing // case? @@ -105,7 +104,7 @@ namespace snmalloc typename SharedStateHandle, typename Slowpath, typename Domesticator> - SNMALLOC_FAST_PATH void* + SNMALLOC_FAST_PATH capptr::Alloc alloc(Domesticator domesticate, size_t size, Slowpath slowpath) { auto& key = entropy.get_free_list_key(); From f0b7dc4b04e9f0837d3ee7b9b4d97d5db2952684 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Sun, 17 Oct 2021 16:27:29 +0100 Subject: [PATCH 121/302] NFC: Extract MetaCommon from ChunkRecord --- src/mem/corealloc.h | 3 ++- src/mem/localalloc.h | 3 ++- src/mem/metaslab.h | 5 +++++ src/mem/slaballocator.h | 12 ++++++++---- 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 87d4434c7..98c80a865 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -322,7 +322,8 @@ namespace snmalloc auto start_of_slab = pointer_align_down( p, snmalloc::sizeclass_to_slab_size(sizeclass)); // TODO Add bounds correctly here - chunk_record->chunk = capptr::Chunk(start_of_slab.unsafe_ptr()); + chunk_record->meta_common.chunk = + capptr::Chunk(start_of_slab.unsafe_ptr()); #ifdef SNMALLOC_TRACING std::cout << "Slab " << start_of_slab.unsafe_ptr() diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index a39eb04e4..cee763f3c 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -541,7 +541,8 @@ namespace snmalloc * a CapPtr to this region internally even while it's * allocated. */ - slab_record->chunk = capptr::Chunk(p_tame.unsafe_ptr()); + slab_record->meta_common.chunk = + capptr::Chunk(p_tame.unsafe_ptr()); check_init( []( CoreAlloc* core_alloc, diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index ce9d6105c..6527b43fa 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -9,6 +9,11 @@ namespace snmalloc { + struct MetaCommon + { + capptr::Chunk chunk; + }; + // The Metaslab represent the status of a single slab. // This can be either a short or a standard slab. class alignas(CACHELINE_SIZE) Metaslab diff --git a/src/mem/slaballocator.h b/src/mem/slaballocator.h index aa4d47fcf..92a9a1c57 100644 --- a/src/mem/slaballocator.h +++ b/src/mem/slaballocator.h @@ -17,9 +17,13 @@ namespace snmalloc */ struct ChunkRecord { + MetaCommon meta_common; std::atomic next; - capptr::Chunk chunk; }; + static_assert(std::is_standard_layout_v); + static_assert( + offsetof(ChunkRecord, meta_common) == 0, + "ChunkRecord and Metaslab must share a common prefix"); /** * How many slab sizes that can be provided. @@ -97,7 +101,7 @@ namespace snmalloc if (chunk_record != nullptr) { - auto slab = chunk_record->chunk; + auto slab = chunk_record->meta_common.chunk; state.memory_in_stacks -= slab_size; auto meta = reinterpret_cast(chunk_record); #ifdef SNMALLOC_TRACING @@ -138,8 +142,8 @@ namespace snmalloc { auto& state = SharedStateHandle::get_slab_allocator_state(&local_state); #ifdef SNMALLOC_TRACING - std::cout << "Return slab:" << p->chunk.unsafe_ptr() << " slab_sizeclass " - << slab_sizeclass << " size " + std::cout << "Return slab:" << p->meta_common.chunk.unsafe_ptr() + << " slab_sizeclass " << slab_sizeclass << " size " << slab_sizeclass_to_size(slab_sizeclass) << " memory in stacks " << state.memory_in_stacks << std::endl; #endif From 6c115eec1823147a4c8112947b70703331f184d8 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Sun, 17 Oct 2021 18:09:00 +0100 Subject: [PATCH 122/302] NFC: doc tweaks --- src/aal/aal.h | 9 ++++++--- src/backend/address_space_core.h | 10 ++++++++++ src/mem/localalloc.h | 9 ++++++--- src/mem/metaslab.h | 3 +-- src/mem/slaballocator.h | 2 +- 5 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/aal/aal.h b/src/aal/aal.h index 5408723bc..496cfcebe 100644 --- a/src/aal/aal.h +++ b/src/aal/aal.h @@ -195,9 +195,12 @@ namespace snmalloc static SNMALLOC_FAST_PATH CapPtr capptr_bound(CapPtr a, size_t size) noexcept { - // Impose constraints on bounds annotations. - static_assert(BIn::spatial >= capptr::dimension::Spatial::Chunk); - static_assert(capptr::is_spatial_refinement()); + static_assert( + BIn::spatial > capptr::dimension::Spatial::Alloc, + "Refusing to re-bound Spatial::Alloc CapPtr"); + static_assert( + capptr::is_spatial_refinement(), + "capptr_bound must preserve non-spatial CapPtr dimensions"); UNUSED(size); return CapPtr(a.template as_static().unsafe_ptr()); diff --git a/src/backend/address_space_core.h b/src/backend/address_space_core.h index 03308c002..dcbca96ee 100644 --- a/src/backend/address_space_core.h +++ b/src/backend/address_space_core.h @@ -20,6 +20,16 @@ namespace snmalloc * * It cannot unreserve memory, so this does not require the * usual complexity of a buddy allocator. + * + * TODO: This manages pieces of memory smaller than (1U << MIN_CHUNK_BITS) to + * source Metaslab and LocalCache objects. On CHERI, where ASLR and guard + * pages are not needed, it may be worth switching to a design where we + * bootstrap allocators with at least two embedded Metaslab-s that can be used + * to construct slabs for LocalCache and, of course, additional Metaslab + * objects. That would let us stop splitting memory below that threshold + * here, and may reduce address space fragmentation or address space committed + * to Metaslab objects in perpetuity; it could also make {set,get}_next less + * scary. */ class AddressSpaceManagerCore { diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index cee763f3c..1e62123bd 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -276,8 +276,10 @@ namespace snmalloc return; } - // Recheck what kind of dealloc we should do incase, the allocator we - // get from lazy_init is the originating allocator. + // Recheck what kind of dealloc we should do in case the allocator we get + // from lazy_init is the originating allocator. (TODO: but note that this + // can't suddenly become a large deallocation; the only distinction is + // between being ours to handle and something to post to a Remote.) lazy_init( [&](CoreAlloc*, CapPtr p) { dealloc(p.unsafe_ptr()); // TODO don't double count statistics @@ -602,9 +604,10 @@ namespace snmalloc // be implicit domestication through the `SharedStateHandle::Pagemap` or // we could just leave well enough alone. - // Note that this should return 0 for nullptr. + // Note that alloc_size should return 0 for nullptr. // Other than nullptr, we know the system will be initialised as it must // be called with something we have already allocated. + // // To handle this case we require the uninitialised pagemap contain an // entry for the first chunk of memory, that states it represents a // large object, so we can pull the check for null off the fast path. diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 6527b43fa..997439ba1 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -15,7 +15,6 @@ namespace snmalloc }; // The Metaslab represent the status of a single slab. - // This can be either a short or a standard slab. class alignas(CACHELINE_SIZE) Metaslab { public: @@ -192,7 +191,7 @@ namespace snmalloc */ class MetaEntry { - Metaslab* meta{nullptr}; + Metaslab* meta{nullptr}; // may also be ChunkRecord* /** * A bit-packed pointer to the owning allocator (if any), and the sizeclass diff --git a/src/mem/slaballocator.h b/src/mem/slaballocator.h index 92a9a1c57..01b0a704b 100644 --- a/src/mem/slaballocator.h +++ b/src/mem/slaballocator.h @@ -34,7 +34,7 @@ namespace snmalloc * Used to ensure the per slab meta data is large enough for both use cases. */ static_assert( - sizeof(Metaslab) >= sizeof(ChunkRecord), "We conflat these two types."); + sizeof(Metaslab) >= sizeof(ChunkRecord), "We conflate these two types."); /** * This is the global state required for the chunk allocator. From 599ae0e632c54f30075a91ca6c00c52dac1e42ed Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Sun, 17 Oct 2021 16:38:19 +0100 Subject: [PATCH 123/302] Metaslab: add MetaCommon field This preserves the chunk pointer through the use of a chunk as a slab. It does grow the structure by one pointer, but on non-CHERI it is still padded to 64 bytes, even with CHECK_CLIENT guards in place: 0: MetaCommon chunk pointer 8: next pointer 16: builder head[0] 24: builder head[1] 32: builder tail[0] 40: builder tail[1] 48: builder length[0] (uint16_t) 50: builder length[1] (uint16_t) 52: padding (4 bytes) 56: needed (uint16_t) 58: sleeping (bool) (Sadly, on CHERI, even without CHECK_CLIENT guards and with no padding, there are now four pointers in the structure -- chunk, next, head, tail -- plus five extra bytes. We will likely wish to explore encoding the head and tail offsets relative to the chunk pointer.) This lets us remove the "subversive amplification" in dealloc() in favor of just preserving the chunk pointer. Speaking of, be sure to assign that in all the right places, and ASSERT that we've got it right. --- src/backend/backend.h | 2 ++ src/mem/corealloc.h | 9 ++++++--- src/mem/localalloc.h | 13 ++++--------- src/mem/metaslab.h | 14 ++++++++++++++ 4 files changed, 26 insertions(+), 12 deletions(-) diff --git a/src/backend/backend.h b/src/backend/backend.h index 36328f462..1cc9c0bd9 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -87,6 +87,8 @@ namespace snmalloc return {p, nullptr}; } + meta->meta_common.chunk = p; + MetaEntry t(meta, remote, sizeclass); Pagemap::set_metaentry(local_state, address_cast(p), size, t); return {p, meta}; diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 98c80a865..b0fc28eed 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -321,13 +321,16 @@ namespace snmalloc // have the whole chunk. auto start_of_slab = pointer_align_down( p, snmalloc::sizeclass_to_slab_size(sizeclass)); - // TODO Add bounds correctly here - chunk_record->meta_common.chunk = - capptr::Chunk(start_of_slab.unsafe_ptr()); + + SNMALLOC_ASSERT( + address_cast(start_of_slab) == + address_cast(chunk_record->meta_common.chunk)); #ifdef SNMALLOC_TRACING std::cout << "Slab " << start_of_slab.unsafe_ptr() << " is unused, Object sizeclass " << sizeclass << std::endl; +#else + UNUSED(start_of_slab); #endif return chunk_record; } diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 1e62123bd..db4bbfaff 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -536,15 +536,10 @@ namespace snmalloc ChunkRecord* slab_record = reinterpret_cast(entry.get_metaslab()); - /* - * StrictProvenance TODO: this is a subversive amplification. p_tame is - * tame but Alloc-bounded, but we're coercing it to Chunk-bounded. We - * should, instead, not be storing ->chunk here, but should be keeping - * a CapPtr to this region internally even while it's - * allocated. - */ - slab_record->meta_common.chunk = - capptr::Chunk(p_tame.unsafe_ptr()); + + SNMALLOC_ASSERT( + address_cast(slab_record->meta_common.chunk) == address_cast(p_tame)); + check_init( []( CoreAlloc* core_alloc, diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 997439ba1..1057c14c6 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -9,6 +9,13 @@ namespace snmalloc { + /** + * A guaranteed type-stable sub-structure of all metadata referenced by the + * Pagemap. Use-specific structures (Metaslab, ChunkRecord) are expected to + * have this at offset zero so that, even in the face of concurrent mutation + * and reuse of the memory backing that metadata, the types of these fields + * remain fixed. + */ struct MetaCommon { capptr::Chunk chunk; @@ -18,6 +25,8 @@ namespace snmalloc class alignas(CACHELINE_SIZE) Metaslab { public: + MetaCommon meta_common; + // Used to link metaslabs together in various other data-structures. Metaslab* next{nullptr}; @@ -186,6 +195,11 @@ namespace snmalloc } }; + static_assert(std::is_standard_layout_v); + static_assert( + offsetof(Metaslab, meta_common) == 0, + "ChunkRecord and Metaslab must share a common prefix"); + /** * Entry stored in the pagemap. */ From 3cdb7be478040e799ca878af2d00180e2b7c5316 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Sun, 17 Oct 2021 18:12:48 +0100 Subject: [PATCH 124/302] MetaEntry: split get_metaslab by intent --- src/backend/address_space_core.h | 2 +- src/mem/localalloc.h | 4 ++-- src/mem/metaslab.h | 16 ++++++++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/backend/address_space_core.h b/src/backend/address_space_core.h index dcbca96ee..f8b725162 100644 --- a/src/backend/address_space_core.h +++ b/src/backend/address_space_core.h @@ -125,7 +125,7 @@ namespace snmalloc const MetaEntry& t = Pagemap::template get_metaentry( local_state, address_cast(base)); return capptr::Chunk( - reinterpret_cast(t.get_metaslab())); + reinterpret_cast(t.get_metaslab_no_remote())); } return base->next; diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index db4bbfaff..43423af9d 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -534,8 +534,8 @@ namespace snmalloc UNUSED(size); # endif - ChunkRecord* slab_record = - reinterpret_cast(entry.get_metaslab()); + auto slab_record = + static_cast(entry.get_metaslab_no_remote()); SNMALLOC_ASSERT( address_cast(slab_record->meta_common.chunk) == address_cast(p_tame)); diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 1057c14c6..fcbb29397 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -243,8 +243,24 @@ namespace snmalloc pointer_offset(reinterpret_cast(remote), sizeclass); } + /** + * Return the Metaslab field as a void*, guarded by an assert that there is + * no remote that owns this chunk. + */ + [[nodiscard]] SNMALLOC_FAST_PATH void* get_metaslab_no_remote() const + { + SNMALLOC_ASSERT(get_remote() == nullptr); + return static_cast(meta); + } + + /** + * Return the Metaslab metadata associated with this chunk, guarded by an + * assert that this chunk is being used as a slab (i.e., has an associated + * owning allocator). + */ [[nodiscard]] SNMALLOC_FAST_PATH Metaslab* get_metaslab() const { + SNMALLOC_ASSERT(get_remote() != nullptr); return meta; } From 8023a6c906bc847378a6dfa4bfbb99e5bc560463 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Mon, 18 Oct 2021 22:18:14 +0100 Subject: [PATCH 125/302] backend/address_space: drop spurious !StrictProvenance guard This dates back to the much earlier design that required the use of an authority map. Since the current CHERI design does not, go ahead and ask the platform whenever underlying allocation requests are sufficiently aligned. --- src/backend/address_space.h | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/backend/address_space.h b/src/backend/address_space.h index e197e7709..02c48f0b5 100644 --- a/src/backend/address_space.h +++ b/src/backend/address_space.h @@ -53,12 +53,10 @@ namespace snmalloc SNMALLOC_ASSERT(size >= sizeof(void*)); /* - * For sufficiently large allocations with platforms that support - * aligned allocations and architectures that don't require - * StrictProvenance, try asking the platform first. + * For sufficiently large allocations with platforms that support aligned + * allocations, try asking the platform directly. */ - if constexpr ( - pal_supports && !aal_supports) + if constexpr (pal_supports) { if (size >= PAL::minimum_alloc_size) { From fa4560801adf6bf689279dc7fb609b429cc3d206 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Mon, 18 Oct 2021 22:04:43 +0100 Subject: [PATCH 126/302] freelist: check_prev expects an address_t So make signed_prev return one. This distinction only matters on CHERI, of course; everywhere else address_t *is* a uintptr_t. --- src/mem/freelist.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mem/freelist.h b/src/mem/freelist.h index 7b6e3ca5e..4640534ef 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -44,7 +44,7 @@ namespace snmalloc /** * This function is used to sign back pointers in the free list. */ - inline static uintptr_t + inline static address_t signed_prev(address_t curr, address_t next, const FreeListKey& key) { auto c = curr; From fe43f0bea8ef3c69c20018c51b914e372bf544dc Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Mon, 18 Oct 2021 22:03:31 +0100 Subject: [PATCH 127/302] Placate some compiler errors on CHERI --- src/backend/pagemap.h | 2 +- src/mem/metaslab.h | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/backend/pagemap.h b/src/backend/pagemap.h index 9c770f051..fe1f9e76d 100644 --- a/src/backend/pagemap.h +++ b/src/backend/pagemap.h @@ -294,7 +294,7 @@ namespace snmalloc void set(address_t p, T t) { #ifdef SNMALLOC_TRACING - std::cout << "Pagemap.Set " << (void*)p << std::endl; + std::cout << "Pagemap.Set " << (void*)(uintptr_t)p << std::endl; #endif if constexpr (has_bounds) { diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index fcbb29397..c68e95b79 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -283,7 +283,10 @@ namespace snmalloc [[nodiscard]] SNMALLOC_FAST_PATH sizeclass_t get_sizeclass() const { - return remote_and_sizeclass & (alignof(RemoteAllocator) - 1); + // TODO: perhaps remove static_cast with resolution of + // https://github.com/CTSRD-CHERI/llvm-project/issues/588 + return static_cast(remote_and_sizeclass) & + (alignof(RemoteAllocator) - 1); } }; From 97a8bd17585009abf801f1e90c93a4a06abfd3af Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Mon, 19 Jul 2021 09:47:38 +0100 Subject: [PATCH 128/302] Initial RISC-V AAL --- src/aal/aal.h | 6 +++++ src/aal/aal_consts.h | 1 + src/aal/aal_riscv.h | 54 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+) create mode 100644 src/aal/aal_riscv.h diff --git a/src/aal/aal.h b/src/aal/aal.h index 496cfcebe..2ecac0f82 100644 --- a/src/aal/aal.h +++ b/src/aal/aal.h @@ -32,6 +32,10 @@ # define PLATFORM_IS_SPARC #endif +#if defined(__riscv) +# define PLATFORM_IS_RISCV +#endif + namespace snmalloc { /** @@ -218,6 +222,8 @@ namespace snmalloc # include "aal_powerpc.h" #elif defined(PLATFORM_IS_SPARC) # include "aal_sparc.h" +#elif defined(PLATFORM_IS_RISCV) +# include "aal_riscv.h" #endif namespace snmalloc diff --git a/src/aal/aal_consts.h b/src/aal/aal_consts.h index 24b31ff73..8990a41df 100644 --- a/src/aal/aal_consts.h +++ b/src/aal/aal_consts.h @@ -33,5 +33,6 @@ namespace snmalloc X86, X86_SGX, Sparc, + RISCV }; } // namespace snmalloc diff --git a/src/aal/aal_riscv.h b/src/aal/aal_riscv.h new file mode 100644 index 000000000..2d2f7a4f1 --- /dev/null +++ b/src/aal/aal_riscv.h @@ -0,0 +1,54 @@ +#pragma once + +#if __riscv_xlen == 64 +# define SNMALLOC_VA_BITS_64 +#elif __riscv_xlen == 32 +# define SNMALLOC_VA_BITS_32 +#endif + +namespace snmalloc +{ + /** + * RISC-V architecture layer, phrased as generically as possible. Specific + * implementations may need to adjust some of these. + */ + class AAL_RISCV + { + public: + static constexpr uint64_t aal_features = IntegerPointers; + + static constexpr size_t smallest_page_size = 0x1000; + + static constexpr AalName aal_name = RISCV; + + static void inline pause() + { + /* + * The "Zihintpause" extension claims to be the right thing to do here, + * and it is expected to be used in analogous places, e.g., Linux's + * cpu_relax(), but... + * + * its specification is somewhat unusual, in that it talks about the rate + * at which a HART's instructions retire rather than the rate at which + * they are dispatched (Intel's PAUSE instruction explicitly promises + * that it "de-pipelines" the spin-wait loop, for example) or anything + * about memory semantics (Intel's PAUSE docs talk about a possible + * memory order violation and pipeline flush upon loop exit). + * + * we don't yet have examples of what implementations have done. + * + * it's not yet understood by C frontends or assembler, meaning we'd have + * to spell it out by hand, as + * __asm__ volatile(".byte 0xF; .byte 0x0; .byte 0x0; .byte 0x1"); + * + * All told, we just leave this function empty for the moment. The good + * news is that, if and when we do add a PAUSE, the instruction is encoded + * by stealing some dead space of the FENCE instruction and so should be + * available everywhere even if it doesn't do anything on a particular + * microarchitecture. + */ + } + }; + + using AAL_Arch = AAL_RISCV; +} From eb0698fc094c4403cabcfd467fa7860493934f1d Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Tue, 19 Oct 2021 12:19:47 +0100 Subject: [PATCH 129/302] CI: Add RISC-V 64 cross-build & qemu-user tests --- .github/workflows/main.yml | 15 ++++++++++++++- CMakeLists.txt | 19 +++++++++++++++---- ci/Toolchain.cmake | 6 +++++- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8383e8d91..22962462b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -119,18 +119,28 @@ jobs: system-processor: arm triple: arm-linux-gnueabihf rtld: ld-linux-armhf.so.3 + ld-flavour: lld - name: arm64 system-processor: aarch64 triple: aarch64-linux-gnu rtld: ld-linux-aarch64.so.1 + ld-flavour: lld - name: ppc64el system-processor: powerpc64le triple: powerpc64le-linux-gnu rtld: ld64.so.2 + ld-flavour: lld + - name: riscv64 + system-processor: riscv64 + triple: riscv64-linux-gnu + rtld: ld-linux-riscv64-lp64d.so.1 + extra-packages: binutils-riscv64-linux-gnu + ld-flavour: bfd + ld: /usr/bin/riscv64-linux-gnu-ld.bfd # Don't abort runners if a single one fails fail-fast: false runs-on: ubuntu-latest - name: Cross-build for ${{ matrix.arch.triple }} + name: ${{matrix.build-type}} cross-build for ${{ matrix.arch.triple }} steps: - uses: actions/checkout@v2 - name: Install cross-compile toolchain and QEMU @@ -141,6 +151,7 @@ jobs: sudo add-apt-repository "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main" sudo apt update sudo apt install libstdc++-9-dev-${{ matrix.arch.name }}-cross qemu-user ninja-build clang-13 lld-13 + sudo apt install ${{matrix.arch.extra-packages}} # The default PowerPC qemu configuration uses the wrong page size. # Wrap it in a script that fixes this. sudo update-binfmts --disable qemu-ppc64le @@ -161,6 +172,8 @@ jobs: -DSNMALLOC_QEMU_WORKAROUND=ON -DSNMALLOC_STATIC_LIBRARY=OFF -DCMAKE_TOOLCHAIN_FILE=ci/Toolchain.cmake + -DSNMALLOC_LINKER=${{matrix.arch.ld}} + -DSNMALLOC_LINKER_FLAVOUR=${{matrix.arch.ld-flavour}} - name: Build working-directory: ${{github.workspace}}/build run: NINJA_STATUS="%p [%f:%s/%t] %o/s, %es" ninja diff --git a/CMakeLists.txt b/CMakeLists.txt index 534331e7c..b10635e41 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -245,10 +245,21 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) set(${result} ${dirlist} PARENT_SCOPE) endfunction() - set(CMAKE_REQUIRED_LINK_OPTIONS -fuse-ld=lld) - check_cxx_source_compiles("int main() { return 1; }" LLD_WORKS) - if (LLD_WORKS) - message(STATUS "Using LLD to link snmalloc shims") + if(NOT (DEFINED SNMALLOC_LINKER_FLAVOUR) OR ("${SNMALLOC_LINKER_FLAVOUR}" MATCHES "^$")) + # Linker not specified externally; probe to see if we can make lld work + set(CMAKE_REQUIRED_LINK_OPTIONS -fuse-ld=lld) + check_cxx_source_compiles("int main() { return 1; }" LLD_WORKS) + if (LLD_WORKS) + message(STATUS "Using LLD to link snmalloc shims") + endif() + elseif(SNMALLOC_LINKER_FLAVOUR STREQUAL "lld") + # Linker specified externally to be lld; assume it works and that the flags + # have also been set for us + set(LLD_WORKS TRUE) + else() + # Linker specified externally as something other than lld; presume it + # doesn't work and don't add its flags, below + set(LLD_WORKS FALSE) endif() function(add_shim name type) diff --git a/ci/Toolchain.cmake b/ci/Toolchain.cmake index e3e041c83..2b5613c47 100644 --- a/ci/Toolchain.cmake +++ b/ci/Toolchain.cmake @@ -7,7 +7,11 @@ set(CMAKE_C_COMPILER clang-13) set(CMAKE_C_COMPILER_TARGET ${triple}) set(CMAKE_CXX_COMPILER clang++-13) set(CMAKE_CXX_COMPILER_TARGET ${triple}) -set(CROSS_LINKER_FLAGS "-fuse-ld=lld -Wl,--dynamic-linker=/usr/${triple}/lib/$ENV{RTLD_NAME},-rpath,/usr/${triple}/lib") + +set(CROSS_LINKER_FLAGS "-fuse-ld=${SNMALLOC_LINKER_FLAVOUR} -Wl,--dynamic-linker=/usr/${triple}/lib/$ENV{RTLD_NAME},-rpath,/usr/${triple}/lib") +if((DEFINED SNMALLOC_LINKER) AND NOT ("${SNMALLOC_LINKER}" MATCHES "^$")) + string(APPEND CROSS_LINKER_FLAGS " --ld-path=${SNMALLOC_LINKER}") +endif() set(CMAKE_EXE_LINKER_FLAGS ${CROSS_LINKER_FLAGS}) set(CMAKE_SHARED_LINKER_FLAGS ${CROSS_LINKER_FLAGS}) set(CMAKE_MODULE_LINKER_FLAGS ${CROSS_LINKER_FLAGS}) From 6f79ddee31b3f50ba4ebef22b35a46fed4ee717d Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Thu, 12 Aug 2021 11:59:27 +0100 Subject: [PATCH 130/302] FreeBSD RISC platforms have fewer address bits --- src/pal/pal_freebsd.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/pal/pal_freebsd.h b/src/pal/pal_freebsd.h index 13d74aec2..6248f7cb5 100644 --- a/src/pal/pal_freebsd.h +++ b/src/pal/pal_freebsd.h @@ -24,6 +24,17 @@ namespace snmalloc * add new features that they should add any required feature flags. */ static constexpr uint64_t pal_features = PALBSD_Aligned::pal_features; + + /** + * FreeBSD uses atypically small address spaces on its 64 bit RISC machines. + * Problematically, these are so small that if we used the default + * address_bits (48), we'd try to allocate the whole AS (or larger!) for the + * Pagemap itself! + */ + static constexpr size_t address_bits = (Aal::bits == 32) ? + Aal::address_bits : + (Aal::aal_name == RISCV ? 38 : Aal::address_bits); + // TODO, if we ever backport to MIPS, this should yield 39 there. }; } // namespace snmalloc #endif From 6ddd11faeea54fdc4943949acb9a4d227e81d6f6 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Mon, 19 Jul 2021 09:54:20 +0100 Subject: [PATCH 131/302] CheriBSD/CHERI support This adds a CHERI AAL and expands the FreeBSD PAL to cover CHERI. It updates a comment in ds/address.h now that there is an example architecture that differentiates uintptr_t and address_t. --- src/aal/aal.h | 8 +++++++ src/aal/aal_cheri.h | 52 +++++++++++++++++++++++++++++++++++++++++++ src/ds/address.h | 7 ++---- src/pal/pal_freebsd.h | 28 +++++++++++++++++++++++ 4 files changed, 90 insertions(+), 5 deletions(-) create mode 100644 src/aal/aal_cheri.h diff --git a/src/aal/aal.h b/src/aal/aal.h index 2ecac0f82..fcff4dc80 100644 --- a/src/aal/aal.h +++ b/src/aal/aal.h @@ -226,9 +226,17 @@ namespace snmalloc # include "aal_riscv.h" #endif +#if defined(__CHERI_PURE_CAPABILITY__) +# include "aal_cheri.h" +#endif + namespace snmalloc { +#if defined(__CHERI_PURE_CAPABILITY__) + using Aal = AAL_Generic>; +#else using Aal = AAL_Generic>; +#endif template constexpr static bool aal_supports = (AAL::aal_features & F) == F; diff --git a/src/aal/aal_cheri.h b/src/aal/aal_cheri.h new file mode 100644 index 000000000..866e0ad14 --- /dev/null +++ b/src/aal/aal_cheri.h @@ -0,0 +1,52 @@ +#pragma once + +#include "../ds/defines.h" + +#include + +namespace snmalloc +{ + /** + * A mixin AAL that applies CHERI to a `Base` architecture. Gives + * architectural teeth to the capptr_bound primitive. + */ + template + class AAL_CHERI : public Base + { + public: + /** + * CHERI pointers are not integers and come with strict provenance + * requirements. + */ + static constexpr uint64_t aal_features = + (Base::aal_features & ~IntegerPointers) | StrictProvenance; + + /** + * On CHERI-aware compilers, ptraddr_t is an integral type that is wide + * enough to hold any address that may be contained within a memory + * capability. It does not carry provenance: it is not a capability, but + * merely an address. + */ + typedef ptraddr_t address_t; + + template< + typename T, + SNMALLOC_CONCEPT(capptr::ConceptBound) BOut, + SNMALLOC_CONCEPT(capptr::ConceptBound) BIn, + typename U = T> + static SNMALLOC_FAST_PATH CapPtr + capptr_bound(CapPtr a, size_t size) noexcept + { + static_assert( + BIn::spatial > capptr::dimension::Spatial::Alloc, + "Refusing to re-bound Spatial::Alloc CapPtr"); + static_assert( + capptr::is_spatial_refinement(), + "capptr_bound must preserve non-spatial CapPtr dimensions"); + SNMALLOC_ASSERT(__builtin_cheri_tag_get(a.unsafe_ptr())); + + void* pb = __builtin_cheri_bounds_set_exact(a.unsafe_ptr(), size); + return CapPtr(static_cast(pb)); + } + }; +} // namespace snmalloc diff --git a/src/ds/address.h b/src/ds/address.h index 90b0bb456..13c6ed848 100644 --- a/src/ds/address.h +++ b/src/ds/address.h @@ -8,11 +8,8 @@ namespace snmalloc { /** - * The type used for an address. Currently, all addresses are assumed to be - * provenance-carrying values and so it is possible to cast back from the - * result of arithmetic on an address_t. Eventually, this will want to be - * separated into two types, one for raw addresses and one for addresses that - * can be cast back to pointers. + * The type used for an address. On CHERI, this is not a provenance-carrying + * value and so cannot be converted back to a pointer. */ using address_t = Aal::address_t; diff --git a/src/pal/pal_freebsd.h b/src/pal/pal_freebsd.h index 6248f7cb5..e17df290d 100644 --- a/src/pal/pal_freebsd.h +++ b/src/pal/pal_freebsd.h @@ -3,6 +3,13 @@ #if defined(__FreeBSD__) && !defined(_KERNEL) # include "pal_bsd_aligned.h" +// On CHERI platforms, we need to know the value of CHERI_PERM_CHERIABI_VMMAP. +// This pollutes the global namespace a little, sadly, but I think only with +// symbols that begin with CHERI_, which is as close to namespaces as C offers. +# if defined(__CHERI_PURE_CAPABILITY__) +# include +# endif + namespace snmalloc { /** @@ -35,6 +42,27 @@ namespace snmalloc Aal::address_bits : (Aal::aal_name == RISCV ? 38 : Aal::address_bits); // TODO, if we ever backport to MIPS, this should yield 39 there. + +# if defined(__CHERI_PURE_CAPABILITY__) + static_assert( + aal_supports, + "CHERI purecap support requires StrictProvenance AAL"); + + /** + * On CheriBSD, exporting a pointer means stripping it of the authority to + * manage the address space it references by clearing the CHERIABI_VMMAP + * permission bit. + */ + template + static SNMALLOC_FAST_PATH CapPtr> + capptr_to_user_address_control(CapPtr p) + { + return CapPtr>( + __builtin_cheri_perms_and( + p.unsafe_ptr(), + ~static_cast(CHERI_PERM_CHERIABI_VMMAP))); + } +# endif }; } // namespace snmalloc #endif From 342d826310820e6634d641016911707ae7345574 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Mon, 18 Oct 2021 21:51:49 +0100 Subject: [PATCH 132/302] CheriBSD bugs workarounds The correct thing to do, of course, is to fix these upstream, but that requires understanding exactly what's wrong, and that's harder than just not tickling the bugs. --- src/pal/pal_consts.h | 7 ++++++- src/pal/pal_posix.h | 5 ++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/pal/pal_consts.h b/src/pal/pal_consts.h index 0cb9e81d4..8497e3305 100644 --- a/src/pal/pal_consts.h +++ b/src/pal/pal_consts.h @@ -14,8 +14,13 @@ namespace snmalloc * - using_readonly * - not_using * model. + * + * TODO: There is a known bug in CheriBSD that means round-tripping through + * PROT_NONE sheds capability load and store permissions (while restoring data + * read/write, for added excitement). For the moment, just force this down on + * CHERI. */ -#ifdef SNMALLOC_CHECK_CLIENT +#if defined(SNMALLOC_CHECK_CLIENT) && !defined(__CHERI_PURE_CAPABILITY__) static constexpr bool PalEnforceAccess = true; #else static constexpr bool PalEnforceAccess = false; diff --git a/src/pal/pal_posix.h b/src/pal/pal_posix.h index 55d96a5e2..3f3d16366 100644 --- a/src/pal/pal_posix.h +++ b/src/pal/pal_posix.h @@ -146,7 +146,10 @@ namespace snmalloc static void print_stack_trace() { -#ifdef SNMALLOC_BACKTRACE_HEADER + // TODO: the backtrace mechanism does not yet work on CHERI, and causes + // tests which expect to be able to hook abort() to fail. Skip it until + // https://github.com/CTSRD-CHERI/cheribsd/issues/962 is fixed. +#if defined(SNMALLOC_BACKTRACE_HEADER) && !defined(__CHERI_PURE_CAPABILITY__) constexpr int SIZE = 1024; void* buffer[SIZE]; auto nptrs = backtrace(buffer, SIZE); From 4a7cd268f8edb6c41b106ca8a53b9a83bf3cf234 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 20 Oct 2021 11:12:09 +0100 Subject: [PATCH 133/302] Rename slab_allocator to chunk_allocator --- src/backend/fixedglobalconfig.h | 8 ++++---- src/backend/globalconfig.h | 8 ++++---- src/mem/{slaballocator.h => chunkallocator.h} | 4 ++-- src/mem/corealloc.h | 2 +- src/mem/pool.h | 2 +- src/override/malloc-extensions.cc | 4 ++-- src/test/func/domestication/domestication.cc | 6 +++--- 7 files changed, 17 insertions(+), 17 deletions(-) rename src/mem/{slaballocator.h => chunkallocator.h} (97%) diff --git a/src/backend/fixedglobalconfig.h b/src/backend/fixedglobalconfig.h index f03bc9946..30f92656b 100644 --- a/src/backend/fixedglobalconfig.h +++ b/src/backend/fixedglobalconfig.h @@ -1,9 +1,9 @@ #pragma once #include "../backend/backend.h" +#include "../mem/chunkallocator.h" #include "../mem/corealloc.h" #include "../mem/pool.h" -#include "../mem/slaballocator.h" #include "commonconfig.h" namespace snmalloc @@ -19,15 +19,15 @@ namespace snmalloc private: using Backend = BackendAllocator; - inline static ChunkAllocatorState slab_allocator_state; + inline static ChunkAllocatorState chunk_allocator_state; inline static GlobalPoolState alloc_pool; public: static ChunkAllocatorState& - get_slab_allocator_state(typename Backend::LocalState*) + get_chunk_allocator_state(typename Backend::LocalState*) { - return slab_allocator_state; + return chunk_allocator_state; } static GlobalPoolState& pool() diff --git a/src/backend/globalconfig.h b/src/backend/globalconfig.h index 15c24bfa4..04646e3b7 100644 --- a/src/backend/globalconfig.h +++ b/src/backend/globalconfig.h @@ -1,9 +1,9 @@ #pragma once #include "../backend/backend.h" +#include "../mem/chunkallocator.h" #include "../mem/corealloc.h" #include "../mem/pool.h" -#include "../mem/slaballocator.h" #include "commonconfig.h" #ifdef SNMALLOC_TRACING @@ -37,7 +37,7 @@ namespace snmalloc private: using Backend = BackendAllocator; SNMALLOC_REQUIRE_CONSTINIT - inline static ChunkAllocatorState slab_allocator_state; + inline static ChunkAllocatorState chunk_allocator_state; SNMALLOC_REQUIRE_CONSTINIT inline static GlobalPoolState alloc_pool; @@ -50,9 +50,9 @@ namespace snmalloc public: static ChunkAllocatorState& - get_slab_allocator_state(Backend::LocalState* = nullptr) + get_chunk_allocator_state(Backend::LocalState* = nullptr) { - return slab_allocator_state; + return chunk_allocator_state; } static GlobalPoolState& pool() diff --git a/src/mem/slaballocator.h b/src/mem/chunkallocator.h similarity index 97% rename from src/mem/slaballocator.h rename to src/mem/chunkallocator.h index 01b0a704b..03b95b260 100644 --- a/src/mem/slaballocator.h +++ b/src/mem/chunkallocator.h @@ -88,7 +88,7 @@ namespace snmalloc RemoteAllocator* remote) { ChunkAllocatorState& state = - SharedStateHandle::get_slab_allocator_state(&local_state); + SharedStateHandle::get_chunk_allocator_state(&local_state); if (slab_sizeclass >= NUM_SLAB_SIZES) { @@ -140,7 +140,7 @@ namespace snmalloc ChunkRecord* p, size_t slab_sizeclass) { - auto& state = SharedStateHandle::get_slab_allocator_state(&local_state); + auto& state = SharedStateHandle::get_chunk_allocator_state(&local_state); #ifdef SNMALLOC_TRACING std::cout << "Return slab:" << p->meta_common.chunk.unsafe_ptr() << " slab_sizeclass " << slab_sizeclass << " size " diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index b0fc28eed..aace7f33e 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -2,12 +2,12 @@ #include "../ds/defines.h" #include "allocconfig.h" +#include "chunkallocator.h" #include "localcache.h" #include "metaslab.h" #include "pool.h" #include "remotecache.h" #include "sizeclasstable.h" -#include "slaballocator.h" namespace snmalloc { diff --git a/src/mem/pool.h b/src/mem/pool.h index 057c9ba16..0c470c7b3 100644 --- a/src/mem/pool.h +++ b/src/mem/pool.h @@ -3,8 +3,8 @@ #include "../ds/flaglock.h" #include "../ds/mpmcstack.h" #include "../pal/pal_concept.h" +#include "chunkallocator.h" #include "pooled.h" -#include "slaballocator.h" namespace snmalloc { diff --git a/src/override/malloc-extensions.cc b/src/override/malloc-extensions.cc index d672e05be..a76040167 100644 --- a/src/override/malloc-extensions.cc +++ b/src/override/malloc-extensions.cc @@ -6,8 +6,8 @@ using namespace snmalloc; void get_malloc_info_v1(malloc_info_v1* stats) { - auto unused_chunks = Globals::get_slab_allocator_state().unused_memory(); - auto peak = Globals::get_slab_allocator_state().peak_memory_usage(); + auto unused_chunks = Globals::get_chunk_allocator_state().unused_memory(); + auto peak = Globals::get_chunk_allocator_state().peak_memory_usage(); stats->current_memory_usage = peak - unused_chunks; stats->peak_memory_usage = peak; } diff --git a/src/test/func/domestication/domestication.cc b/src/test/func/domestication/domestication.cc index a1607de47..e12c31d80 100644 --- a/src/test/func/domestication/domestication.cc +++ b/src/test/func/domestication/domestication.cc @@ -25,7 +25,7 @@ namespace snmalloc private: using Backend = BackendAllocator; SNMALLOC_REQUIRE_CONSTINIT - inline static ChunkAllocatorState slab_allocator_state; + inline static ChunkAllocatorState chunk_allocator_state; SNMALLOC_REQUIRE_CONSTINIT inline static GlobalPoolState alloc_pool; @@ -47,9 +47,9 @@ namespace snmalloc (); static ChunkAllocatorState& - get_slab_allocator_state(Backend::LocalState* = nullptr) + get_chunk_allocator_state(Backend::LocalState* = nullptr) { - return slab_allocator_state; + return chunk_allocator_state; } static GlobalPoolState& pool() From eb6c1a05c893269c96e8469ab48948de04f45311 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 21 Oct 2021 11:35:25 +0100 Subject: [PATCH 134/302] Make chunk size at least the page size. On systems with larger than 16KiB page size, we have chunks that divide a page. This seems a little strange, and if we want to disable the pages backing a chunk, this is not possible. This change ensures the chunk is always at least a single page. --- src/mem/allocconfig.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mem/allocconfig.h b/src/mem/allocconfig.h index 660bc941e..45937dad1 100644 --- a/src/mem/allocconfig.h +++ b/src/mem/allocconfig.h @@ -76,7 +76,8 @@ namespace snmalloc static constexpr size_t MIN_ALLOC_BITS = bits::ctz_const(MIN_ALLOC_SIZE); // Minimum slab size. - static constexpr size_t MIN_CHUNK_BITS = 14; + static constexpr size_t MIN_CHUNK_BITS = bits::max( + static_cast(14), bits::next_pow2_bits_const(OS_PAGE_SIZE)); static constexpr size_t MIN_CHUNK_SIZE = bits::one_at_bit(MIN_CHUNK_BITS); // Minimum number of objects on a slab From 5215bffa9eacea31072b67c530e389313254ea6c Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 21 Oct 2021 16:05:58 +0100 Subject: [PATCH 135/302] Change malloc test suite for errno Errno is not required to be 0 on return from malloc, so don't bother trying to make it 0. Leads to false test failures where libc calls have not reset it after a failure. --- src/test/func/malloc/malloc.cc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index 0c426a781..d6e5a58fc 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -6,10 +6,12 @@ using namespace snmalloc; +constexpr int SUCCESS = 0; + void check_result(size_t size, size_t align, void* p, int err, bool null) { bool failed = false; - if (errno != err) + if (errno != err && err != SUCCESS) { printf("Expected error: %d but got %d\n", err, errno); failed = true; @@ -88,10 +90,10 @@ void check_result(size_t size, size_t align, void* p, int err, bool null) void test_calloc(size_t nmemb, size_t size, int err, bool null) { printf("calloc(%zu, %zu) combined size %zu\n", nmemb, size, nmemb * size); - errno = 0; + errno = SUCCESS; void* p = our_calloc(nmemb, size); - if ((p != nullptr) && (errno == 0)) + if (p != nullptr) { for (size_t i = 0; i < (size * nmemb); i++) { @@ -112,7 +114,7 @@ void test_realloc(void* p, size_t size, int err, bool null) old_size = our_malloc_usable_size(p); printf("realloc(%p(%zu), %zu)\n", p, old_size, size); - errno = 0; + errno = SUCCESS; auto new_p = our_realloc(p, size); // Realloc failure case, deallocate original block if (new_p == nullptr && size != 0) @@ -131,7 +133,7 @@ void test_posix_memalign(size_t size, size_t align, int err, bool null) void test_memalign(size_t size, size_t align, int err, bool null) { printf("memalign(%zu, %zu)\n", align, size); - errno = 0; + errno = SUCCESS; void* p = our_memalign(align, size); check_result(size, align, p, err, null); } @@ -145,15 +147,13 @@ int main(int argc, char** argv) our_free(nullptr); - constexpr int SUCCESS = 0; - for (sizeclass_t sc = 0; sc < (MAX_SIZECLASS_BITS + 4); sc++) { const size_t size = bits::one_at_bit(sc); printf("malloc: %zu\n", size); - errno = 0; + errno = SUCCESS; check_result(size, 1, our_malloc(size), SUCCESS, false); - errno = 0; + errno = SUCCESS; check_result(size + 1, 1, our_malloc(size + 1), SUCCESS, false); } From 4987a19fe9b1a6b0a3e3e5cb892ae8e30310e26e Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 22 Oct 2021 16:12:44 +0100 Subject: [PATCH 136/302] Rename --- src/ds/{seqqueue.h => seqset.h} | 0 src/mem/metaslab.h | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename src/ds/{seqqueue.h => seqset.h} (100%) diff --git a/src/ds/seqqueue.h b/src/ds/seqset.h similarity index 100% rename from src/ds/seqqueue.h rename to src/ds/seqset.h diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index c68e95b79..733759be7 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -2,7 +2,7 @@ #include "../ds/dllist.h" #include "../ds/helpers.h" -#include "../ds/seqqueue.h" +#include "../ds/seqset.h" #include "../mem/remoteallocator.h" #include "freelist.h" #include "sizeclasstable.h" From cec015f29609b36bb087f82eb63df040887e9f07 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 22 Oct 2021 16:12:59 +0100 Subject: [PATCH 137/302] Add FIFO behaviour for unchecked code. Consuming available slabs in LIFO order makes predicting address reuse harder but appears to have performance implications. Condition this on CHECK_CLIENT and instead use FIFO order on !CHECK_CLIENT builds. --- src/ds/seqset.h | 104 ++++++++++++++++++++++++++++++++++---------- src/mem/corealloc.h | 11 ++--- src/mem/metaslab.h | 8 +++- 3 files changed, 94 insertions(+), 29 deletions(-) diff --git a/src/ds/seqset.h b/src/ds/seqset.h index 420e4e780..13f9fafa9 100644 --- a/src/ds/seqset.h +++ b/src/ds/seqset.h @@ -10,34 +10,66 @@ namespace snmalloc { /** - * Simple sequential queue of T. + * Simple sequential set of T. * * Linked using the T::next field. + * + * Can be used in either Fifo or Lifo mode, which is + * specified by template parameter. */ - template - class SeqQueue + template + class SeqSet { + /** + * Field representation for Fifo behaviour. + */ + struct FieldFifo + { + T* head{nullptr}; + }; + + /** + * Field representation for Lifo behaviour. + */ + struct FieldLifo + { + T* head{nullptr}; + T** end{&head}; + }; + static_assert( std::is_same::value, "T->next must be a queue pointer to T"); - T* head{nullptr}; - T** end{&head}; - public: /** - * Empty queue + * Field indirection to actual representation. + * Different numbers of fields are required for the + * two behaviours. */ - constexpr SeqQueue() = default; + std::conditional_t v; /** * Check for empty */ SNMALLOC_FAST_PATH bool is_empty() { - SNMALLOC_ASSERT(end != nullptr); - return &head == end; + if constexpr (Fifo) + { + return v.head == nullptr; + } + else + { + SNMALLOC_ASSERT(v.end != nullptr); + return &(v.head) == v.end; + } } + public: + /** + * Empty queue + */ + constexpr SeqSet() = default; + /** * Remove an element from the queue * @@ -46,11 +78,18 @@ namespace snmalloc SNMALLOC_FAST_PATH T* pop() { SNMALLOC_ASSERT(!this->is_empty()); - auto result = head; - if (&(head->next) == end) - end = &head; + auto result = v.head; + if constexpr (Fifo) + { + v.head = result->next; + } else - head = head->next; + { + if (&(v.head->next) == v.end) + v.end = &(v.head); + else + v.head = v.head->next; + } return result; } @@ -64,13 +103,20 @@ namespace snmalloc template SNMALLOC_FAST_PATH void filter(Fn&& f) { - T** prev = &head; // Check for empty case. - if (prev == end) + if (is_empty()) return; + T** prev = &(v.head); + while (true) { + if constexpr (Fifo) + { + if (*prev == nullptr) + break; + } + T* curr = *prev; // Note must read curr->next before calling `f` as `f` is allowed to // mutate that field. @@ -85,12 +131,16 @@ namespace snmalloc // Keep element prev = &(curr->next); } - - if (&(curr->next) == end) - break; + if constexpr (!Fifo) + { + if (&(curr->next) == v.end) + break; + } + } + if constexpr (!Fifo) + { + v.end = prev; } - - end = prev; } /** @@ -98,8 +148,16 @@ namespace snmalloc */ SNMALLOC_FAST_PATH void insert(T* item) { - *end = item; - end = &(item->next); + if constexpr (Fifo) + { + item->next = v.head; + v.head = item; + } + else + { + *(v.end) = item; + v.end = &(item->next); + } } }; } // namespace snmalloc diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index aace7f33e..212d14f02 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -339,7 +339,8 @@ namespace snmalloc SNMALLOC_SLOW_PATH void dealloc_local_slabs(sizeclass_t sizeclass) { // Return unused slabs of sizeclass_t back to global allocator - alloc_classes[sizeclass].queue.filter([this, sizeclass](Metaslab* meta) { + alloc_classes[sizeclass].available.filter([this, + sizeclass](Metaslab* meta) { auto domesticate = [this](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { auto res = @@ -397,7 +398,7 @@ namespace snmalloc // Wake slab up. meta->set_not_sleeping(sizeclass); - alloc_classes[sizeclass].queue.insert(meta); + alloc_classes[sizeclass].available.insert(meta); alloc_classes[sizeclass].length++; #ifdef SNMALLOC_TRACING @@ -665,7 +666,7 @@ namespace snmalloc size_t rsize = sizeclass_to_size(sizeclass); // Look to see if we can grab a free list. - auto& sl = alloc_classes[sizeclass].queue; + auto& sl = alloc_classes[sizeclass].available; if (likely(alloc_classes[sizeclass].length > 0)) { #ifdef SNMALLOC_CHECK_CLIENT @@ -762,7 +763,7 @@ namespace snmalloc if (still_active) { alloc_classes[sizeclass].length++; - alloc_classes[sizeclass].queue.insert(meta); + alloc_classes[sizeclass].available.insert(meta); } return finish_alloc(p, sizeclass); @@ -861,7 +862,7 @@ namespace snmalloc for (auto& alloc_class : alloc_classes) { - test(alloc_class.queue); + test(alloc_class.available); } // Place the static stub message on the queue. diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 733759be7..357616686 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -292,7 +292,13 @@ namespace snmalloc struct MetaslabCache { - SeqQueue queue; +#ifdef SNMALLOC_CHECK_CLIENT + SeqSet available; +#else + // This is slightly faster in some cases, + // but makes memory reuse more predictable. + SeqSet available; +#endif uint16_t unused = 0; uint16_t length = 0; }; From c1062e629eaa4d270ec1834d19e38bf6587a5c11 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 20 Oct 2021 16:54:45 +0100 Subject: [PATCH 138/302] Add Madvise free to Linux layer --- src/pal/pal_linux.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/pal/pal_linux.h b/src/pal/pal_linux.h index 8104b335c..f597ff110 100644 --- a/src/pal/pal_linux.h +++ b/src/pal/pal_linux.h @@ -65,6 +65,26 @@ namespace snmalloc ::memset(p, 0, size); } } + + static void notify_not_using(void* p, size_t size) noexcept + { + SNMALLOC_ASSERT(is_aligned_block(p, size)); + + if constexpr (PalEnforceAccess) + { +# if !defined(NDEBUG) + // Fill memory so that when we switch the pages back on we don't make + // assumptions on the content. + memset(p, 0x5a, size); +# endif + madvise(p, size, MADV_FREE); + mprotect(p, size, PROT_NONE); + } + else + { + madvise(p, size, MADV_FREE); + } + } }; } // namespace snmalloc #endif From 20a114cb62a231bf80cd811f6d1d35ae06116da6 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 20 Oct 2021 09:31:45 +0100 Subject: [PATCH 139/302] Add a timer to the PAL This adds a way to periodically pool the PAL to see if any timers have expired. Timers can be used to periodically provide callbacks to the rest of snmalloc. --- CMakeLists.txt | 2 +- src/ds/flaglock.h | 2 + src/ds/helpers.h | 1 + src/mem/corealloc.h | 14 +++- src/mem/ticker.h | 97 +++++++++++++++++++++ src/pal/pal_concept.h | 2 +- src/pal/pal_consts.h | 66 +-------------- src/pal/pal_ds.h | 163 ++++++++++++++++++++++++++++++++++++ src/pal/pal_noalloc.h | 3 +- src/pal/pal_plain.h | 3 +- src/pal/pal_posix.h | 17 +++- src/pal/pal_timer_default.h | 32 +++++++ src/pal/pal_windows.h | 13 ++- 13 files changed, 343 insertions(+), 72 deletions(-) create mode 100644 src/mem/ticker.h create mode 100644 src/pal/pal_ds.h create mode 100644 src/pal/pal_timer_default.h diff --git a/CMakeLists.txt b/CMakeLists.txt index b10635e41..a3c138890 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -295,7 +295,7 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) if (NOT ${SNMALLOC_CLEANUP} STREQUAL CXX11_DESTRUCTORS) check_linker_flag(CXX "-nostdlib++" SNMALLOC_LINKER_SUPPORT_NOSTDLIBXX) if (SNMALLOC_LINKER_SUPPORT_NOSTDLIBXX) - target_link_options(${name} PRIVATE -nostdlib++) + target_link_options(${name} PRIVATE -nostdlib++) else() set_target_properties(${name} PROPERTIES LINKER_LANGUAGE C) endif() diff --git a/src/ds/flaglock.h b/src/ds/flaglock.h index 9b2458d0d..69ec3079e 100644 --- a/src/ds/flaglock.h +++ b/src/ds/flaglock.h @@ -2,6 +2,8 @@ #include "bits.h" +#include + namespace snmalloc { class FlagLock diff --git a/src/ds/helpers.h b/src/ds/helpers.h index f8ad92c80..b8bc15b17 100644 --- a/src/ds/helpers.h +++ b/src/ds/helpers.h @@ -4,6 +4,7 @@ #include "flaglock.h" #include +#include #include namespace snmalloc diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 212d14f02..ed97a6b02 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -8,6 +8,7 @@ #include "pool.h" #include "remotecache.h" #include "sizeclasstable.h" +#include "ticker.h" namespace snmalloc { @@ -96,6 +97,11 @@ namespace snmalloc */ LocalCache* attached_cache; + /** + * Ticker to query the clock regularly at a lower cost. + */ + Ticker ticker; + /** * The message queue needs to be accessible from other threads * @@ -405,6 +411,7 @@ namespace snmalloc std::cout << "Slab is woken up" << std::endl; #endif + ticker.check_tick(); return; } @@ -419,6 +426,7 @@ namespace snmalloc { dealloc_local_slabs(sizeclass); } + ticker.check_tick(); } /** @@ -698,7 +706,8 @@ namespace snmalloc sl.insert(meta); } - return finish_alloc(p, sizeclass); + auto r = finish_alloc(p, sizeclass); + return ticker.check_tick(r); } return small_alloc_slow(sizeclass, fast_free_list, rsize); } @@ -766,7 +775,8 @@ namespace snmalloc alloc_classes[sizeclass].available.insert(meta); } - return finish_alloc(p, sizeclass); + auto r = finish_alloc(p, sizeclass); + return ticker.check_tick(r); } /** diff --git a/src/mem/ticker.h b/src/mem/ticker.h new file mode 100644 index 000000000..af397c686 --- /dev/null +++ b/src/mem/ticker.h @@ -0,0 +1,97 @@ +#pragma once + +#include "../ds/defines.h" + +#include + +namespace snmalloc +{ + /** + * This class will attempt to call the PAL every 50ms to check the time. + * If the caller of check_tick, does so more frequently, it will attempt + * to back-off to only query the time, every n calls to check_tick, where + * `n` adapts to the current frequency of calling. + * + * The aim is to reduce the time spent querying the time as this might be + * an expensive operation if time has been virtualised. + */ + template + class Ticker + { + /** + * Calls to check_tick required before the time is next queried + */ + uint64_t count_down = 1; + + /** + * Number of ticks next time we check the time. + * That is, + * counted - count_down + * Is how many ticks, since last_epoch_ms was updated. + */ + uint64_t counted = 1; + + /** + * Last time we queried the clock. + */ + uint64_t last_query_ms = 0; + + /** + * Slow path that actually queries clock and sets up + * how many calls for the next time we hit the slow path. + */ + template + SNMALLOC_SLOW_PATH T check_tick_slow(T p = nullptr) + { + uint64_t now_ms = PAL::time_in_ms(); + + // Set up clock. + if (last_query_ms == 0) + { + last_query_ms = now_ms; + count_down = 1; + counted = 1; + return p; + } + + uint64_t duration_ms = now_ms - last_query_ms; + last_query_ms = now_ms; + + // Check is below clock resolution + if (duration_ms == 0) + { + // Exponential back off + count_down = counted; + counted *= 2; + return p; + } + + constexpr size_t deadline_in_ms = 50; + + // Estimate number of ticks to get to the new deadline, based on the + // current interval + auto new_deadline_in_ticks = + ((1 + counted) * deadline_in_ms) / duration_ms; + + counted = new_deadline_in_ticks; + count_down = new_deadline_in_ticks; + + return p; + } + + public: + template + SNMALLOC_FAST_PATH T check_tick(T p = nullptr) + { + // Check before decrement, so that later calcations can use + // count_down == 0 for check on the next call. + // This is used if the ticks are way below the frequency of + // heart beat. + if (--count_down == 0) + { + return check_tick_slow(p); + } + return p; + } + }; +} // namespace snmalloc diff --git a/src/pal/pal_concept.h b/src/pal/pal_concept.h index aff935469..a476cce9e 100644 --- a/src/pal/pal_concept.h +++ b/src/pal/pal_concept.h @@ -3,7 +3,7 @@ #ifdef __cpp_concepts # include "../ds/concept.h" # include "pal_consts.h" - +# include "pal_ds.h" # include namespace snmalloc diff --git a/src/pal/pal_consts.h b/src/pal/pal_consts.h index 8497e3305..537a7c8d4 100644 --- a/src/pal/pal_consts.h +++ b/src/pal/pal_consts.h @@ -3,6 +3,7 @@ #include "../ds/defines.h" #include +#include namespace snmalloc { @@ -85,70 +86,7 @@ namespace snmalloc /** * Default Tag ID for the Apple class */ - static const uint8_t PALAnonDefaultID = 241; - - /** - * This struct is used to represent callbacks for notification from the - * platform. It contains a next pointer as client is responsible for - * allocation as we cannot assume an allocator at this point. - */ - struct PalNotificationObject - { - std::atomic pal_next = nullptr; - - void (*pal_notify)(PalNotificationObject* self); - - PalNotificationObject(void (*pal_notify)(PalNotificationObject* self)) - : pal_notify(pal_notify) - {} - }; - - /*** - * Wrapper for managing notifications for PAL events - */ - class PalNotifier - { - /** - * List of callbacks to notify - */ - std::atomic callbacks{nullptr}; - - public: - /** - * Register a callback object to be notified - * - * The object should never be deallocated by the client after calling - * this. - */ - void register_notification(PalNotificationObject* callback) - { - callback->pal_next = nullptr; - - auto prev = &callbacks; - auto curr = prev->load(); - do - { - while (curr != nullptr) - { - prev = &(curr->pal_next); - curr = prev->load(); - } - } while (!prev->compare_exchange_weak(curr, callback)); - } - - /** - * Calls the pal_notify of all the registered objects. - */ - void notify_all() - { - PalNotificationObject* curr = callbacks; - while (curr != nullptr) - { - curr->pal_notify(curr); - curr = curr->pal_next; - } - } - }; + static const int PALAnonDefaultID = 241; /** * Query whether the PAL supports a specific feature. diff --git a/src/pal/pal_ds.h b/src/pal/pal_ds.h new file mode 100644 index 000000000..852d83cbd --- /dev/null +++ b/src/pal/pal_ds.h @@ -0,0 +1,163 @@ +#pragma once + +#include "../ds/defines.h" +#include "../ds/helpers.h" + +#include +#include + +namespace snmalloc +{ + template + class PalList + { + /** + * List of callbacks to notify + */ + std::atomic elements{nullptr}; + + static_assert( + std::is_same>::value, + "Required pal_next type."); + + public: + /** + * Add an element to the list + */ + void add(T* element) + { + auto prev = &elements; + auto curr = prev->load(); + do + { + while (curr != nullptr) + { + prev = &(curr->pal_next); + curr = prev->load(); + } + } while (!prev->compare_exchange_weak(curr, element)); + } + + /** + * Applies function to all the elements of the list + */ + void apply_all(function_ref func) + { + T* curr = elements; + while (curr != nullptr) + { + func(curr); + curr = curr->pal_next; + } + } + }; + + /** + * This struct is used to represent callbacks for notification from the + * platform. It contains a next pointer as client is responsible for + * allocation as we cannot assume an allocator at this point. + */ + struct PalNotificationObject + { + std::atomic pal_next = nullptr; + + void (*pal_notify)(PalNotificationObject* self); + + PalNotificationObject(void (*pal_notify)(PalNotificationObject* self)) + : pal_notify(pal_notify) + {} + }; + + /*** + * Wrapper for managing notifications for PAL events + */ + class PalNotifier + { + /** + * List of callbacks to notify + */ + PalList callbacks; + + public: + /** + * Register a callback object to be notified + * + * The object should never be deallocated by the client after calling + * this. + */ + void register_notification(PalNotificationObject* callback) + { + callbacks.add(callback); + } + + /** + * Calls the pal_notify of all the registered objects. + */ + void notify_all() + { + callbacks.apply_all([](auto curr) { curr->pal_notify(curr); }); + } + }; + + class PalTimerObject + { + friend class PalTimer; + template + friend class PalList; + + std::atomic pal_next; + + void (*pal_notify)(PalTimerObject* self); + + uint64_t last_run = 0; + uint64_t repeat; + + public: + PalTimerObject(void (*pal_notify)(PalTimerObject* self), uint64_t repeat) + : pal_notify(pal_notify), repeat(repeat) + {} + }; + + /** + * Simple mechanism for handling timers. + * + * Note: This is really designed for a very small number of timers, + * and this design should be changed if that is no longer the case. + */ + class PalTimer + { + /** + * List of callbacks to notify + */ + PalList timers; + + public: + /** + * Register a callback to be called every repeat milliseconds. + */ + void register_timer(PalTimerObject* timer) + { + timers.add(timer); + } + + void check(uint64_t time_ms) + { + static std::atomic_flag lock = ATOMIC_FLAG_INIT; + + // Depulicate calls into here, and make single threaded. + if (lock.test_and_set()) + { + timers.apply_all([time_ms](PalTimerObject* curr) { + if ( + (curr->last_run == 0) || + ((time_ms - curr->last_run) > curr->repeat)) + { + curr->last_run = time_ms; + curr->pal_notify(curr); + } + }); + lock.clear(); + } + } + }; +} // namespace snmalloc diff --git a/src/pal/pal_noalloc.h b/src/pal/pal_noalloc.h index a98add62c..b6882c344 100644 --- a/src/pal/pal_noalloc.h +++ b/src/pal/pal_noalloc.h @@ -6,6 +6,7 @@ #include "../aal/aal.h" #include "pal_concept.h" #include "pal_consts.h" +#include "pal_timer_default.h" #include @@ -27,7 +28,7 @@ namespace snmalloc * ever use. */ template - struct PALNoAlloc + struct PALNoAlloc : public PalTimerDefaultImpl { /** * Bitmap of PalFeatures flags indicating the optional features that this diff --git a/src/pal/pal_plain.h b/src/pal/pal_plain.h index 9d8c8d9ec..57bfe1879 100644 --- a/src/pal/pal_plain.h +++ b/src/pal/pal_plain.h @@ -1,13 +1,14 @@ #pragma once #include "../ds/bits.h" +#include "pal_timer_default.h" namespace snmalloc { // Can be extended // Will require a reserve method in subclasses. template - class PALPlainMixin : public State + class PALPlainMixin : public State, public PalTimerDefaultImpl { public: // Notify platform that we will not be using these pages diff --git a/src/pal/pal_posix.h b/src/pal/pal_posix.h index 3f3d16366..e166bd277 100644 --- a/src/pal/pal_posix.h +++ b/src/pal/pal_posix.h @@ -4,6 +4,7 @@ # include #endif #include "../ds/address.h" +#include "pal_timer_default.h" #if defined(SNMALLOC_BACKTRACE_HEADER) # include SNMALLOC_BACKTRACE_HEADER #endif @@ -39,7 +40,7 @@ namespace snmalloc * version. */ template - class PALPOSIX + class PALPOSIX : public PalTimerDefaultImpl> { /** * Helper class to access the `default_mmap_flags` field of `OS` if one @@ -348,5 +349,19 @@ namespace snmalloc } error("Entropy requested on platform that does not provide entropy"); } + + static uint64_t internal_time_in_ms() + { + auto hold = KeepErrno(); + + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts) == -1) + { + error("Failed to get time"); + } + + return (static_cast(ts.tv_sec) * 1000) + + (static_cast(ts.tv_nsec) / 1000000); + } }; } // namespace snmalloc diff --git a/src/pal/pal_timer_default.h b/src/pal/pal_timer_default.h new file mode 100644 index 000000000..a38094a1c --- /dev/null +++ b/src/pal/pal_timer_default.h @@ -0,0 +1,32 @@ + +#pragma once + +#include "pal_consts.h" +#include "pal_ds.h" + +#include + +namespace snmalloc +{ + template + class PalTimerDefaultImpl + { + inline static PalTimer timers{}; + + public: + static uint64_t time_in_ms() + { + auto time = PalTime::internal_time_in_ms(); + + // Process timers + timers.check(time); + + return time; + } + + static void register_timer(PalTimerObject* timer) + { + timers.register_timer(timer); + } + }; +} \ No newline at end of file diff --git a/src/pal/pal_windows.h b/src/pal/pal_windows.h index 425810bb6..b7de864a5 100644 --- a/src/pal/pal_windows.h +++ b/src/pal/pal_windows.h @@ -2,6 +2,7 @@ #include "../ds/address.h" #include "../ds/bits.h" +#include "pal_timer_default.h" #ifdef _WIN32 # ifndef _MSC_VER @@ -22,9 +23,11 @@ # endif # endif +# include + namespace snmalloc { - class PALWindows + class PALWindows : public PalTimerDefaultImpl { /** * A flag indicating that we have tried to register for low-memory @@ -206,6 +209,14 @@ namespace snmalloc error("Failed to get entropy."); return result; } + + static uint64_t internal_time_in_ms() + { + return static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count()); + } }; } #endif From 72ccb23d02afcf4ed113bf6f600c27766915d721 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 20 Oct 2021 16:55:08 +0100 Subject: [PATCH 140/302] Add local caching to chunk allocator --- src/mem/allocconfig.h | 2 - src/mem/chunkallocator.h | 166 ++++++++++++++++++++++++++++++++++-- src/mem/corealloc.h | 10 +++ src/mem/localalloc.h | 2 + src/pal/pal_timer_default.h | 2 +- 5 files changed, 174 insertions(+), 8 deletions(-) diff --git a/src/mem/allocconfig.h b/src/mem/allocconfig.h index 45937dad1..00690ef8b 100644 --- a/src/mem/allocconfig.h +++ b/src/mem/allocconfig.h @@ -68,8 +68,6 @@ namespace snmalloc // Used to isolate values on cache lines to prevent false sharing. static constexpr size_t CACHELINE_SIZE = 64; - static constexpr size_t PAGE_ALIGNED_SIZE = OS_PAGE_SIZE << INTERMEDIATE_BITS; - // Minimum allocation size is space for two pointers. static_assert(bits::next_pow2_const(sizeof(void*)) == sizeof(void*)); static constexpr size_t MIN_ALLOC_SIZE = 2 * sizeof(void*); diff --git a/src/mem/chunkallocator.h b/src/mem/chunkallocator.h index 03b95b260..cdfe8c704 100644 --- a/src/mem/chunkallocator.h +++ b/src/mem/chunkallocator.h @@ -3,6 +3,7 @@ #include "../ds/mpmcstack.h" #include "../mem/metaslab.h" #include "../mem/sizeclasstable.h" +#include "../pal/pal_ds.h" #ifdef SNMALLOC_TRACING # include @@ -36,6 +37,35 @@ namespace snmalloc static_assert( sizeof(Metaslab) >= sizeof(ChunkRecord), "We conflate these two types."); + /** + * Number of free stacks per chunk size that each allocator will use. + * For performance ideally a power of 2. We will return to the central + * pool anything that has not be used in the last NUM_EPOCHS - 1, where + * each epoch is separated by DecayMemoryTimerObject::PERIOD. + * I.e. if period is 500ms and num of epochs is 4, then we will return to + * the central pool anything not used for the last 1500-2000ms. + */ + constexpr size_t NUM_EPOCHS = 4; + static_assert(bits::is_pow2(NUM_EPOCHS), "Code assumes power of two."); + + class ChunkAllocatorLocalState + { + friend class ChunkAllocator; + + /** + * Stack of slabs that have been returned for reuse. + */ + ModArray< + NUM_SLAB_SIZES, + ModArray>> + chunk_stack; + + /** + * Used for list of all ChunkAllocatorLocalStates. + */ + std::atomic next{nullptr}; + }; + /** * This is the global state required for the chunk allocator. * It must be provided as a part of the shared state handle @@ -47,7 +77,8 @@ namespace snmalloc /** * Stack of slabs that have been returned for reuse. */ - ModArray> chunk_stack; + ModArray> + decommitted_chunk_stack; /** * All memory issued by this address space manager @@ -56,6 +87,17 @@ namespace snmalloc std::atomic memory_in_stacks{0}; + std::atomic all_local{nullptr}; + + /** + * Which is the current epoch to place dealloced chunks, and the + * first place we look for allocating chunks. + */ + std::atomic epoch{0}; + + // Flag to ensure one-shot registration with the PAL for notifications. + std::atomic_flag register_decay{}; + public: size_t unused_memory() { @@ -78,10 +120,68 @@ namespace snmalloc class ChunkAllocator { + template + class DecayMemoryTimerObject : public PalTimerObject + { + ChunkAllocatorState* state; + + /*** + * Method for callback object to perform lazy decommit. + */ + static void process(PalTimerObject* p) + { + // Unsafe downcast here. Don't want vtable and RTTI. + auto self = reinterpret_cast(p); + ChunkAllocator::handle_decay_tick(self->state); + } + + // Specify that we notify the ChunkAllocator every 500ms. + static constexpr size_t PERIOD = 500; + + public: + DecayMemoryTimerObject(ChunkAllocatorState* state) + : PalTimerObject(&process, PERIOD), state(state) + {} + }; + + static void handle_decay_tick(ChunkAllocatorState* state) + { + auto new_epoch = (state->epoch + 1) % NUM_EPOCHS; + // Flush old index for all threads. + ChunkAllocatorLocalState* curr = state->all_local; + while (curr != nullptr) + { + for (size_t sc = 0; sc < NUM_SLAB_SIZES; sc++) + { + auto& old_stack = curr->chunk_stack[sc][new_epoch]; + ChunkRecord* record = old_stack.pop_all(); + while (record != nullptr) + { + auto next = record->next.load(); + + // Disable pages for this + Pal::notify_not_using( + record->meta_common.chunk.unsafe_ptr(), + slab_sizeclass_to_size(sc)); + + // Add to global state + state->decommitted_chunk_stack[sc].push(record); + + record = next; + } + } + curr = curr->next; + } + + // Advance current index + state->epoch = new_epoch; + } + public: template static std::pair, Metaslab*> alloc_chunk( typename SharedStateHandle::LocalState& local_state, + ChunkAllocatorLocalState& chunk_alloc_local_state, sizeclass_t sizeclass, sizeclass_t slab_sizeclass, // TODO sizeclass_t size_t slab_size, @@ -96,8 +196,24 @@ namespace snmalloc return {nullptr, nullptr}; } - // Pop a slab - auto chunk_record = state.chunk_stack[slab_sizeclass].pop(); + ChunkRecord* chunk_record = nullptr; + // Try local cache of chunks first + for (size_t e = 0; e < NUM_EPOCHS && chunk_record == nullptr; e++) + { + chunk_record = + chunk_alloc_local_state + .chunk_stack[slab_sizeclass][(state.epoch - e) % NUM_EPOCHS] + .pop(); + } + + // Try global cache. + if (chunk_record == nullptr) + { + chunk_record = state.decommitted_chunk_stack[slab_sizeclass].pop(); + if (chunk_record != nullptr) + Pal::notify_using( + chunk_record->meta_common.chunk.unsafe_ptr(), slab_size); + } if (chunk_record != nullptr) { @@ -137,17 +253,21 @@ namespace snmalloc template SNMALLOC_SLOW_PATH static void dealloc( typename SharedStateHandle::LocalState& local_state, + ChunkAllocatorLocalState& chunk_alloc_local_state, ChunkRecord* p, size_t slab_sizeclass) { - auto& state = SharedStateHandle::get_chunk_allocator_state(&local_state); + ChunkAllocatorState& state = + SharedStateHandle::get_chunk_allocator_state(&local_state); #ifdef SNMALLOC_TRACING std::cout << "Return slab:" << p->meta_common.chunk.unsafe_ptr() << " slab_sizeclass " << slab_sizeclass << " size " << slab_sizeclass_to_size(slab_sizeclass) << " memory in stacks " << state.memory_in_stacks << std::endl; #endif - state.chunk_stack[slab_sizeclass].push(p); + + chunk_alloc_local_state.chunk_stack[slab_sizeclass][state.epoch].push(p); + state.memory_in_stacks += slab_sizeclass_to_size(slab_sizeclass); } @@ -175,5 +295,41 @@ namespace snmalloc return new (p.unsafe_ptr()) U(std::forward(args)...); } + + template + static void register_local_state( + typename SharedStateHandle::LocalState& local_state, + ChunkAllocatorLocalState& chunk_alloc_local_state) + { + ChunkAllocatorState& state = + SharedStateHandle::get_chunk_allocator_state(&local_state); + + // Register with the Pal to receive notifications. + if (!state.register_decay.test_and_set()) + { + auto timer = alloc_meta_data< + DecayMemoryTimerObject, + SharedStateHandle>(&local_state, &state); + if (timer != nullptr) + { + SharedStateHandle::Pal::register_timer(timer); + } + else + { + // We failed to register the notification. + // This is not catarophic, but if we can't allocate this + // state something else will fail shortly. + state.register_decay.clear(); + } + } + + // Add to the list of local states. + auto* head = state.all_local.load(); + do + { + chunk_alloc_local_state.next = head; + } while (!state.all_local.compare_exchange_strong( + head, &chunk_alloc_local_state)); + } }; } // namespace snmalloc diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index ed97a6b02..6443566bf 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -58,6 +58,11 @@ namespace snmalloc */ MetaslabCache alloc_classes[NUM_SIZECLASSES]; + /** + * Local cache for the Chunk allocator. + */ + ChunkAllocatorLocalState chunk_local_state; + /** * Local entropy source and current version of keys for * this thread @@ -376,6 +381,7 @@ namespace snmalloc auto chunk_record = clear_slab(meta, sizeclass); ChunkAllocator::dealloc( get_backend_local_state(), + chunk_local_state, chunk_record, sizeclass_to_slab_sizeclass(sizeclass)); @@ -548,6 +554,9 @@ namespace snmalloc message_queue().invariant(); } + ChunkAllocator::register_local_state( + get_backend_local_state(), chunk_local_state); + #ifndef NDEBUG for (sizeclass_t i = 0; i < NUM_SIZECLASSES; i++) { @@ -746,6 +755,7 @@ namespace snmalloc auto [slab, meta] = snmalloc::ChunkAllocator::alloc_chunk( get_backend_local_state(), + chunk_local_state, sizeclass, slab_sizeclass, slab_size, diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 43423af9d..fa9d2c77e 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -182,6 +182,7 @@ namespace snmalloc // Set remote as large allocator remote. auto [chunk, meta] = ChunkAllocator::alloc_chunk( core_alloc->get_backend_local_state(), + core_alloc->chunk_local_state, bits::next_pow2_bits(size), // TODO large_size_to_chunk_sizeclass(size), large_size_to_chunk_size(size), @@ -547,6 +548,7 @@ namespace snmalloc size_t slab_sizeclass) { ChunkAllocator::dealloc( core_alloc->get_backend_local_state(), + core_alloc->chunk_local_state, slab_record, slab_sizeclass); return nullptr; diff --git a/src/pal/pal_timer_default.h b/src/pal/pal_timer_default.h index a38094a1c..c7761effe 100644 --- a/src/pal/pal_timer_default.h +++ b/src/pal/pal_timer_default.h @@ -29,4 +29,4 @@ namespace snmalloc timers.register_timer(timer); } }; -} \ No newline at end of file +} // namespace snmalloc From 3407347925bc44d93ed06162574360c549688ab0 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 28 Oct 2021 14:22:23 +0100 Subject: [PATCH 141/302] Add custom datastructure for local chunk cache This commit adds a datastructure that provides efficient single-thread stack behaviour, while allowing other threads to steal the whole contents. --- src/ds/spmcstack.h | 72 ++++++++++++++++++++++++++++++++++++++++ src/mem/chunkallocator.h | 5 ++- 2 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 src/ds/spmcstack.h diff --git a/src/ds/spmcstack.h b/src/ds/spmcstack.h new file mode 100644 index 000000000..5651a09ba --- /dev/null +++ b/src/ds/spmcstack.h @@ -0,0 +1,72 @@ +#pragma once + +#include "aba.h" +#include "ptrwrap.h" + +namespace snmalloc +{ + /** + * Concurrent Stack + * + * This stack supports the following clients + * (push|pop)* || pop_all* || ... || pop_all* + * + * That is a single thread that can do push and pop, and other threads + * that do pop_all. pop_all if it returns a value, returns all of the + * stack, however, it may return nullptr if it races with either a push + * or a pop. + * + * The primary use case is single-threaded access, where other threads + * can attempt to steal all the values. + */ + template + class SPMCStack + { + private: + alignas(CACHELINE_SIZE) std::atomic stack{}; + + public: + constexpr SPMCStack() = default; + + void push(T* item) + { + static_assert( + std::is_same>::value, + "T->next must be an std::atomic"); + + return push(item, item); + } + + void push(T* first, T* last) + { + T* old_head = stack.exchange(nullptr, std::memory_order_relaxed); + last->next.store(old_head, std::memory_order_relaxed); + // Assume stays null as not allowed to race with pop or other pushes. + SNMALLOC_ASSERT(stack.load() == nullptr); + stack.store(first, std::memory_order_release); + } + + T* pop() + { + if (stack.load(std::memory_order_relaxed) == nullptr) + return nullptr; + T* old_head = stack.exchange(nullptr); + if (unlikely(old_head == nullptr)) + return nullptr; + + auto next = old_head->next.load(std::memory_order_relaxed); + + // Assume stays null as not allowed to race with pop or other pushes. + SNMALLOC_ASSERT(stack.load() == nullptr); + + stack.store(next, std::memory_order_release); + + return old_head; + } + + T* pop_all() + { + return stack.exchange(nullptr); + } + }; +} // namespace snmalloc diff --git a/src/mem/chunkallocator.h b/src/mem/chunkallocator.h index cdfe8c704..0a8e41eb0 100644 --- a/src/mem/chunkallocator.h +++ b/src/mem/chunkallocator.h @@ -1,6 +1,7 @@ #pragma once #include "../ds/mpmcstack.h" +#include "../ds/spmcstack.h" #include "../mem/metaslab.h" #include "../mem/sizeclasstable.h" #include "../pal/pal_ds.h" @@ -55,9 +56,7 @@ namespace snmalloc /** * Stack of slabs that have been returned for reuse. */ - ModArray< - NUM_SLAB_SIZES, - ModArray>> + ModArray>> chunk_stack; /** From fd408637a7a172d260f38c73667323148439a09e Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 28 Oct 2021 14:23:15 +0100 Subject: [PATCH 142/302] Move some structures to cache lines Prevent some bad sharing of cache lines by aligning some concurrently accessed strucutures. --- src/ds/mpmcstack.h | 2 +- src/mem/chunkallocator.h | 12 ++++++------ src/mem/pool.h | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/ds/mpmcstack.h b/src/ds/mpmcstack.h index d7a4f9259..b701c052c 100644 --- a/src/ds/mpmcstack.h +++ b/src/ds/mpmcstack.h @@ -11,7 +11,7 @@ namespace snmalloc using ABAT = ABA; private: - ABAT stack; + alignas(CACHELINE_SIZE) ABAT stack; public: constexpr MPMCStack() = default; diff --git a/src/mem/chunkallocator.h b/src/mem/chunkallocator.h index 0a8e41eb0..286d7d0d5 100644 --- a/src/mem/chunkallocator.h +++ b/src/mem/chunkallocator.h @@ -79,6 +79,12 @@ namespace snmalloc ModArray> decommitted_chunk_stack; + /** + * Which is the current epoch to place dealloced chunks, and the + * first place we look for allocating chunks. + */ + alignas(CACHELINE_SIZE) std::atomic epoch{0}; + /** * All memory issued by this address space manager */ @@ -88,12 +94,6 @@ namespace snmalloc std::atomic all_local{nullptr}; - /** - * Which is the current epoch to place dealloced chunks, and the - * first place we look for allocating chunks. - */ - std::atomic epoch{0}; - // Flag to ensure one-shot registration with the PAL for notifications. std::atomic_flag register_decay{}; diff --git a/src/mem/pool.h b/src/mem/pool.h index 0c470c7b3..c773e89f8 100644 --- a/src/mem/pool.h +++ b/src/mem/pool.h @@ -28,8 +28,8 @@ namespace snmalloc friend class Pool; private: - std::atomic_flag lock = ATOMIC_FLAG_INIT; MPMCStack stack; + std::atomic_flag lock = ATOMIC_FLAG_INIT; T* list{nullptr}; public: From 19de27fdaf8616948bb6ee011803d6cf9e427afb Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 28 Oct 2021 14:26:41 +0100 Subject: [PATCH 143/302] Improve fast path of handle remote deallocs This adds some branch predictor and cache hints to the fast path of processing remote deallocation. It also removes the batching. --- src/mem/allocconfig.h | 10 ---------- src/mem/corealloc.h | 5 ++--- src/mem/remoteallocator.h | 16 ++++++++++++---- src/test/func/domestication/domestication.cc | 4 +--- 4 files changed, 15 insertions(+), 20 deletions(-) diff --git a/src/mem/allocconfig.h b/src/mem/allocconfig.h index 00690ef8b..ce294e863 100644 --- a/src/mem/allocconfig.h +++ b/src/mem/allocconfig.h @@ -24,16 +24,6 @@ namespace snmalloc 1 << 20 #endif ; - - // Handle at most this many object from the remote dealloc queue at a time. - static constexpr size_t REMOTE_BATCH = -#ifdef USE_REMOTE_BATCH - REMOTE_BATCH -#else - 4096 -#endif - ; - enum DecommitStrategy { /** diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 6443566bf..ee2c006a2 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -457,8 +457,7 @@ namespace snmalloc [local_state](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { return capptr_domesticate(local_state, p); }; - size_t i = 0; - auto cb = [this, local_state, &need_post, &i](freelist::HeadPtr msg) + auto cb = [this, local_state, &need_post](freelist::HeadPtr msg) SNMALLOC_FAST_PATH_LAMBDA { #ifdef SNMALLOC_TRACING std::cout << "Handling remote" << std::endl; @@ -469,7 +468,7 @@ namespace snmalloc handle_dealloc_remote(entry, msg.as_void(), need_post); - return (i++ < REMOTE_BATCH); + return true; }; if constexpr (SharedStateHandle::Options.QueueHeadsAreTame) diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index d316d3c32..8056c534a 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -146,12 +146,21 @@ namespace snmalloc Cb cb) { invariant(); + + // Use back to bound, so we don't handle new entries. + auto b = back.load(std::memory_order_relaxed); freelist::HeadPtr curr = domesticate_head(front); - freelist::HeadPtr next = curr->atomic_read_next(key, domesticate_queue); - while (next != nullptr) + while (address_cast(curr) != address_cast(b)) { - if (!cb(curr)) + freelist::HeadPtr next = curr->atomic_read_next(key, domesticate_queue); + // We have observed a non-linearisable effect of the queue. + // Just go back to allocating normally. + if (unlikely(next == nullptr)) + break; + // We want this element next, so start it loading. + Aal::prefetch(next.unsafe_ptr()); + if (unlikely(!cb(curr))) { /* * We've domesticate_queue-d next so that we can read through it, but @@ -168,7 +177,6 @@ namespace snmalloc } curr = next; - next = next->atomic_read_next(key, domesticate_queue); } /* diff --git a/src/test/func/domestication/domestication.cc b/src/test/func/domestication/domestication.cc index e12c31d80..7a5d2c077 100644 --- a/src/test/func/domestication/domestication.cc +++ b/src/test/func/domestication/domestication.cc @@ -154,14 +154,12 @@ int main() * * - RemoteAllocator::dequeue domesticating the stub's next pointer (p) * - * - RemoteAllocator::dequeue domesticating nullptr (p is the last message) - * * - Metaslab::alloc_free_list, domesticating the successor object in the * newly minted freelist::Iter (i.e., the thing that would be allocated * after q). */ static constexpr size_t expected_count = - snmalloc::CustomGlobals::Options.QueueHeadsAreTame ? 3 : 4; + snmalloc::CustomGlobals::Options.QueueHeadsAreTame ? 2 : 3; SNMALLOC_CHECK(snmalloc::CustomGlobals::domesticate_count == expected_count); // Prevent the allocators from going out of scope during the above test From dd5c91eb439af1688a57c90f910b8b1b5665a744 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 28 Oct 2021 14:28:01 +0100 Subject: [PATCH 144/302] Change Remote Cache default size With the new snmalloc2 changes it seems the larger window is leading to more fragmentation and harming performance. Reducing size still provides good batching, improves memory overhead. --- src/mem/allocconfig.h | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/mem/allocconfig.h b/src/mem/allocconfig.h index ce294e863..323c5665b 100644 --- a/src/mem/allocconfig.h +++ b/src/mem/allocconfig.h @@ -16,14 +16,6 @@ namespace snmalloc #endif ; - // Return remote small allocs when the local cache reaches this size. - static constexpr int64_t REMOTE_CACHE = -#ifdef USE_REMOTE_CACHE - USE_REMOTE_CACHE -#else - 1 << 20 -#endif - ; enum DecommitStrategy { /** @@ -91,4 +83,13 @@ namespace snmalloc static_assert( MIN_ALLOC_SIZE >= (sizeof(void*) * 2), "MIN_ALLOC_SIZE must be sufficient for two pointers"); + + // Return remote small allocs when the local cache reaches this size. + static constexpr int64_t REMOTE_CACHE = +#ifdef USE_REMOTE_CACHE + USE_REMOTE_CACHE +#else + 1 << MIN_CHUNK_BITS +#endif + ; } // namespace snmalloc From 7ef1dfdd517aac3e2d9e3d13c0bdce1f1320260c Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Wed, 3 Nov 2021 14:22:57 +0000 Subject: [PATCH 145/302] Fix ARM AAL on Morello The spelling of inline assembler constraints on Morello would be different, but Jessica Clarke points out that we should just be using the builtin when available, so do that instead. Co-authored-by: Jessica Clarke --- src/aal/aal_arm.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/aal/aal_arm.h b/src/aal/aal_arm.h index b238de5c4..39bcd95a4 100644 --- a/src/aal/aal_arm.h +++ b/src/aal/aal_arm.h @@ -48,12 +48,12 @@ namespace snmalloc { #ifdef _MSC_VER __prefetch(ptr); -#else -# ifdef SNMALLOC_VA_BITS_64 +#elif __has_builtin(__builtin_prefetch) && !defined(SNMALLOC_NO_AAL_BUILTINS) + __builtin_prefetch(ptr); +#elif defined(SNMALLOC_VA_BITS_64) __asm__ volatile("prfm pldl1keep, [%0]" : "=r"(ptr)); -# else +#else __asm__ volatile("pld\t[%0]" : "=r"(ptr)); -# endif #endif } }; From d01e2cdf60bfc320642195756eb6e4533d1cbc09 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 3 Nov 2021 16:23:39 +0000 Subject: [PATCH 146/302] Fix tests. Release tests require CTest to actually be passed the -C option. --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 22962462b..dd5153dfd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -99,7 +99,7 @@ jobs: - name: Test if: ${{ matrix.build-only != 'yes' }} working-directory: ${{github.workspace}}/build - run: ctest --output-on-failure -j 4 + run: ctest --output-on-failure -j 4 -C ${{ matrix.build-type }} - name: Selfhost if: ${{ matrix.self-host }} working-directory: ${{github.workspace}}/build From cf89339762fb79ab82ba626f543711b63bde84fb Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 3 Nov 2021 16:24:28 +0000 Subject: [PATCH 147/302] Move rsize calculation to a slow path. --- src/mem/corealloc.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index ee2c006a2..1d714f5fe 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -679,8 +679,6 @@ namespace snmalloc SNMALLOC_SLOW_PATH capptr::Alloc small_alloc(sizeclass_t sizeclass, freelist::Iter<>& fast_free_list) { - size_t rsize = sizeclass_to_size(sizeclass); - // Look to see if we can grab a free list. auto& sl = alloc_classes[sizeclass].available; if (likely(alloc_classes[sizeclass].length > 0)) @@ -691,7 +689,7 @@ namespace snmalloc unlikely(alloc_classes[sizeclass].length == 1) && (entropy.next_bit() == 0)) { - return small_alloc_slow(sizeclass, fast_free_list, rsize); + return small_alloc_slow(sizeclass, fast_free_list); } #endif @@ -717,7 +715,7 @@ namespace snmalloc auto r = finish_alloc(p, sizeclass); return ticker.check_tick(r); } - return small_alloc_slow(sizeclass, fast_free_list, rsize); + return small_alloc_slow(sizeclass, fast_free_list); } /** @@ -739,9 +737,11 @@ namespace snmalloc } template - SNMALLOC_SLOW_PATH capptr::Alloc small_alloc_slow( - sizeclass_t sizeclass, freelist::Iter<>& fast_free_list, size_t rsize) + SNMALLOC_SLOW_PATH capptr::Alloc + small_alloc_slow(sizeclass_t sizeclass, freelist::Iter<>& fast_free_list) { + size_t rsize = sizeclass_to_size(sizeclass); + // No existing free list get a new slab. size_t slab_size = sizeclass_to_slab_size(sizeclass); size_t slab_sizeclass = sizeclass_to_slab_sizeclass(sizeclass); From 02f36a4b31eb9c2aef897ed659229492f30afcac Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 3 Nov 2021 16:24:50 +0000 Subject: [PATCH 148/302] Make test fail with more information. --- src/test/func/release-rounding/rounding.cc | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/test/func/release-rounding/rounding.cc b/src/test/func/release-rounding/rounding.cc index 0d9d15988..305528f75 100644 --- a/src/test/func/release-rounding/rounding.cc +++ b/src/test/func/release-rounding/rounding.cc @@ -15,6 +15,8 @@ int main(int argc, char** argv) UNUSED(argc); UNUSED(argv); + bool failed = false; + for (size_t size_class = 0; size_class < NUM_SIZECLASSES; size_class++) { size_t rsize = sizeclass_to_size((uint8_t)size_class); @@ -28,18 +30,22 @@ int main(int argc, char** argv) if (rounded != opt_rounded) { std::cout << "rsize " << rsize << " offset " << offset << " opt " - << opt_rounded << std::endl; - abort(); + << opt_rounded << " correct " << rounded << std::endl + << std::flush; + failed = true; } bool opt_mod_0 = is_multiple_of_sizeclass(size_class, offset); if (opt_mod_0 != mod_0) { std::cout << "rsize " << rsize << " offset " << offset << " opt_mod " - << opt_mod_0 << std::endl; - abort(); + << opt_mod_0 << " correct " << mod_0 << std::endl + << std::flush; + failed = true; } } } + if (failed) + abort(); return 0; } From 3d403aef7f9cabefb8c1958763023e6fc98b85de Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 10 Nov 2021 16:35:44 +0000 Subject: [PATCH 149/302] Refactor use of sizeclasses (#415) The primary aim for this refactor is to use a representation for sizeclasses that uniformly covers both large and small. This allows certain operations such as alloc_size and external_pointer to be uniformly implemented. The additional types make clear which kind of sizeclass is in use. This also tidies up the code for sizeclass based divisible by and modulus. It fixes a bug in rust_realloc that didn't correctly determine a realloc was required for large classes. --- src/backend/address_space_core.h | 2 +- src/mem/allocconfig.h | 6 +- src/mem/allocstats.h | 30 +- src/mem/chunkallocator.h | 2 +- src/mem/corealloc.h | 32 +- src/mem/localalloc.h | 134 ++++--- src/mem/localcache.h | 12 +- src/mem/metaslab.h | 28 +- src/mem/remoteallocator.h | 13 +- src/mem/remotecache.h | 2 +- src/mem/sizeclasstable.h | 361 +++++++++++++----- src/override/memcpy.cc | 3 +- src/override/rust.cc | 3 +- .../func/first_operation/first_operation.cc | 2 +- src/test/func/malloc/malloc.cc | 19 +- src/test/func/memory/memory.cc | 52 ++- src/test/func/release-rounding/rounding.cc | 22 +- src/test/func/sizeclass/sizeclass.cc | 11 +- src/test/func/teardown/teardown.cc | 2 +- 19 files changed, 464 insertions(+), 272 deletions(-) diff --git a/src/backend/address_space_core.h b/src/backend/address_space_core.h index f8b725162..f3485421d 100644 --- a/src/backend/address_space_core.h +++ b/src/backend/address_space_core.h @@ -97,7 +97,7 @@ namespace snmalloc // // dealloc() can reject attempts to free such MetaEntry-s due to the // zero sizeclass. - MetaEntry t(reinterpret_cast(next.unsafe_ptr()), nullptr, 0); + MetaEntry t(reinterpret_cast(next.unsafe_ptr()), nullptr); Pagemap::set_metaentry(local_state, address_cast(base), 1, t); return; } diff --git a/src/mem/allocconfig.h b/src/mem/allocconfig.h index 323c5665b..a83d2a8d1 100644 --- a/src/mem/allocconfig.h +++ b/src/mem/allocconfig.h @@ -68,9 +68,9 @@ namespace snmalloc #endif // Maximum size of an object that uses sizeclasses. - static constexpr size_t MAX_SIZECLASS_BITS = 16; - static constexpr size_t MAX_SIZECLASS_SIZE = - bits::one_at_bit(MAX_SIZECLASS_BITS); + static constexpr size_t MAX_SMALL_SIZECLASS_BITS = 16; + static constexpr size_t MAX_SMALL_SIZECLASS_SIZE = + bits::one_at_bit(MAX_SMALL_SIZECLASS_BITS); // Number of slots for remote deallocation. static constexpr size_t REMOTE_SLOT_BITS = 8; diff --git a/src/mem/allocstats.h b/src/mem/allocstats.h index 937a3a945..18896c6e2 100644 --- a/src/mem/allocstats.h +++ b/src/mem/allocstats.h @@ -179,7 +179,7 @@ namespace snmalloc #endif } - void sizeclass_alloc(sizeclass_t sc) + void sizeclass_alloc(smallsizeclass_t sc) { UNUSED(sc); @@ -189,7 +189,7 @@ namespace snmalloc #endif } - void sizeclass_dealloc(sizeclass_t sc) + void sizeclass_dealloc(smallsizeclass_t sc) { UNUSED(sc); @@ -209,7 +209,7 @@ namespace snmalloc #endif } - void sizeclass_alloc_slab(sizeclass_t sc) + void sizeclass_alloc_slab(smallsizeclass_t sc) { UNUSED(sc); @@ -219,7 +219,7 @@ namespace snmalloc #endif } - void sizeclass_dealloc_slab(sizeclass_t sc) + void sizeclass_dealloc_slab(smallsizeclass_t sc) { UNUSED(sc); @@ -266,7 +266,7 @@ namespace snmalloc #endif } - void remote_free(sizeclass_t sc) + void remote_free(smallsizeclass_t sc) { UNUSED(sc); @@ -282,7 +282,7 @@ namespace snmalloc #endif } - void remote_receive(sizeclass_t sc) + void remote_receive(smallsizeclass_t sc) { UNUSED(sc); @@ -374,7 +374,7 @@ namespace snmalloc << "Count" << csv.endl; } - for (sizeclass_t i = 0; i < N; i++) + for (smallsizeclass_t i = 0; i < N; i++) { if (sizeclass[i].count.is_unused()) continue; @@ -387,15 +387,15 @@ namespace snmalloc sizeclass[i].print(csv, sizeclass_to_size(i)); } - for (uint8_t i = 0; i < LARGE_N; i++) - { - if ((large_push_count[i] == 0) && (large_pop_count[i] == 0)) - continue; + // for (uint8_t i = 0; i < LARGE_N; i++) + // { + // if ((large_push_count[i] == 0) && (large_pop_count[i] == 0)) + // continue; - csv << "LargeBucketedStats" << dumpid << allocatorid << (i + N) - << large_sizeclass_to_size(i) << large_push_count[i] - << large_pop_count[i] << csv.endl; - } + // csv << "LargeBucketedStats" << dumpid << allocatorid << (i + N) + // << large_sizeclass_to_size(i) << large_push_count[i] + // << large_pop_count[i] << csv.endl; + // } size_t low = 0; size_t high = 0; diff --git a/src/mem/chunkallocator.h b/src/mem/chunkallocator.h index 286d7d0d5..f2d69e931 100644 --- a/src/mem/chunkallocator.h +++ b/src/mem/chunkallocator.h @@ -182,7 +182,7 @@ namespace snmalloc typename SharedStateHandle::LocalState& local_state, ChunkAllocatorLocalState& chunk_alloc_local_state, sizeclass_t sizeclass, - sizeclass_t slab_sizeclass, // TODO sizeclass_t + chunksizeclass_t slab_sizeclass, size_t slab_size, RemoteAllocator* remote) { diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 1d714f5fe..5811079b0 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -56,7 +56,7 @@ namespace snmalloc /** * Per size class list of active slabs for this allocator. */ - MetaslabCache alloc_classes[NUM_SIZECLASSES]; + MetaslabCache alloc_classes[NUM_SMALL_SIZECLASSES]; /** * Local cache for the Chunk allocator. @@ -196,7 +196,9 @@ namespace snmalloc }; // Use attached cache, and fill it if it is empty. return attached_cache->template alloc( - domesticate, size, [&](sizeclass_t sizeclass, freelist::Iter<>* fl) { + domesticate, + size, + [&](smallsizeclass_t sizeclass, freelist::Iter<>* fl) { return small_alloc(sizeclass, *fl); }); } @@ -285,7 +287,7 @@ namespace snmalloc bumpptr = slab_end; } - ChunkRecord* clear_slab(Metaslab* meta, sizeclass_t sizeclass) + ChunkRecord* clear_slab(Metaslab* meta, smallsizeclass_t sizeclass) { auto& key = entropy.get_free_list_key(); freelist::Iter<> fl; @@ -347,7 +349,7 @@ namespace snmalloc } template - SNMALLOC_SLOW_PATH void dealloc_local_slabs(sizeclass_t sizeclass) + SNMALLOC_SLOW_PATH void dealloc_local_slabs(smallsizeclass_t sizeclass) { // Return unused slabs of sizeclass_t back to global allocator alloc_classes[sizeclass].available.filter([this, @@ -400,7 +402,7 @@ namespace snmalloc // TODO: Handle message queue on this path? Metaslab* meta = entry.get_metaslab(); - sizeclass_t sizeclass = entry.get_sizeclass(); + smallsizeclass_t sizeclass = entry.get_sizeclass().as_small(); UNUSED(entropy); if (meta->is_sleeping()) @@ -557,11 +559,11 @@ namespace snmalloc get_backend_local_state(), chunk_local_state); #ifndef NDEBUG - for (sizeclass_t i = 0; i < NUM_SIZECLASSES; i++) + for (smallsizeclass_t i = 0; i < NUM_SMALL_SIZECLASSES; i++) { size_t size = sizeclass_to_size(i); - sizeclass_t sc1 = size_to_sizeclass(size); - sizeclass_t sc2 = size_to_sizeclass_const(size); + smallsizeclass_t sc1 = size_to_sizeclass(size); + smallsizeclass_t sc2 = size_to_sizeclass_const(size); size_t size1 = sizeclass_to_size(sc1); size_t size2 = sizeclass_to_size(sc2); @@ -662,7 +664,8 @@ namespace snmalloc SNMALLOC_ASSERT(!meta->is_unused()); check_client( - Metaslab::is_start_of_object(entry.get_sizeclass(), address_cast(p)), + Metaslab::is_start_of_object( + entry.get_sizeclass().as_small(), address_cast(p)), "Not deallocating start of an object"); auto cp = p.as_static>(); @@ -677,7 +680,7 @@ namespace snmalloc template SNMALLOC_SLOW_PATH capptr::Alloc - small_alloc(sizeclass_t sizeclass, freelist::Iter<>& fast_free_list) + small_alloc(smallsizeclass_t sizeclass, freelist::Iter<>& fast_free_list) { // Look to see if we can grab a free list. auto& sl = alloc_classes[sizeclass].available; @@ -737,8 +740,8 @@ namespace snmalloc } template - SNMALLOC_SLOW_PATH capptr::Alloc - small_alloc_slow(sizeclass_t sizeclass, freelist::Iter<>& fast_free_list) + SNMALLOC_SLOW_PATH capptr::Alloc small_alloc_slow( + smallsizeclass_t sizeclass, freelist::Iter<>& fast_free_list) { size_t rsize = sizeclass_to_size(sizeclass); @@ -755,7 +758,7 @@ namespace snmalloc snmalloc::ChunkAllocator::alloc_chunk( get_backend_local_state(), chunk_local_state, - sizeclass, + sizeclass_t::from_small_class(sizeclass), slab_sizeclass, slab_size, public_state()); @@ -831,7 +834,8 @@ namespace snmalloc [&](capptr::Alloc p) { dealloc_local_object(p); }); // We may now have unused slabs, return to the global allocator. - for (sizeclass_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; sizeclass++) + for (smallsizeclass_t sizeclass = 0; sizeclass < NUM_SMALL_SIZECLASSES; + sizeclass++) { dealloc_local_slabs(sizeclass); } diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index fa9d2c77e..f994a4b3e 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -183,7 +183,7 @@ namespace snmalloc auto [chunk, meta] = ChunkAllocator::alloc_chunk( core_alloc->get_backend_local_state(), core_alloc->chunk_local_state, - bits::next_pow2_bits(size), // TODO + size_to_sizeclass_full(size), large_size_to_chunk_sizeclass(size), large_size_to_chunk_size(size), SharedStateHandle::fake_large_remote); @@ -211,21 +211,20 @@ namespace snmalloc template SNMALLOC_FAST_PATH capptr::Alloc small_alloc(size_t size) { - // SNMALLOC_ASSUME(size <= sizeclass_to_size(NUM_SIZECLASSES)); auto domesticate = [this](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { return capptr_domesticate( core_alloc->backend_state_ptr(), p); }; auto slowpath = [&]( - sizeclass_t sizeclass, + smallsizeclass_t sizeclass, freelist::Iter<>* fl) SNMALLOC_FAST_PATH_LAMBDA { if (likely(core_alloc != nullptr)) { return core_alloc->handle_message_queue( []( CoreAlloc* core_alloc, - sizeclass_t sizeclass, + smallsizeclass_t sizeclass, freelist::Iter<>* fl) { return core_alloc->template small_alloc(sizeclass, *fl); }, @@ -234,7 +233,7 @@ namespace snmalloc fl); } return lazy_init( - [&](CoreAlloc*, sizeclass_t sizeclass) { + [&](CoreAlloc*, smallsizeclass_t sizeclass) { return small_alloc(sizeclass_to_size(sizeclass)); }, sizeclass); @@ -429,7 +428,8 @@ namespace snmalloc #else // Perform the - 1 on size, so that zero wraps around and ends up on // slow path. - if (likely((size - 1) <= (sizeclass_to_size(NUM_SIZECLASSES - 1) - 1))) + if (likely( + (size - 1) <= (sizeclass_to_size(NUM_SMALL_SIZECLASSES - 1) - 1))) { // Small allocations are more likely. Improve // branch prediction by placing this case first. @@ -446,7 +446,6 @@ namespace snmalloc template SNMALLOC_FAST_PATH ALLOCATOR void* alloc() { - // TODO optimise return alloc(size); } @@ -455,7 +454,6 @@ namespace snmalloc #ifdef SNMALLOC_PASS_THROUGH external_alloc::free(p_raw); #else - // TODO: // Care is needed so that dealloc(nullptr) works before init // The backend allocator must ensure that a minimal page map exists // before init, that maps null to a remote_deallocator that will never @@ -508,16 +506,10 @@ namespace snmalloc } // Large deallocation or null. - if (likely(p_tame != nullptr)) + // also checks for managed by page map. + if (likely((p_tame != nullptr) && !entry.get_sizeclass().is_default())) { - size_t entry_sizeclass = entry.get_sizeclass(); - - // Check this is managed by this pagemap. - // - // TODO: Should this be tested even in the !CHECK_CLIENT case? Things - // go fairly pear-shaped, with the ASM's ranges[] getting cross-linked - // with a ChunkAllocator's chunk_stack[0], which seems bad. - check_client(entry_sizeclass != 0, "Not allocated by snmalloc."); + size_t entry_sizeclass = entry.get_sizeclass().as_large(); size_t size = bits::one_at_bit(entry_sizeclass); size_t slab_sizeclass = @@ -558,6 +550,11 @@ namespace snmalloc return; } + // If p_tame is not null, then dealloc has been call on something + // it shouldn't be called on. + // TODO: Should this be tested even in the !CHECK_CLIENT case? + check_client(p_tame == nullptr, "Not allocated by snmalloc."); + # ifdef SNMALLOC_TRACING std::cout << "nullptr deallocation" << std::endl; # endif @@ -611,14 +608,7 @@ namespace snmalloc MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry( core_alloc->backend_state_ptr(), address_cast(p_raw)); - if (likely(entry.get_remote() != SharedStateHandle::fake_large_remote)) - return sizeclass_to_size(entry.get_sizeclass()); - - // Sizeclass zero is for large is actually zero - if (likely(entry.get_sizeclass() != 0)) - return bits::one_at_bit(entry.get_sizeclass()); - - return 0; + return sizeclass_full_to_size(entry.get_sizeclass()); #endif } @@ -630,61 +620,65 @@ namespace snmalloc * the potential pointer space. */ template - void* external_pointer(void* p_raw) + void* external_pointer(void* p) + { + // Note that each case uses `pointer_offset`, so that on + // CHERI it is monotone with respect to the capability. + // Note that the returned pointer could be outside the CHERI + // bounds of `p`, and thus not something that can be followed. + if constexpr (location == Start) + { + size_t index = index_in_object(p); + return pointer_offset(p, 0 - index); + } + else if constexpr (location == End) + { + return pointer_offset(p, remaining_bytes(p) - 1); + } + else + { + return pointer_offset(p, remaining_bytes(p)); + } + } + + /** + * Returns the number of remaining bytes in an object. + * + * auto p = (char*)malloc(size) + * remaining_bytes(p + n) == size - n provided n < size + */ + size_t remaining_bytes(const void* p) { #ifndef SNMALLOC_PASS_THROUGH - // TODO What's the domestication policy here? At the moment we just - // probe the pagemap with the raw address, without checks. There could - // be implicit domestication through the `SharedStateHandle::Pagemap` or - // we could just leave well enough alone. + MetaEntry entry = + SharedStateHandle::Pagemap::template get_metaentry( + core_alloc->backend_state_ptr(), address_cast(p)); - capptr::AllocWild p = capptr_from_client(p_raw); + auto sizeclass = entry.get_sizeclass(); + return snmalloc::remaining_bytes(sizeclass, address_cast(p)); +#else + return pointer_diff(p, reinterpret_cast(UINTPTR_MAX)); +#endif + } + /** + * Returns the byte offset into an object. + * + * auto p = (char*)malloc(size) + * index_in_object(p + n) == n provided n < size + */ + size_t index_in_object(const void* p) + { +#ifndef SNMALLOC_PASS_THROUGH MetaEntry entry = SharedStateHandle::Pagemap::template get_metaentry( core_alloc->backend_state_ptr(), address_cast(p)); - auto sizeclass = entry.get_sizeclass(); - if (likely(entry.get_remote() != SharedStateHandle::fake_large_remote)) - { - auto rsize = sizeclass_to_size(sizeclass); - auto offset = address_cast(p) & (sizeclass_to_slab_size(sizeclass) - 1); - auto start_offset = round_by_sizeclass(sizeclass, offset); - if constexpr (location == Start) - { - UNUSED(rsize); - return capptr_reveal_wild(pointer_offset(p, start_offset - offset)); - } - else if constexpr (location == End) - return capptr_reveal_wild( - pointer_offset(p, rsize + start_offset - offset - 1)); - else - return capptr_reveal_wild( - pointer_offset(p, rsize + start_offset - offset)); - } - // Sizeclass zero of a large allocation is used for not managed by us. - if (likely(sizeclass != 0)) - { - // This is a large allocation, find start by masking. - auto rsize = bits::one_at_bit(sizeclass); - auto start = pointer_align_down(p, rsize); - if constexpr (location == Start) - return capptr_reveal_wild(start); - else if constexpr (location == End) - return capptr_reveal_wild(pointer_offset(start, rsize - 1)); - else - return capptr_reveal_wild(pointer_offset(start, rsize)); - } + auto sizeclass = entry.get_sizeclass(); + return snmalloc::index_in_object(sizeclass, address_cast(p)); #else - UNUSED(p_raw); + return reinterpret_cast(p); #endif - - if constexpr ((location == End) || (location == OnePastEnd)) - // We don't know the End, so return MAX_PTR - return reinterpret_cast(UINTPTR_MAX); - else - // We don't know the Start, so return MIN_PTR - return nullptr; } /** diff --git a/src/mem/localcache.h b/src/mem/localcache.h index b7c2be79a..6e98bb8a4 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -10,10 +10,10 @@ namespace snmalloc { - using Stats = AllocStats; + using Stats = AllocStats; inline static SNMALLOC_FAST_PATH capptr::Alloc - finish_alloc_no_zero(freelist::HeadPtr p, sizeclass_t sizeclass) + finish_alloc_no_zero(freelist::HeadPtr p, smallsizeclass_t sizeclass) { SNMALLOC_ASSERT(Metaslab::is_start_of_object(sizeclass, address_cast(p))); UNUSED(sizeclass); @@ -23,7 +23,7 @@ namespace snmalloc template inline static SNMALLOC_FAST_PATH capptr::Alloc - finish_alloc(freelist::HeadPtr p, sizeclass_t sizeclass) + finish_alloc(freelist::HeadPtr p, smallsizeclass_t sizeclass) { auto r = finish_alloc_no_zero(p, sizeclass); @@ -45,7 +45,7 @@ namespace snmalloc // Free list per small size class. These are used for // allocation on the fast path. This part of the code is inspired by // mimalloc. - freelist::Iter<> small_fast_free_lists[NUM_SIZECLASSES] = {}; + freelist::Iter<> small_fast_free_lists[NUM_SMALL_SIZECLASSES] = {}; // This is the entropy for a particular thread. LocalEntropy entropy; @@ -83,7 +83,7 @@ namespace snmalloc return capptr_domesticate(local_state, p); }; - for (size_t i = 0; i < NUM_SIZECLASSES; i++) + for (size_t i = 0; i < NUM_SMALL_SIZECLASSES; i++) { // TODO could optimise this, to return the whole list in one append // call. @@ -108,7 +108,7 @@ namespace snmalloc alloc(Domesticator domesticate, size_t size, Slowpath slowpath) { auto& key = entropy.get_free_list_key(); - sizeclass_t sizeclass = size_to_sizeclass(size); + smallsizeclass_t sizeclass = size_to_sizeclass(size); stats.alloc_request(size); stats.sizeclass_alloc(sizeclass); auto& fl = small_fast_free_lists[sizeclass]; diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 357616686..8ece19a83 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -73,7 +73,7 @@ namespace snmalloc /** * Initialise Metaslab for a slab. */ - void initialise(sizeclass_t sizeclass) + void initialise(smallsizeclass_t sizeclass) { free_queue.init(); // Set up meta data as if the entire slab has been turned into a free @@ -112,7 +112,7 @@ namespace snmalloc * and will return true, otherwise it will return false. */ SNMALLOC_FAST_PATH bool - set_sleeping(sizeclass_t sizeclass, uint16_t remaining) + set_sleeping(smallsizeclass_t sizeclass, uint16_t remaining) { auto threshold = threshold_for_waking_slab(sizeclass); if (remaining >= threshold) @@ -130,7 +130,7 @@ namespace snmalloc return true; } - SNMALLOC_FAST_PATH void set_not_sleeping(sizeclass_t sizeclass) + SNMALLOC_FAST_PATH void set_not_sleeping(smallsizeclass_t sizeclass) { auto allocated = sizeclass_to_slab_object_count(sizeclass); needed() = allocated - threshold_for_waking_slab(sizeclass); @@ -145,9 +145,9 @@ namespace snmalloc } static SNMALLOC_FAST_PATH bool - is_start_of_object(sizeclass_t sizeclass, address_t p) + is_start_of_object(smallsizeclass_t sizeclass, address_t p) { - return is_multiple_of_sizeclass( + return divisible_by_sizeclass( sizeclass, p - (bits::align_down(p, sizeclass_to_slab_size(sizeclass)))); } @@ -170,7 +170,7 @@ namespace snmalloc Metaslab* meta, freelist::Iter<>& fast_free_list, LocalEntropy& entropy, - sizeclass_t sizeclass) + smallsizeclass_t sizeclass) { auto& key = entropy.get_free_list_key(); @@ -213,8 +213,8 @@ namespace snmalloc * * * log_2(size), at least MIN_CHUNK_BITS, for large allocations. * - * * a value in [0, NUM_SIZECLASSES] for small allocations. These may be - * directly passed to the sizeclass (not slab_sizeclass) functions of + * * a value in [0, NUM_SMALL_SIZECLASSES] for small allocations. These + * may be directly passed to the sizeclass (not slab_sizeclass) functions of * sizeclasstable.h * */ @@ -235,12 +235,15 @@ namespace snmalloc {} SNMALLOC_FAST_PATH - MetaEntry(Metaslab* meta, RemoteAllocator* remote, sizeclass_t sizeclass) + MetaEntry( + Metaslab* meta, + RemoteAllocator* remote, + sizeclass_t sizeclass = sizeclass_t()) : meta(meta) { /* remote might be nullptr; cast to uintptr_t before offsetting */ remote_and_sizeclass = - pointer_offset(reinterpret_cast(remote), sizeclass); + pointer_offset(reinterpret_cast(remote), sizeclass.raw()); } /** @@ -285,8 +288,9 @@ namespace snmalloc { // TODO: perhaps remove static_cast with resolution of // https://github.com/CTSRD-CHERI/llvm-project/issues/588 - return static_cast(remote_and_sizeclass) & - (alignof(RemoteAllocator) - 1); + return sizeclass_t::from_raw( + static_cast(remote_and_sizeclass) & + (alignof(RemoteAllocator) - 1)); } }; diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index 8056c534a..55205937e 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -12,17 +12,8 @@ namespace snmalloc { // Remotes need to be aligned enough that the bottom bits have enough room for // all the size classes, both large and small. - // - // Including large classes in this calculation might seem remarkably strange, - // since large allocations don't have associated Remotes, that is, their - // remote is taken to be 0. However, if there are very few small size - // classes and many large classes, the attempt to align that 0 down by the - // alignment of a Remote might result in a nonzero value. - static constexpr size_t REMOTE_MIN_ALIGN = bits::max( - CACHELINE_SIZE, - bits::max( - bits::next_pow2_const(NUM_SIZECLASSES + 1), - bits::next_pow2_const(NUM_LARGE_CLASSES + 1))); + static constexpr size_t REMOTE_MIN_ALIGN = + bits::max(CACHELINE_SIZE, SIZECLASS_REP_SIZE); /** * Global key for all remote lists. diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index 46cb4288a..2ecb05655 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -55,7 +55,7 @@ namespace snmalloc SNMALLOC_FAST_PATH bool reserve_space(const MetaEntry& entry) { auto size = - static_cast(sizeclass_to_size(entry.get_sizeclass())); + static_cast(sizeclass_full_to_size(entry.get_sizeclass())); bool result = capacity > size; if (result) diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index ebd091fa6..899d5c2f9 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -5,22 +5,28 @@ #include "../ds/helpers.h" #include "allocconfig.h" +/** + * This file contains all the code for transforming transforming sizes to + * sizeclasses and back. It also contains various sizeclass pre-calculated + * tables for operations based on size class such as `modulus` and `divisible + * by`, and constants for the slab based allocator. + * + * TODO: Due to the current structure for constexpr evaluation this file does + * not well delimit internal versus external APIs. Some refactoring should be + * done. + */ + namespace snmalloc { - // Both usings should compile - // We use size_t as it generates better code. - using sizeclass_t = size_t; - // using sizeclass_t = uint8_t; - using sizeclass_compress_t = uint8_t; - - constexpr static uintptr_t SIZECLASS_MASK = 0xFF; + using smallsizeclass_t = size_t; + using chunksizeclass_t = size_t; - constexpr static inline sizeclass_t size_to_sizeclass_const(size_t size) + constexpr static inline smallsizeclass_t size_to_sizeclass_const(size_t size) { // Don't use sizeclasses that are not a multiple of the alignment. // For example, 24 byte allocations can be // problematic for some data due to alignment issues. - auto sc = static_cast( + auto sc = static_cast( bits::to_exp_mant_const(size)); SNMALLOC_ASSERT(sc == static_cast(sc)); @@ -28,20 +34,98 @@ namespace snmalloc return sc; } - static inline size_t large_sizeclass_to_size(uint8_t large_class) + static constexpr size_t NUM_SMALL_SIZECLASSES = + size_to_sizeclass_const(MAX_SMALL_SIZECLASS_SIZE); + + // Large classes range from [MAX_SMALL_SIZECLASS_SIZE, ADDRESS_SPACE). + static constexpr size_t NUM_LARGE_CLASSES = + Pal::address_bits - MAX_SMALL_SIZECLASS_BITS; + + // How many bits are required to represent either a large or a small + // sizeclass. + static constexpr size_t TAG_SIZECLASS_BITS = bits::max( + bits::next_pow2_bits_const(NUM_SMALL_SIZECLASSES + 1), + bits::next_pow2_bits_const(NUM_LARGE_CLASSES + 1)); + + // Number of bits required to represent a tagged sizeclass that can be + // either small or large. + static constexpr size_t SIZECLASS_REP_SIZE = + bits::one_at_bit(TAG_SIZECLASS_BITS + 1); + + /** + * Encapsulates a tagged union of large and small sizeclasses. + * + * Used in various lookup tables to make efficient code that handles + * all objects allocated by snmalloc. + */ + class sizeclass_t { - // TODO. Remove - UNUSED(large_class); - abort(); - // return bits::one_at_bit(large_class + SUPERSLAB_BITS); - } + static constexpr size_t TAG = bits::one_at_bit(TAG_SIZECLASS_BITS); - static constexpr size_t NUM_SIZECLASSES = - size_to_sizeclass_const(MAX_SIZECLASS_SIZE); + size_t value{0}; - // Large classes range from [SUPERSLAB, ADDRESS_SPACE).// TODO - static constexpr size_t NUM_LARGE_CLASSES = - Pal::address_bits - MAX_SIZECLASS_BITS; + constexpr sizeclass_t(size_t value) : value(value) {} + + public: + constexpr sizeclass_t() = default; + + constexpr static sizeclass_t from_small_class(smallsizeclass_t sc) + { + SNMALLOC_ASSERT(sc < TAG); + // Note could use `+` or `|`. Using `+` as will combine nicely with array + // offset. + return {TAG + sc}; + } + + /** + * Takes the number of leading zero bits from the actual large size-1. + * See size_to_sizeclass_full + */ + constexpr static sizeclass_t from_large_class(size_t large_class) + { + SNMALLOC_ASSERT(large_class < TAG); + return {large_class}; + } + + constexpr static sizeclass_t from_raw(size_t raw) + { + return {raw}; + } + + constexpr size_t index() + { + return value & (TAG - 1); + } + + constexpr smallsizeclass_t as_small() + { + SNMALLOC_ASSERT(is_small()); + return value & (TAG - 1); + } + + constexpr chunksizeclass_t as_large() + { + SNMALLOC_ASSERT(!is_small()); + return bits::BITS - (value & (TAG - 1)); + } + + constexpr size_t raw() + { + return value; + } + + constexpr bool is_small() + { + return (value & TAG) != 0; + } + + constexpr bool is_default() + { + return value == 0; + } + }; + + using sizeclass_compress_t = uint8_t; inline SNMALLOC_FAST_PATH static size_t aligned_size(size_t alignment, size_t size) @@ -64,9 +148,9 @@ namespace snmalloc // the slab. size_t slab_mask; // Table of constants for reciprocal division for each sizeclass. - size_t div_mult; - // Table of constants for reciprocal modulus for each sizeclass. size_t mod_mult; + // Table of constants for reciprocal modulus for each sizeclass. + size_t mod_zero_mult; }; /** @@ -81,64 +165,118 @@ namespace snmalloc struct SizeClassTable { - ModArray fast; - ModArray slow; + ModArray fast_; + ModArray slow_; + + [[nodiscard]] constexpr sizeclass_data_fast& fast(sizeclass_t index) + { + return fast_[index.raw()]; + } + + [[nodiscard]] constexpr sizeclass_data_fast fast(sizeclass_t index) const + { + return fast_[index.raw()]; + } + + [[nodiscard]] constexpr sizeclass_data_fast& fast_small(smallsizeclass_t sc) + { + return fast_[sizeclass_t::from_small_class(sc).raw()]; + } - constexpr SizeClassTable() : fast(), slow() + [[nodiscard]] constexpr sizeclass_data_fast + fast_small(smallsizeclass_t sc) const { - for (sizeclass_compress_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; + return fast_[sizeclass_t::from_small_class(sc).raw()]; + } + + [[nodiscard]] constexpr sizeclass_data_slow& slow(sizeclass_t index) + { + return slow_[index.raw()]; + } + + [[nodiscard]] constexpr sizeclass_data_slow slow(sizeclass_t index) const + { + return slow_[index.raw()]; + } + + constexpr SizeClassTable() : fast_(), slow_() + { + for (sizeclass_compress_t sizeclass = 0; + sizeclass < NUM_SMALL_SIZECLASSES; sizeclass++) { + auto& meta = fast_small(sizeclass); + size_t rsize = bits::from_exp_mant(sizeclass); - fast[sizeclass].size = rsize; + meta.size = rsize; size_t slab_bits = bits::max( bits::next_pow2_bits_const(MIN_OBJECT_COUNT * rsize), MIN_CHUNK_BITS); - fast[sizeclass].slab_mask = bits::one_at_bit(slab_bits) - 1; + meta.slab_mask = bits::one_at_bit(slab_bits) - 1; - slow[sizeclass].capacity = - static_cast((fast[sizeclass].slab_mask + 1) / rsize); + auto& meta_slow = slow(sizeclass_t::from_small_class(sizeclass)); + meta_slow.capacity = + static_cast((meta.slab_mask + 1) / rsize); - slow[sizeclass].waking = + meta_slow.waking = #ifdef SNMALLOC_CHECK_CLIENT - static_cast(slow[sizeclass].capacity / 4); + static_cast(meta_slow.capacity / 4); #else - static_cast(bits::min((slow[sizeclass].capacity / 4), 32)); + static_cast(bits::min((meta_slow.capacity / 4), 32)); #endif } - for (sizeclass_compress_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; + for (sizeclass_compress_t sizeclass = 0; + sizeclass < NUM_SMALL_SIZECLASSES; sizeclass++) { - fast[sizeclass].div_mult = // TODO is MAX_SIZECLASS_BITS right? - (bits::one_at_bit(bits::BITS - 24) / - (fast[sizeclass].size / MIN_ALLOC_SIZE)); - if (!bits::is_pow2(fast[sizeclass].size)) - fast[sizeclass].div_mult++; - - fast[sizeclass].mod_mult = - (bits::one_at_bit(bits::BITS - 1) / fast[sizeclass].size); - if (!bits::is_pow2(fast[sizeclass].size)) - fast[sizeclass].mod_mult++; - // Shift multiplier, so that the result of division completely - // overflows, and thus the top SUPERSLAB_BITS will be zero if the mod is - // zero. - fast[sizeclass].mod_mult *= 2; + // Calculate reciprocal modulus constant like reciprocal division, but + // constant is choosen to overflow and only leave the modulus as the + // result. + auto& meta = fast_small(sizeclass); + meta.mod_mult = bits::one_at_bit(bits::BITS - 1) / meta.size; + meta.mod_mult *= 2; + + if (bits::is_pow2(meta.size)) + { + // Set to zero, so masking path is taken if power of 2. + meta.mod_mult = 0; + } + + size_t zero = 0; + meta.mod_zero_mult = (~zero / meta.size) + 1; + } + + // Set up table for large classes. + // Note skipping sizeclass == 0 as this is size == 0, so the tables can be + // all zero. + for (size_t sizeclass = 1; sizeclass < bits::BITS; sizeclass++) + { + auto lsc = sizeclass_t::from_large_class(sizeclass); + fast(lsc).size = bits::one_at_bit(lsc.as_large()); + + // Use slab mask as 0 for power of two sizes. + fast(lsc).slab_mask = 0; } } }; static inline constexpr SizeClassTable sizeclass_metadata = SizeClassTable(); - constexpr static inline size_t sizeclass_to_size(sizeclass_t sizeclass) + constexpr static inline size_t sizeclass_to_size(smallsizeclass_t sizeclass) + { + return sizeclass_metadata.fast_small(sizeclass).size; + } + + static inline size_t sizeclass_full_to_size(sizeclass_t sizeclass) { - return sizeclass_metadata.fast[sizeclass].size; + return sizeclass_metadata.fast(sizeclass).size; } - inline static size_t sizeclass_to_slab_size(sizeclass_t sizeclass) + inline static size_t sizeclass_to_slab_size(smallsizeclass_t sizeclass) { - return sizeclass_metadata.fast[sizeclass].slab_mask + 1; + return sizeclass_metadata.fast_small(sizeclass).slab_mask + 1; } /** @@ -148,19 +286,20 @@ namespace snmalloc * * It also increases entropy, when we have randomisation. */ - inline uint16_t threshold_for_waking_slab(sizeclass_t sizeclass) + inline uint16_t threshold_for_waking_slab(smallsizeclass_t sizeclass) { - return sizeclass_metadata.slow[sizeclass].waking; + return sizeclass_metadata.slow(sizeclass_t::from_small_class(sizeclass)) + .waking; } - inline static size_t sizeclass_to_slab_sizeclass(sizeclass_t sizeclass) + inline static size_t sizeclass_to_slab_sizeclass(smallsizeclass_t sizeclass) { size_t ssize = sizeclass_to_slab_size(sizeclass); return bits::next_pow2_bits(ssize) - MIN_CHUNK_BITS; } - inline static size_t slab_sizeclass_to_size(sizeclass_t sizeclass) + inline static size_t slab_sizeclass_to_size(chunksizeclass_t sizeclass) { return bits::one_at_bit(MIN_CHUNK_BITS + sizeclass); } @@ -170,74 +309,71 @@ namespace snmalloc * which must be shifted into the index space of slab_sizeclass-es. */ inline static size_t - metaentry_chunk_sizeclass_to_slab_sizeclass(sizeclass_t sizeclass) + metaentry_chunk_sizeclass_to_slab_sizeclass(chunksizeclass_t sizeclass) { return sizeclass - MIN_CHUNK_BITS; } inline constexpr static uint16_t - sizeclass_to_slab_object_count(sizeclass_t sizeclass) + sizeclass_to_slab_object_count(smallsizeclass_t sizeclass) { - return sizeclass_metadata.slow[sizeclass].capacity; + return sizeclass_metadata.slow(sizeclass_t::from_small_class(sizeclass)) + .capacity; } - inline static size_t round_by_sizeclass(sizeclass_t sc, size_t offset) + inline static size_t mod_by_sizeclass(smallsizeclass_t sc, size_t offset) { - // Only works up to certain offsets, exhaustively tested upto - // SUPERSLAB_SIZE. - // SNMALLOC_ASSERT(offset <= SUPERSLAB_SIZE); + // Only works up to certain offsets, exhaustively tested by rounding.cc + auto meta = sizeclass_metadata.fast_small(sc); - auto rsize = sizeclass_to_size(sc); + // Powers of two should use straigt mask. + SNMALLOC_ASSERT(meta.mod_mult != 0); if constexpr (sizeof(offset) >= 8) { // Only works for 64 bit multiplication, as the following will overflow in // 32bit. - // The code is using reciprocal division. If SUPERSLABS - // get larger then we should review this code. For 24 bits, there are in - // sufficient bits to do this completely efficiently as 24 * 3 is larger - // than 64 bits. But we can pre-round by MIN_ALLOC_SIZE which gets us an - // extra 4 * 3 bits, and thus achievable in 64bit multiplication. - // static_assert( - // SUPERSLAB_BITS <= 24, "The following code assumes max of 24 bits"); - - // TODO 24 hack - static_assert(bits::BITS >= 24, "About to attempt a negative shift"); - static_assert( - (8 * sizeof(offset)) >= (bits::BITS - 24), - "About to shift further than the type"); - return (((offset >> MIN_ALLOC_BITS) * - sizeclass_metadata.fast[sc].div_mult) >> - (bits::BITS - 24)) * - rsize; + // Could be made nicer with 128bit multiply (umulh): + // https://lemire.me/blog/2019/02/20/more-fun-with-fast-remainders-when-the-divisor-is-a-constant/ + auto bits_l = bits::BITS / 2; + auto bits_h = bits::BITS - bits_l; + return ( + ((((offset + 1) * meta.mod_mult) >> (bits_l)) * meta.size) >> bits_h); } else // Use 32-bit division as considerably faster than 64-bit, and // everything fits into 32bits here. - return static_cast(offset / rsize) * rsize; + return static_cast(offset % meta.size); } - inline static bool is_multiple_of_sizeclass(sizeclass_t sc, size_t offset) + inline static size_t index_in_object(sizeclass_t sc, address_t addr) { - // Only works up to certain offsets, exhaustively tested upto - // SUPERSLAB_SIZE. - // SNMALLOC_ASSERT(offset <= SUPERSLAB_SIZE); + if (sizeclass_metadata.fast(sc).mod_mult == 0) + { + return addr & (sizeclass_metadata.fast(sc).size - 1); + } + + address_t offset = addr & (sizeclass_to_slab_size(sc.as_small()) - 1); + return mod_by_sizeclass(sc.as_small(), offset); + } + + inline static size_t remaining_bytes(sizeclass_t sc, address_t addr) + { + return sizeclass_metadata.fast(sc).size - index_in_object(sc, addr); + } + + inline static bool divisible_by_sizeclass(smallsizeclass_t sc, size_t offset) + { + // Only works up to certain offsets, exhaustively tested by rounding.cc if constexpr (sizeof(offset) >= 8) { // Only works for 64 bit multiplication, as the following will overflow in // 32bit. - // The code is using reciprocal division. If SUPERSLABS - // get larger then we should review this code. The modulus code - // has fewer restrictions than division, as it only requires the - // square of the offset to be representable. - // TODO 24 hack. Redo the maths given the multiple - // slab sizes - static_assert(bits::BITS >= 25); - static constexpr size_t MASK = - ~(bits::one_at_bit(bits::BITS - 1 - 24) - 1); - - return ((offset * sizeclass_metadata.fast[sc].mod_mult) & MASK) == 0; + // This is based on: + // https://lemire.me/blog/2019/02/20/more-fun-with-fast-remainders-when-the-divisor-is-a-constant/ + auto mod_zero_mult = sizeclass_metadata.fast_small(sc).mod_zero_mult; + return (offset * mod_zero_mult) < mod_zero_mult; } else // Use 32-bit division as considerably faster than 64-bit, and @@ -262,10 +398,10 @@ namespace snmalloc return (s - 1) >> MIN_ALLOC_BITS; } - static inline sizeclass_t size_to_sizeclass(size_t size) + static inline smallsizeclass_t size_to_sizeclass(size_t size) { constexpr static size_t sizeclass_lookup_size = - sizeclass_lookup_index(MAX_SIZECLASS_SIZE); + sizeclass_lookup_index(MAX_SMALL_SIZECLASS_SIZE); /** * This struct is used to statically initialise a table for looking up @@ -278,10 +414,11 @@ namespace snmalloc constexpr SizeClassLookup() { size_t curr = 1; - for (sizeclass_compress_t sizeclass = 0; sizeclass < NUM_SIZECLASSES; + for (sizeclass_compress_t sizeclass = 0; + sizeclass < NUM_SMALL_SIZECLASSES; sizeclass++) { - for (; curr <= sizeclass_metadata.fast[sizeclass].size; + for (; curr <= sizeclass_metadata.fast_small(sizeclass).size; curr += 1 << MIN_ALLOC_BITS) { auto i = sizeclass_lookup_index(curr); @@ -307,9 +444,29 @@ namespace snmalloc return 0; } + /** + * A compressed size representation, + * either a small size class with the 7th bit set + * or a large class with the 7th bit not set. + * Large classes are stored as a mask shift. + * size = (~0 >> lc) + 1; + * Thus large size class 0, has size 0. + * And large size class 33, has size 2^31 + */ + static inline sizeclass_t size_to_sizeclass_full(size_t size) + { + if ((size - 1) < sizeclass_to_size(NUM_SMALL_SIZECLASSES - 1)) + { + return sizeclass_t::from_small_class(size_to_sizeclass(size)); + } + // bits::clz is undefined on 0, but we have size == 1 has already been + // handled here. We conflate 0 and sizes larger than we can allocate. + return sizeclass_t::from_large_class(bits::clz(size - 1)); + } + inline SNMALLOC_FAST_PATH static size_t round_size(size_t size) { - if (size > sizeclass_to_size(NUM_SIZECLASSES - 1)) + if (size > sizeclass_to_size(NUM_SMALL_SIZECLASSES - 1)) { return bits::next_pow2(size); } diff --git a/src/override/memcpy.cc b/src/override/memcpy.cc index 82bbf9a0d..c193dc28e 100644 --- a/src/override/memcpy.cc +++ b/src/override/memcpy.cc @@ -136,8 +136,7 @@ namespace auto& alloc = ThreadAlloc::get(); void* p = const_cast(ptr); - if (unlikely( - pointer_diff(ptr, alloc.external_pointer(p)) < len)) + if (unlikely(alloc.remaining_bytes(ptr) < len)) { if constexpr (FailFast) { diff --git a/src/override/rust.cc b/src/override/rust.cc index b54be6bc9..23f0c687f 100644 --- a/src/override/rust.cc +++ b/src/override/rust.cc @@ -32,7 +32,8 @@ rust_realloc(void* ptr, size_t alignment, size_t old_size, size_t new_size) size_t aligned_old_size = aligned_size(alignment, old_size), aligned_new_size = aligned_size(alignment, new_size); if ( - size_to_sizeclass(aligned_old_size) == size_to_sizeclass(aligned_new_size)) + size_to_sizeclass_full(aligned_old_size).raw() == + size_to_sizeclass_full(aligned_new_size).raw()) return ptr; void* p = ThreadAlloc::get().alloc(aligned_new_size); if (p) diff --git a/src/test/func/first_operation/first_operation.cc b/src/test/func/first_operation/first_operation.cc index 0337b2252..aa22635b8 100644 --- a/src/test/func/first_operation/first_operation.cc +++ b/src/test/func/first_operation/first_operation.cc @@ -166,7 +166,7 @@ int main(int, char**) f(5); f(7); printf("\n"); - for (size_t exp = 1; exp < snmalloc::MAX_SIZECLASS_BITS; exp++) + for (size_t exp = 1; exp < snmalloc::MAX_SMALL_SIZECLASS_BITS; exp++) { auto shifted = [exp](size_t v) { return v << exp; }; diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index d6e5a58fc..80667f32e 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -147,7 +147,7 @@ int main(int argc, char** argv) our_free(nullptr); - for (sizeclass_t sc = 0; sc < (MAX_SIZECLASS_BITS + 4); sc++) + for (smallsizeclass_t sc = 0; sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) { const size_t size = bits::one_at_bit(sc); printf("malloc: %zu\n", size); @@ -161,12 +161,13 @@ int main(int argc, char** argv) our_free(nullptr); - for (sizeclass_t sc = 0; sc < NUM_SIZECLASSES; sc++) + for (smallsizeclass_t sc = 0; sc < NUM_SMALL_SIZECLASSES; sc++) { const size_t size = sizeclass_to_size(sc); bool overflow = false; - for (size_t n = 1; bits::umul(size, n, overflow) <= MAX_SIZECLASS_SIZE; + for (size_t n = 1; + bits::umul(size, n, overflow) <= MAX_SMALL_SIZECLASS_SIZE; n *= 5) { if (overflow) @@ -178,13 +179,13 @@ int main(int argc, char** argv) test_calloc(0, size, SUCCESS, false); } - for (sizeclass_t sc = 0; sc < NUM_SIZECLASSES; sc++) + for (smallsizeclass_t sc = 0; sc < NUM_SMALL_SIZECLASSES; sc++) { const size_t size = sizeclass_to_size(sc); test_realloc(our_malloc(size), size, SUCCESS, false); test_realloc(nullptr, size, SUCCESS, false); test_realloc(our_malloc(size), ((size_t)-1) / 2, ENOMEM, true); - for (sizeclass_t sc2 = 0; sc2 < NUM_SIZECLASSES; sc2++) + for (smallsizeclass_t sc2 = 0; sc2 < NUM_SMALL_SIZECLASSES; sc2++) { const size_t size2 = sizeclass_to_size(sc2); test_realloc(our_malloc(size), size2, SUCCESS, false); @@ -192,13 +193,13 @@ int main(int argc, char** argv) } } - for (sizeclass_t sc = 0; sc < (MAX_SIZECLASS_BITS + 4); sc++) + for (smallsizeclass_t sc = 0; sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) { const size_t size = bits::one_at_bit(sc); test_realloc(our_malloc(size), size, SUCCESS, false); test_realloc(nullptr, size, SUCCESS, false); test_realloc(our_malloc(size), ((size_t)-1) / 2, ENOMEM, true); - for (sizeclass_t sc2 = 0; sc2 < (MAX_SIZECLASS_BITS + 4); sc2++) + for (smallsizeclass_t sc2 = 0; sc2 < (MAX_SMALL_SIZECLASS_BITS + 4); sc2++) { const size_t size2 = bits::one_at_bit(sc2); printf("size1: %zu, size2:%zu\n", size, size2); @@ -213,10 +214,10 @@ int main(int argc, char** argv) test_posix_memalign(((size_t)-1) / 2, 0, EINVAL, true); test_posix_memalign(OS_PAGE_SIZE, sizeof(uintptr_t) / 2, EINVAL, true); - for (size_t align = sizeof(uintptr_t); align < MAX_SIZECLASS_SIZE * 8; + for (size_t align = sizeof(uintptr_t); align < MAX_SMALL_SIZECLASS_SIZE * 8; align <<= 1) { - for (sizeclass_t sc = 0; sc < NUM_SIZECLASSES - 6; sc++) + for (smallsizeclass_t sc = 0; sc < NUM_SMALL_SIZECLASSES - 6; sc++) { const size_t size = sizeclass_to_size(sc); test_posix_memalign(size, align, SUCCESS, false); diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index a0daef725..6185c5a93 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -236,11 +236,18 @@ void test_external_pointer() // Malloc does not have an external pointer querying mechanism. auto& alloc = ThreadAlloc::get(); - for (uint8_t sc = 0; sc < NUM_SIZECLASSES; sc++) + for (uint8_t sc = 0; sc < NUM_SMALL_SIZECLASSES; sc++) { size_t size = sizeclass_to_size(sc); void* p1 = alloc.alloc(size); + if (size != alloc.alloc_size(p1)) + { + std::cout << "Requested size: " << size + << " alloc_size: " << alloc.alloc_size(p1) << std::endl; + abort(); + } + for (size_t offset = 0; offset < size; offset += 17) { void* p2 = pointer_offset(p1, offset); @@ -248,8 +255,9 @@ void test_external_pointer() void* p4 = alloc.external_pointer(p2); if (p1 != p3) { - std::cout << "size: " << size << " offset: " << offset << " p1: " << p1 - << " p3: " << p3 << std::endl; + std::cout << "size: " << size << " alloc_size: " << alloc.alloc_size(p1) + << " offset: " << offset << " p1: " << p1 << " p3: " << p3 + << std::endl; } SNMALLOC_CHECK(p1 == p3); if ((size_t)p4 != (size_t)p1 + size - 1) @@ -272,7 +280,11 @@ void check_offset(void* base, void* interior) auto& alloc = ThreadAlloc::get(); void* calced_base = alloc.external_pointer((void*)interior); if (calced_base != (void*)base) + { + std::cout << "Calced base: " << calced_base << " actual base: " << base + << " for interior: " << interior << std::endl; abort(); + } } void check_external_pointer_large(size_t* base) @@ -301,7 +313,7 @@ void test_external_pointer_large() for (size_t i = 0; i < count; i++) { - size_t b = MAX_SIZECLASS_BITS + 3; + size_t b = MAX_SMALL_SIZECLASS_BITS + 3; size_t rand = r.next() & ((1 << b) - 1); size_t size = (1 << 24) + rand; total_size += size; @@ -383,7 +395,7 @@ void test_calloc_large_bug() // Some PALS have special paths for PAGE aligned zeroing of large // allocations. This is a large allocation that is intentionally // not a multiple of page size. - const size_t size = (MAX_SIZECLASS_SIZE << 3) - 7; + const size_t size = (MAX_SMALL_SIZECLASS_SIZE << 3) - 7; void* p1 = alloc.alloc(size); SNMALLOC_CHECK(alloc.alloc_size(alloc.external_pointer(p1)) >= size); @@ -415,7 +427,7 @@ void test_static_sized_allocs() { // For each small, medium, and large class, do each kind dealloc. This is // mostly to ensure that all of these forms compile. - for (size_t sc = 0; sc < NUM_SIZECLASSES; sc++) + for (size_t sc = 0; sc < NUM_SMALL_SIZECLASSES; sc++) { // test_static_sized_alloc(); // test_static_sized_alloc(); @@ -430,6 +442,32 @@ void test_static_sized_allocs() // test_static_sized_alloc(); } +void test_remaining_bytes() +{ + auto& alloc = ThreadAlloc::get(); + for (size_t sc = 0; sc < NUM_SMALL_SIZECLASSES; sc++) + { + auto size = sizeclass_to_size(sc); + char* p = (char*)alloc.alloc(size); + for (size_t offset = 0; offset < size; offset++) + { + auto rem = alloc.remaining_bytes(p + offset); + if (rem != (size - offset)) + { + printf( + "Allocation size: %zu, Offset: %zu, Remaining bytes: %zu, " + "Expected: %zu\n", + size, + offset, + rem, + size - offset); + abort(); + } + } + alloc.dealloc(p); + } +} + int main(int argc, char** argv) { setup(); @@ -455,12 +493,12 @@ int main(int argc, char** argv) UNUSED(argc); UNUSED(argv); #endif - test_alloc_dealloc_64k(); test_random_allocation(); test_calloc(); test_double_alloc(); #ifndef SNMALLOC_PASS_THROUGH // Depends on snmalloc specific features + test_remaining_bytes(); test_static_sized_allocs(); test_calloc_large_bug(); test_external_pointer_dealloc_bug(); diff --git a/src/test/func/release-rounding/rounding.cc b/src/test/func/release-rounding/rounding.cc index 305528f75..72fe59563 100644 --- a/src/test/func/release-rounding/rounding.cc +++ b/src/test/func/release-rounding/rounding.cc @@ -17,35 +17,37 @@ int main(int argc, char** argv) bool failed = false; - for (size_t size_class = 0; size_class < NUM_SIZECLASSES; size_class++) + for (size_t size_class = 0; size_class < NUM_SMALL_SIZECLASSES; size_class++) { size_t rsize = sizeclass_to_size((uint8_t)size_class); size_t max_offset = sizeclass_to_slab_size(size_class); + sizeclass_t sc = sizeclass_t::from_small_class(size_class); for (size_t offset = 0; offset < max_offset; offset++) { - size_t rounded = (offset / rsize) * rsize; + size_t mod = offset % rsize; bool mod_0 = (offset % rsize) == 0; - size_t opt_rounded = round_by_sizeclass(size_class, offset); - if (rounded != opt_rounded) + size_t opt_mod = index_in_object(sc, offset); + if (mod != opt_mod) { std::cout << "rsize " << rsize << " offset " << offset << " opt " - << opt_rounded << " correct " << rounded << std::endl + << opt_mod << " correct " << mod << std::endl << std::flush; failed = true; } - bool opt_mod_0 = is_multiple_of_sizeclass(size_class, offset); + bool opt_mod_0 = divisible_by_sizeclass(size_class, offset); if (opt_mod_0 != mod_0) { - std::cout << "rsize " << rsize << " offset " << offset << " opt_mod " - << opt_mod_0 << " correct " << mod_0 << std::endl + std::cout << "rsize " << rsize << " offset " << offset + << " opt_mod0 " << opt_mod_0 << " correct " << mod_0 + << std::endl << std::flush; failed = true; } } + if (failed) + abort(); } - if (failed) - abort(); return 0; } diff --git a/src/test/func/sizeclass/sizeclass.cc b/src/test/func/sizeclass/sizeclass.cc index 8b2faf345..5f3c4aa86 100644 --- a/src/test/func/sizeclass/sizeclass.cc +++ b/src/test/func/sizeclass/sizeclass.cc @@ -3,7 +3,7 @@ #include NOINLINE -snmalloc::sizeclass_t size_to_sizeclass(size_t size) +snmalloc::smallsizeclass_t size_to_sizeclass(size_t size) { return snmalloc::size_to_sizeclass(size); } @@ -15,7 +15,7 @@ void test_align_size() SNMALLOC_CHECK(snmalloc::aligned_size(128, 160) == 256); for (size_t size = 1; - size < snmalloc::sizeclass_to_size(snmalloc::NUM_SIZECLASSES - 1); + size < snmalloc::sizeclass_to_size(snmalloc::NUM_SMALL_SIZECLASSES - 1); size++) { size_t rsize = snmalloc::round_size(size); @@ -39,7 +39,7 @@ void test_align_size() } for (size_t alignment_bits = 0; - alignment_bits < snmalloc::MAX_SIZECLASS_BITS; + alignment_bits < snmalloc::MAX_SMALL_SIZECLASS_BITS; alignment_bits++) { auto alignment = (size_t)1 << alignment_bits; @@ -78,10 +78,11 @@ int main(int, char**) std::cout << "sizeclass |-> [size_low, size_high] " << std::endl; size_t slab_size = 0; - for (snmalloc::sizeclass_t sz = 0; sz < snmalloc::NUM_SIZECLASSES; sz++) + for (snmalloc::smallsizeclass_t sz = 0; sz < snmalloc::NUM_SMALL_SIZECLASSES; + sz++) { if ( - sz < snmalloc::NUM_SIZECLASSES && + sz < snmalloc::NUM_SMALL_SIZECLASSES && slab_size != snmalloc::sizeclass_to_slab_size(sz)) { slab_size = snmalloc::sizeclass_to_slab_size(sz); diff --git a/src/test/func/teardown/teardown.cc b/src/test/func/teardown/teardown.cc index 7bfdbff20..b3d26457a 100644 --- a/src/test/func/teardown/teardown.cc +++ b/src/test/func/teardown/teardown.cc @@ -188,7 +188,7 @@ int main(int, char**) f(5); f(7); printf("\n"); - for (size_t exp = 1; exp < snmalloc::MAX_SIZECLASS_BITS; exp++) + for (size_t exp = 1; exp < snmalloc::MAX_SMALL_SIZECLASS_BITS; exp++) { auto shifted = [exp](size_t v) { return v << exp; }; From 91919f6014eba888473a86afd0190a38c7459b14 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 17 Nov 2021 09:19:08 +0000 Subject: [PATCH 150/302] Update src/backend/pagemap.h Co-authored-by: Amaury Chamayou --- src/backend/pagemap.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/pagemap.h b/src/backend/pagemap.h index fe1f9e76d..e80d94556 100644 --- a/src/backend/pagemap.h +++ b/src/backend/pagemap.h @@ -169,7 +169,7 @@ namespace snmalloc if (new_body_untyped == nullptr) { - PAL::error("Failed to initialisation snmalloc."); + PAL::error("Failed to initialise snmalloc."); } #ifdef SNMALLOC_CHECK_CLIENT From dc542cdc9d8ca7ce23394bdcc2d592cbbcdb3e4e Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 17 Nov 2021 14:11:46 +0000 Subject: [PATCH 151/302] Various fixes for OE (#418) * Add default for getting chunk allocator state Makes the API same between the two configurations. * Reduce address space usage for Open Enclave * Fix OE Pal concept * Add support for Pal not to provide time. The lazy return of pages to the OS uses a simple time based heuristic. This enables a PAL to not support time, and return the memory to a central pool immediately. * Update src/backend/backend.h Co-authored-by: Amaury Chamayou Co-authored-by: Amaury Chamayou --- src/backend/backend.h | 36 ++++++++--- src/backend/fixedglobalconfig.h | 2 +- src/mem/chunkallocator.h | 106 +++++++++++++++++++++----------- src/mem/ticker.h | 15 +++-- src/pal/pal_apple.h | 2 +- src/pal/pal_consts.h | 11 ++++ src/pal/pal_noalloc.h | 4 +- src/pal/pal_open_enclave.h | 2 + src/pal/pal_posix.h | 2 +- src/pal/pal_windows.h | 3 +- 10 files changed, 127 insertions(+), 56 deletions(-) diff --git a/src/backend/backend.h b/src/backend/backend.h index 1cc9c0bd9..1c43dbfa9 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -6,6 +6,19 @@ #include "commonconfig.h" #include "pagemap.h" +#if defined(SNMALLOC_CHECK_CLIENT) && !defined(OPEN_ENCLAVE) +/** + * Protect meta data blocks by allocating separate from chunks for + * user allocations. This involves leaving gaps in address space. + * This is less efficient, so should only be applied for the checked + * build. + * + * On Open Enclave the address space is limited, so we disable this + * feature. + */ +# define SNMALLOC_META_PROTECTED +#endif + namespace snmalloc { /** @@ -22,14 +35,23 @@ namespace snmalloc // Size of local address space requests. Currently aimed at 2MiB large // pages but should make this configurable (i.e. for OE, so we don't need as // much space). +#ifdef OPEN_ENCLAVE + // Don't prefetch address space on OE, as it is limited. + // This could cause perf issues during warm-up phases. + constexpr static size_t LOCAL_CACHE_BLOCK = 0; +#else constexpr static size_t LOCAL_CACHE_BLOCK = bits::one_at_bit(21); +#endif -#ifdef SNMALLOC_CHECK_CLIENT +#ifdef SNMALLOC_META_PROTECTED // When protecting the meta-data, we use a smaller block for the meta-data // that is randomised inside a larger block. This needs to be at least a // page so that we can use guard pages. constexpr static size_t LOCAL_CACHE_META_BLOCK = bits::max(MIN_CHUNK_SIZE * 2, OS_PAGE_SIZE); + static_assert( + LOCAL_CACHE_META_BLOCK <= LOCAL_CACHE_BLOCK, + "LOCAL_CACHE_META_BLOCK must be smaller than LOCAL_CACHE_BLOCK"); #endif public: @@ -103,7 +125,7 @@ namespace snmalloc static capptr::Chunk reserve( AddressSpaceManager& global, LocalState* local_state, size_t size) { -#ifdef SNMALLOC_CHECK_CLIENT +#ifdef SNMALLOC_META_PROTECTED constexpr auto MAX_CACHED_SIZE = is_meta ? LOCAL_CACHE_META_BLOCK : LOCAL_CACHE_BLOCK; #else @@ -113,7 +135,7 @@ namespace snmalloc capptr::Chunk p; if ((local_state != nullptr) && (size <= MAX_CACHED_SIZE)) { -#ifdef SNMALLOC_CHECK_CLIENT +#ifdef SNMALLOC_META_PROTECTED auto& local = is_meta ? local_state->local_meta_address_space : local_state->local_address_space; #else @@ -133,7 +155,7 @@ namespace snmalloc if (refill == nullptr) return nullptr; -#ifdef SNMALLOC_CHECK_CLIENT +#ifdef SNMALLOC_META_PROTECTED if (is_meta) { refill = sub_range(refill, LOCAL_CACHE_BLOCK, LOCAL_CACHE_META_BLOCK); @@ -149,7 +171,7 @@ namespace snmalloc local_state, size); } -#ifdef SNMALLOC_CHECK_CLIENT +#ifdef SNMALLOC_META_PROTECTED // During start up we need meta-data before we have a local allocator // This code protects that meta-data with randomisation, and guard pages. if (local_state == nullptr && is_meta) @@ -178,7 +200,7 @@ namespace snmalloc return p; } -#ifdef SNMALLOC_CHECK_CLIENT +#ifdef SNMALLOC_META_PROTECTED /** * Returns a sub-range of [return, return+sub_size] that is contained in * the range [base, base+full_size]. The first and last slot are not used @@ -236,7 +258,7 @@ namespace snmalloc AddressSpaceManagerCore local_address_space; -#ifdef SNMALLOC_CHECK_CLIENT +#ifdef SNMALLOC_META_PROTECTED /** * Secondary local address space, so we can apply some randomisation * and guard pages to protect the meta-data. diff --git a/src/backend/fixedglobalconfig.h b/src/backend/fixedglobalconfig.h index 30f92656b..b3290dab6 100644 --- a/src/backend/fixedglobalconfig.h +++ b/src/backend/fixedglobalconfig.h @@ -25,7 +25,7 @@ namespace snmalloc public: static ChunkAllocatorState& - get_chunk_allocator_state(typename Backend::LocalState*) + get_chunk_allocator_state(typename Backend::LocalState* = nullptr) { return chunk_allocator_state; } diff --git a/src/mem/chunkallocator.h b/src/mem/chunkallocator.h index f2d69e931..9f4de928a 100644 --- a/src/mem/chunkallocator.h +++ b/src/mem/chunkallocator.h @@ -131,7 +131,7 @@ namespace snmalloc { // Unsafe downcast here. Don't want vtable and RTTI. auto self = reinterpret_cast(p); - ChunkAllocator::handle_decay_tick(self->state); + ChunkAllocator::handle_decay_tick(self->state); } // Specify that we notify the ChunkAllocator every 500ms. @@ -143,6 +143,7 @@ namespace snmalloc {} }; + template static void handle_decay_tick(ChunkAllocatorState* state) { auto new_epoch = (state->epoch + 1) % NUM_EPOCHS; @@ -186,6 +187,7 @@ namespace snmalloc size_t slab_size, RemoteAllocator* remote) { + using PAL = typename SharedStateHandle::Pal; ChunkAllocatorState& state = SharedStateHandle::get_chunk_allocator_state(&local_state); @@ -196,13 +198,16 @@ namespace snmalloc } ChunkRecord* chunk_record = nullptr; - // Try local cache of chunks first - for (size_t e = 0; e < NUM_EPOCHS && chunk_record == nullptr; e++) + if constexpr (pal_supports) { - chunk_record = - chunk_alloc_local_state - .chunk_stack[slab_sizeclass][(state.epoch - e) % NUM_EPOCHS] - .pop(); + // Try local cache of chunks first + for (size_t e = 0; e < NUM_EPOCHS && chunk_record == nullptr; e++) + { + chunk_record = + chunk_alloc_local_state + .chunk_stack[slab_sizeclass][(state.epoch - e) % NUM_EPOCHS] + .pop(); + } } // Try global cache. @@ -210,8 +215,10 @@ namespace snmalloc { chunk_record = state.decommitted_chunk_stack[slab_sizeclass].pop(); if (chunk_record != nullptr) - Pal::notify_using( + { + PAL::template notify_using( chunk_record->meta_common.chunk.unsafe_ptr(), slab_size); + } } if (chunk_record != nullptr) @@ -258,14 +265,31 @@ namespace snmalloc { ChunkAllocatorState& state = SharedStateHandle::get_chunk_allocator_state(&local_state); + + if constexpr (pal_supports) + { + // If we have a time source use decay based local cache. #ifdef SNMALLOC_TRACING - std::cout << "Return slab:" << p->meta_common.chunk.unsafe_ptr() - << " slab_sizeclass " << slab_sizeclass << " size " - << slab_sizeclass_to_size(slab_sizeclass) - << " memory in stacks " << state.memory_in_stacks << std::endl; + std::cout << "Return slab:" << p->meta_common.chunk.unsafe_ptr() + << " slab_sizeclass " << slab_sizeclass << " size " + << slab_sizeclass_to_size(slab_sizeclass) + << " memory in stacks " << state.memory_in_stacks + << std::endl; #endif - - chunk_alloc_local_state.chunk_stack[slab_sizeclass][state.epoch].push(p); + chunk_alloc_local_state.chunk_stack[slab_sizeclass][state.epoch].push( + p); + } + else + { + // No time source share immediately with global state. + // Disable pages for this chunk. + SharedStateHandle::Pal::notify_not_using( + p->meta_common.chunk.unsafe_ptr(), + slab_sizeclass_to_size(slab_sizeclass)); + + // Add to global state + state.decommitted_chunk_stack[slab_sizeclass].push(p); + } state.memory_in_stacks += slab_sizeclass_to_size(slab_sizeclass); } @@ -300,35 +324,43 @@ namespace snmalloc typename SharedStateHandle::LocalState& local_state, ChunkAllocatorLocalState& chunk_alloc_local_state) { - ChunkAllocatorState& state = - SharedStateHandle::get_chunk_allocator_state(&local_state); - - // Register with the Pal to receive notifications. - if (!state.register_decay.test_and_set()) + if constexpr (pal_supports) { - auto timer = alloc_meta_data< - DecayMemoryTimerObject, - SharedStateHandle>(&local_state, &state); - if (timer != nullptr) + ChunkAllocatorState& state = + SharedStateHandle::get_chunk_allocator_state(&local_state); + + // Register with the Pal to receive notifications. + if (!state.register_decay.test_and_set()) { - SharedStateHandle::Pal::register_timer(timer); + auto timer = alloc_meta_data< + DecayMemoryTimerObject, + SharedStateHandle>(&local_state, &state); + if (timer != nullptr) + { + SharedStateHandle::Pal::register_timer(timer); + } + else + { + // We failed to register the notification. + // This is not catarophic, but if we can't allocate this + // state something else will fail shortly. + state.register_decay.clear(); + } } - else + + // Add to the list of local states. + auto* head = state.all_local.load(); + do { - // We failed to register the notification. - // This is not catarophic, but if we can't allocate this - // state something else will fail shortly. - state.register_decay.clear(); - } + chunk_alloc_local_state.next = head; + } while (!state.all_local.compare_exchange_strong( + head, &chunk_alloc_local_state)); } - - // Add to the list of local states. - auto* head = state.all_local.load(); - do + else { - chunk_alloc_local_state.next = head; - } while (!state.all_local.compare_exchange_strong( - head, &chunk_alloc_local_state)); + UNUSED(local_state); + UNUSED(chunk_alloc_local_state); + } } }; } // namespace snmalloc diff --git a/src/mem/ticker.h b/src/mem/ticker.h index af397c686..4dee55eb6 100644 --- a/src/mem/ticker.h +++ b/src/mem/ticker.h @@ -83,13 +83,16 @@ namespace snmalloc template SNMALLOC_FAST_PATH T check_tick(T p = nullptr) { - // Check before decrement, so that later calcations can use - // count_down == 0 for check on the next call. - // This is used if the ticks are way below the frequency of - // heart beat. - if (--count_down == 0) + if constexpr (pal_supports) { - return check_tick_slow(p); + // Check before decrement, so that later calcations can use + // count_down == 0 for check on the next call. + // This is used if the ticks are way below the frequency of + // heart beat. + if (--count_down == 0) + { + return check_tick_slow(p); + } } return p; } diff --git a/src/pal/pal_apple.h b/src/pal/pal_apple.h index 14fb9f0c0..1115e8c95 100644 --- a/src/pal/pal_apple.h +++ b/src/pal/pal_apple.h @@ -28,7 +28,7 @@ namespace snmalloc * The features exported by this PAL. */ static constexpr uint64_t pal_features = - AlignedAllocation | LazyCommit | Entropy; + AlignedAllocation | LazyCommit | Entropy | Time; /* * `page_size` diff --git a/src/pal/pal_consts.h b/src/pal/pal_consts.h index 537a7c8d4..31df5470b 100644 --- a/src/pal/pal_consts.h +++ b/src/pal/pal_consts.h @@ -41,6 +41,7 @@ namespace snmalloc * whether low memory conditions are still in effect. */ LowMemoryNotification = (1 << 0), + /** * This PAL natively supports allocation with a guaranteed alignment. If * this is not supported, then we will over-allocate and round the @@ -51,22 +52,31 @@ namespace snmalloc * `request()` method that takes only a size. */ AlignedAllocation = (1 << 1), + /** * This PAL natively supports lazy commit of pages. This means have large * allocations and not touching them does not increase memory usage. This is * exposed in the Pal. */ LazyCommit = (1 << 2), + /** * This Pal does not support allocation. All memory used with this Pal * should be pre-allocated. */ NoAllocation = (1 << 3), + /** * This Pal provides a source of Entropy */ Entropy = (1 << 4), + + /** + * This Pal provides a millisecond time source + */ + Time = (1 << 5), }; + /** * Flag indicating whether requested memory should be zeroed. */ @@ -76,6 +86,7 @@ namespace snmalloc * Memory should not be zeroed, contents are undefined. */ NoZero, + /** * Memory must be zeroed. This can be lazily allocated via a copy-on-write * mechanism as long as any load from the memory returns zero. diff --git a/src/pal/pal_noalloc.h b/src/pal/pal_noalloc.h index b6882c344..8228d4437 100644 --- a/src/pal/pal_noalloc.h +++ b/src/pal/pal_noalloc.h @@ -28,7 +28,7 @@ namespace snmalloc * ever use. */ template - struct PALNoAlloc : public PalTimerDefaultImpl + struct PALNoAlloc : public BasePAL { /** * Bitmap of PalFeatures flags indicating the optional features that this @@ -38,7 +38,7 @@ namespace snmalloc static constexpr size_t page_size = BasePAL::page_size; - static constexpr size_t address_bits = Aal::address_bits; + static constexpr size_t address_bits = BasePAL::address_bits; /** * Print a stack trace. diff --git a/src/pal/pal_open_enclave.h b/src/pal/pal_open_enclave.h index d6036eb11..276af8219 100644 --- a/src/pal/pal_open_enclave.h +++ b/src/pal/pal_open_enclave.h @@ -19,6 +19,8 @@ namespace snmalloc UNUSED(str); oe_abort(); } + static constexpr size_t address_bits = Aal::address_bits; + static constexpr size_t page_size = Aal::smallest_page_size; }; using OpenEnclaveBasePAL = PALNoAlloc; diff --git a/src/pal/pal_posix.h b/src/pal/pal_posix.h index e166bd277..c4b85ab1e 100644 --- a/src/pal/pal_posix.h +++ b/src/pal/pal_posix.h @@ -125,7 +125,7 @@ namespace snmalloc * POSIX systems are assumed to support lazy commit. The build system checks * getentropy is available, only then this PAL supports Entropy. */ - static constexpr uint64_t pal_features = LazyCommit + static constexpr uint64_t pal_features = LazyCommit | Time #if defined(SNMALLOC_PLATFORM_HAS_GETENTROPY) | Entropy #endif diff --git a/src/pal/pal_windows.h b/src/pal/pal_windows.h index b7de864a5..1667dce9f 100644 --- a/src/pal/pal_windows.h +++ b/src/pal/pal_windows.h @@ -55,7 +55,8 @@ namespace snmalloc * Bitmap of PalFeatures flags indicating the optional features that this * PAL supports. This PAL supports low-memory notifications. */ - static constexpr uint64_t pal_features = LowMemoryNotification | Entropy + static constexpr uint64_t pal_features = LowMemoryNotification | Entropy | + Time # if defined(PLATFORM_HAS_VIRTUALALLOC2) && !defined(USE_SYSTEMATIC_TESTING) | AlignedAllocation # endif From cd0311b26fb320eeb1440744d50349d426815eb2 Mon Sep 17 00:00:00 2001 From: Schrodinger ZHU Yifan Date: Thu, 18 Nov 2021 00:02:47 +0800 Subject: [PATCH 152/302] Nits for rust release (#419) * adjust gitignore Signed-off-by: SchrodingerZhu * also add prefix for rust objects Signed-off-by: SchrodingerZhu * export statistics api for rust Signed-off-by: SchrodingerZhu * conform clang-format-9 Signed-off-by: SchrodingerZhu --- .gitignore | 12 ++++++++++++ src/override/rust.cc | 20 +++++++++++++++----- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index f25dbf668..8737c737a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,20 @@ +# conventional build dirs release*/ debug*/ build*/ +cmake-build-*/ + +# cmake intermediate files CMakeFiles/ + +# vscode dirs .vscode/ .vs/ + +# jetbrains IDE dirs +.idea/ + +# special endings *~ *.sw? + diff --git a/src/override/rust.cc b/src/override/rust.cc index 23f0c687f..ebce39b15 100644 --- a/src/override/rust.cc +++ b/src/override/rust.cc @@ -9,25 +9,26 @@ using namespace snmalloc; -extern "C" SNMALLOC_EXPORT void* rust_alloc(size_t alignment, size_t size) +extern "C" SNMALLOC_EXPORT void* + SNMALLOC_NAME_MANGLE(rust_alloc)(size_t alignment, size_t size) { return ThreadAlloc::get().alloc(aligned_size(alignment, size)); } extern "C" SNMALLOC_EXPORT void* -rust_alloc_zeroed(size_t alignment, size_t size) + SNMALLOC_NAME_MANGLE(rust_alloc_zeroed)(size_t alignment, size_t size) { return ThreadAlloc::get().alloc(aligned_size(alignment, size)); } extern "C" SNMALLOC_EXPORT void -rust_dealloc(void* ptr, size_t alignment, size_t size) + SNMALLOC_NAME_MANGLE(rust_dealloc)(void* ptr, size_t alignment, size_t size) { ThreadAlloc::get().dealloc(ptr, aligned_size(alignment, size)); } -extern "C" SNMALLOC_EXPORT void* -rust_realloc(void* ptr, size_t alignment, size_t old_size, size_t new_size) +extern "C" SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(rust_realloc)( + void* ptr, size_t alignment, size_t old_size, size_t new_size) { size_t aligned_old_size = aligned_size(alignment, old_size), aligned_new_size = aligned_size(alignment, new_size); @@ -43,3 +44,12 @@ rust_realloc(void* ptr, size_t alignment, size_t old_size, size_t new_size) } return p; } + +extern "C" SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(rust_statistics)( + size_t* current_memory_usage, size_t* peak_memory_usage) +{ + auto unused_chunks = Globals::get_chunk_allocator_state().unused_memory(); + auto peak = Globals::get_chunk_allocator_state().peak_memory_usage(); + *current_memory_usage = peak - unused_chunks; + *peak_memory_usage = peak; +} \ No newline at end of file From faa80037bb0b61c86bf55a86ecfa6f4bd487ca97 Mon Sep 17 00:00:00 2001 From: Schrodinger ZHU Yifan Date: Thu, 18 Nov 2021 00:05:52 +0800 Subject: [PATCH 153/302] put likely/unlikely in scope (#420) * put likely/unlikely in scope Signed-off-by: SchrodingerZhu * make clang-format happy Signed-off-by: SchrodingerZhu --- src/backend/pagemap.h | 2 +- src/ds/defines.h | 10 +++++----- src/ds/helpers.h | 2 +- src/ds/spmcstack.h | 2 +- src/mem/corealloc.h | 15 ++++++++------- src/mem/localalloc.h | 16 +++++++++------- src/mem/localcache.h | 2 +- src/mem/remoteallocator.h | 4 ++-- src/override/malloc.cc | 8 ++++---- src/override/memcpy.cc | 4 ++-- src/pal/pal_apple.h | 4 ++-- 11 files changed, 36 insertions(+), 33 deletions(-) diff --git a/src/backend/pagemap.h b/src/backend/pagemap.h index e80d94556..48699cd96 100644 --- a/src/backend/pagemap.h +++ b/src/backend/pagemap.h @@ -240,7 +240,7 @@ namespace snmalloc { if constexpr (potentially_out_of_range) { - if (unlikely(body_opt == nullptr)) + if (SNMALLOC_UNLIKELY(body_opt == nullptr)) return default_value; } diff --git a/src/ds/defines.h b/src/ds/defines.h index 30194935a..946f5b22c 100644 --- a/src/ds/defines.h +++ b/src/ds/defines.h @@ -6,8 +6,8 @@ # define SNMALLOC_FAST_FAIL() __fastfail(28) # define ALWAYSINLINE __forceinline # define NOINLINE __declspec(noinline) -# define likely(x) !!(x) -# define unlikely(x) !!(x) +# define SNMALLOC_LIKELY(x) !!(x) +# define SNMALLOC_UNLIKELY(x) !!(x) # define SNMALLOC_SLOW_PATH NOINLINE # define SNMALLOC_FAST_PATH ALWAYSINLINE /** @@ -26,8 +26,8 @@ # define SNMALLOC_UNUSED_FUNCTION #else # define SNMALLOC_FAST_FAIL() __builtin_trap() -# define likely(x) __builtin_expect(!!(x), 1) -# define unlikely(x) __builtin_expect(!!(x), 0) +# define SNMALLOC_LIKELY(x) __builtin_expect(!!(x), 1) +# define SNMALLOC_UNLIKELY(x) __builtin_expect(!!(x), 0) # define ALWAYSINLINE __attribute__((always_inline)) # define NOINLINE __attribute__((noinline)) # define SNMALLOC_SLOW_PATH NOINLINE @@ -152,7 +152,7 @@ inline SNMALLOC_FAST_PATH void check_client_error(const char* const str) inline SNMALLOC_FAST_PATH void check_client_impl(bool test, const char* const str) { - if (unlikely(!test)) + if (SNMALLOC_UNLIKELY(!test)) check_client_error(str); } #ifdef SNMALLOC_CHECK_CLIENT diff --git a/src/ds/helpers.h b/src/ds/helpers.h index b8bc15b17..dba61064e 100644 --- a/src/ds/helpers.h +++ b/src/ds/helpers.h @@ -32,7 +32,7 @@ namespace snmalloc // If defined should be initially false; SNMALLOC_ASSERT(first == nullptr || *first == false); - if (unlikely(!initialised.load(std::memory_order_acquire))) + if (SNMALLOC_UNLIKELY(!initialised.load(std::memory_order_acquire))) { FlagLock lock(flag); if (!initialised) diff --git a/src/ds/spmcstack.h b/src/ds/spmcstack.h index 5651a09ba..7c6ea70e3 100644 --- a/src/ds/spmcstack.h +++ b/src/ds/spmcstack.h @@ -51,7 +51,7 @@ namespace snmalloc if (stack.load(std::memory_order_relaxed) == nullptr) return nullptr; T* old_head = stack.exchange(nullptr); - if (unlikely(old_head == nullptr)) + if (SNMALLOC_UNLIKELY(old_head == nullptr)) return nullptr; auto next = old_head->next.load(std::memory_order_relaxed); diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 5811079b0..ca9937ee2 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -513,9 +513,10 @@ namespace snmalloc // TODO this needs to not double revoke if using MTE // TODO thread capabilities? - if (likely(entry.get_remote() == public_state())) + if (SNMALLOC_LIKELY(entry.get_remote() == public_state())) { - if (likely(dealloc_local_object_fast(entry, p.as_void(), entropy))) + if (SNMALLOC_LIKELY( + dealloc_local_object_fast(entry, p.as_void(), entropy))) return; dealloc_local_object_slow(entry); @@ -635,7 +636,7 @@ namespace snmalloc handle_message_queue(Action action, Args... args) { // Inline the empty check, but not necessarily the full queue handling. - if (likely(!has_messages())) + if (SNMALLOC_LIKELY(!has_messages())) { return action(args...); } @@ -648,7 +649,7 @@ namespace snmalloc { auto entry = SharedStateHandle::Pagemap::get_metaentry( backend_state_ptr(), snmalloc::address_cast(p)); - if (likely(dealloc_local_object_fast(entry, p, entropy))) + if (SNMALLOC_LIKELY(dealloc_local_object_fast(entry, p, entropy))) return; dealloc_local_object_slow(entry); @@ -675,7 +676,7 @@ namespace snmalloc // Update the head and the next pointer in the free list. meta->free_queue.add(cp, key, entropy); - return likely(!meta->return_object()); + return SNMALLOC_LIKELY(!meta->return_object()); } template @@ -684,12 +685,12 @@ namespace snmalloc { // Look to see if we can grab a free list. auto& sl = alloc_classes[sizeclass].available; - if (likely(alloc_classes[sizeclass].length > 0)) + if (SNMALLOC_LIKELY(alloc_classes[sizeclass].length > 0)) { #ifdef SNMALLOC_CHECK_CLIENT // Occassionally don't use the last list. if ( - unlikely(alloc_classes[sizeclass].length == 1) && + SNMALLOC_UNLIKELY(alloc_classes[sizeclass].length == 1) && (entropy.next_bit() == 0)) { return small_alloc_slow(sizeclass, fast_free_list); diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index f994a4b3e..21886baee 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -97,7 +97,7 @@ namespace snmalloc template SNMALLOC_FAST_PATH decltype(auto) check_init(Action action, Args... args) { - if (likely(core_alloc != nullptr)) + if (SNMALLOC_LIKELY(core_alloc != nullptr)) { return core_alloc->handle_message_queue(action, core_alloc, args...); } @@ -219,7 +219,7 @@ namespace snmalloc auto slowpath = [&]( smallsizeclass_t sizeclass, freelist::Iter<>* fl) SNMALLOC_FAST_PATH_LAMBDA { - if (likely(core_alloc != nullptr)) + if (SNMALLOC_LIKELY(core_alloc != nullptr)) { return core_alloc->handle_message_queue( []( @@ -428,7 +428,7 @@ namespace snmalloc #else // Perform the - 1 on size, so that zero wraps around and ends up on // slow path. - if (likely( + if (SNMALLOC_LIKELY( (size - 1) <= (sizeclass_to_size(NUM_SMALL_SIZECLASSES - 1) - 1))) { // Small allocations are more likely. Improve @@ -478,16 +478,17 @@ namespace snmalloc const MetaEntry& entry = SharedStateHandle::Pagemap::get_metaentry( core_alloc->backend_state_ptr(), address_cast(p_tame)); - if (likely(local_cache.remote_allocator == entry.get_remote())) + if (SNMALLOC_LIKELY(local_cache.remote_allocator == entry.get_remote())) { - if (likely(CoreAlloc::dealloc_local_object_fast( + if (SNMALLOC_LIKELY(CoreAlloc::dealloc_local_object_fast( entry, p_tame, local_cache.entropy))) return; core_alloc->dealloc_local_object_slow(entry); return; } - if (likely(entry.get_remote() != SharedStateHandle::fake_large_remote)) + if (SNMALLOC_LIKELY( + entry.get_remote() != SharedStateHandle::fake_large_remote)) { // Check if we have space for the remote deallocation if (local_cache.remote_dealloc_cache.reserve_space(entry)) @@ -507,7 +508,8 @@ namespace snmalloc // Large deallocation or null. // also checks for managed by page map. - if (likely((p_tame != nullptr) && !entry.get_sizeclass().is_default())) + if (SNMALLOC_LIKELY( + (p_tame != nullptr) && !entry.get_sizeclass().is_default())) { size_t entry_sizeclass = entry.get_sizeclass().as_large(); diff --git a/src/mem/localcache.h b/src/mem/localcache.h index 6e98bb8a4..caa19c6b3 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -112,7 +112,7 @@ namespace snmalloc stats.alloc_request(size); stats.sizeclass_alloc(sizeclass); auto& fl = small_fast_free_lists[sizeclass]; - if (likely(!fl.empty())) + if (SNMALLOC_LIKELY(!fl.empty())) { auto p = fl.take(key, domesticate); return finish_alloc(p, sizeclass); diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index 55205937e..6e33b58b2 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -147,11 +147,11 @@ namespace snmalloc freelist::HeadPtr next = curr->atomic_read_next(key, domesticate_queue); // We have observed a non-linearisable effect of the queue. // Just go back to allocating normally. - if (unlikely(next == nullptr)) + if (SNMALLOC_UNLIKELY(next == nullptr)) break; // We want this element next, so start it loading. Aal::prefetch(next.unsafe_ptr()); - if (unlikely(!cb(curr))) + if (SNMALLOC_UNLIKELY(!cb(curr))) { /* * We've domesticate_queue-d next so that we can read through it, but diff --git a/src/override/malloc.cc b/src/override/malloc.cc index 6db61ae8e..658e2badf 100644 --- a/src/override/malloc.cc +++ b/src/override/malloc.cc @@ -47,7 +47,7 @@ extern "C" { bool overflow = false; size_t sz = bits::umul(size, nmemb, overflow); - if (unlikely(overflow)) + if (SNMALLOC_UNLIKELY(overflow)) { return SNMALLOC_NAME_MANGLE(snmalloc_set_error)(); } @@ -85,7 +85,7 @@ extern "C" } void* p = a.alloc(size); - if (likely(p != nullptr)) + if (SNMALLOC_LIKELY(p != nullptr)) { sz = bits::min(size, sz); // Guard memcpy as GCC is assuming not nullptr for ptr after the memcpy @@ -94,7 +94,7 @@ extern "C" memcpy(p, ptr, sz); a.dealloc(ptr); } - else if (likely(size == 0)) + else if (SNMALLOC_LIKELY(size == 0)) { a.dealloc(ptr); } @@ -150,7 +150,7 @@ extern "C" } void* p = SNMALLOC_NAME_MANGLE(memalign)(alignment, size); - if (unlikely(p == nullptr)) + if (SNMALLOC_UNLIKELY(p == nullptr)) { if (size != 0) return ENOMEM; diff --git a/src/override/memcpy.cc b/src/override/memcpy.cc index c193dc28e..6d1ebce83 100644 --- a/src/override/memcpy.cc +++ b/src/override/memcpy.cc @@ -136,7 +136,7 @@ namespace auto& alloc = ThreadAlloc::get(); void* p = const_cast(ptr); - if (unlikely(alloc.remaining_bytes(ptr) < len)) + if (SNMALLOC_UNLIKELY(alloc.remaining_bytes(ptr) < len)) { if constexpr (FailFast) { @@ -209,7 +209,7 @@ extern "C" // 0 is a very common size for memcpy and we don't need to do external // pointer checks if we hit it. It's also the fastest case, to encourage // the compiler to favour the other cases. - if (unlikely(len == 0)) + if (SNMALLOC_UNLIKELY(len == 0)) { return dst; } diff --git a/src/pal/pal_apple.h b/src/pal/pal_apple.h index 1115e8c95..edf511bdd 100644 --- a/src/pal/pal_apple.h +++ b/src/pal/pal_apple.h @@ -175,7 +175,7 @@ namespace snmalloc anonymous_memory_fd, 0); - if (likely(r != MAP_FAILED)) + if (SNMALLOC_LIKELY(r != MAP_FAILED)) { return; } @@ -238,7 +238,7 @@ namespace snmalloc VM_PROT_READ | VM_PROT_WRITE, VM_INHERIT_COPY); - if (unlikely(kr != KERN_SUCCESS)) + if (SNMALLOC_UNLIKELY(kr != KERN_SUCCESS)) { return nullptr; } From f731bc169ba0f4810b3126ef0fae30583712dc7b Mon Sep 17 00:00:00 2001 From: David Carlier Date: Thu, 18 Nov 2021 07:39:29 +0000 Subject: [PATCH 154/302] build fix for some 3rd party oses. --- src/override/memcpy.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/override/memcpy.cc b/src/override/memcpy.cc index 6d1ebce83..b854e16a5 100644 --- a/src/override/memcpy.cc +++ b/src/override/memcpy.cc @@ -10,7 +10,8 @@ using namespace snmalloc; // glibc lacks snprintf_l -#ifdef __linux__ +#if defined(__linux__) || defined(__OpenBSD__) || defined(__DragonFly__) || \ + defined(__HAIKU__) # define snprintf_l(buf, size, loc, msg, ...) \ snprintf(buf, size, msg, __VA_ARGS__) // Windows has it with an underscore prefix From 8e5514bd5ac0b07c8ba3f4661fc158faa7b2f2ae Mon Sep 17 00:00:00 2001 From: Schrodinger ZHU Yifan Date: Fri, 19 Nov 2021 21:23:39 +0800 Subject: [PATCH 155/302] clean up unused usages (#421) * clean up unused usages Signed-off-by: SchrodingerZhu * remove names for arg pack Signed-off-by: SchrodingerZhu * fix namespace in setup.h Signed-off-by: SchrodingerZhu * format Signed-off-by: SchrodingerZhu * set UNUSED as fast path Signed-off-by: SchrodingerZhu --- src/backend/address_space_core.h | 3 +-- src/ds/defines.h | 5 +++-- src/mem/allocstats.h | 4 +--- src/mem/globalalloc.h | 3 +-- src/override/memcpy.cc | 8 ++------ src/pal/pal_noalloc.h | 3 +-- src/pal/pal_plain.h | 3 +-- src/pal/pal_posix.h | 9 +++------ src/test/func/bits/bits.cc | 3 +-- src/test/func/memory/memory.cc | 6 ++---- src/test/func/pagemap/pagemap.cc | 3 +-- src/test/func/pool/pool.cc | 3 +-- src/test/func/two_alloc_types/main.cc | 5 ++--- src/test/setup.h | 6 +++--- 14 files changed, 23 insertions(+), 41 deletions(-) diff --git a/src/backend/address_space_core.h b/src/backend/address_space_core.h index f3485421d..d050790ee 100644 --- a/src/backend/address_space_core.h +++ b/src/backend/address_space_core.h @@ -65,8 +65,7 @@ namespace snmalloc bits::align_up(address_cast(base), bits::one_at_bit(align_bits))); // All blocks need to be bigger than a pointer. SNMALLOC_ASSERT(bits::one_at_bit(align_bits) >= sizeof(void*)); - UNUSED(base); - UNUSED(align_bits); + UNUSED(base, align_bits); } /** diff --git a/src/ds/defines.h b/src/ds/defines.h index 946f5b22c..69b9106a4 100644 --- a/src/ds/defines.h +++ b/src/ds/defines.h @@ -90,8 +90,6 @@ # define __has_builtin(x) 0 #endif -#define UNUSED(x) ((void)(x)) - namespace snmalloc { // Forwards reference so that the platform can define how to handle errors. @@ -168,4 +166,7 @@ namespace snmalloc #else static constexpr bool CHECK_CLIENT = false; #endif + template + SNMALLOC_FAST_PATH_INLINE void UNUSED(Args&&...) + {} } // namespace snmalloc diff --git a/src/mem/allocstats.h b/src/mem/allocstats.h index 18896c6e2..2d3ae452d 100644 --- a/src/mem/allocstats.h +++ b/src/mem/allocstats.h @@ -322,9 +322,7 @@ namespace snmalloc template void print(std::ostream& o, uint64_t dumpid = 0, uint64_t allocatorid = 0) { - UNUSED(o); - UNUSED(dumpid); - UNUSED(allocatorid); + UNUSED(o, dumpid, allocatorid); CSVStream csv(&o); diff --git a/src/mem/globalalloc.h b/src/mem/globalalloc.h index 2a0b9caf9..f32413eaa 100644 --- a/src/mem/globalalloc.h +++ b/src/mem/globalalloc.h @@ -44,8 +44,7 @@ namespace snmalloc template inline static void print_all_stats(void*& o, uint64_t dumpid = 0) { - UNUSED(o); - UNUSED(dumpid); + UNUSED(o, dumpid); } #endif diff --git a/src/override/memcpy.cc b/src/override/memcpy.cc index b854e16a5..43c27b25d 100644 --- a/src/override/memcpy.cc +++ b/src/override/memcpy.cc @@ -141,9 +141,7 @@ namespace { if constexpr (FailFast) { - UNUSED(ptr); - UNUSED(len); - UNUSED(msg); + UNUSED(ptr, len, msg); SNMALLOC_FAST_FAIL(); } else @@ -154,9 +152,7 @@ namespace } else { - UNUSED(ptr); - UNUSED(len); - UNUSED(msg); + UNUSED(ptr, len, msg); } } diff --git a/src/pal/pal_noalloc.h b/src/pal/pal_noalloc.h index 8228d4437..9092b65fe 100644 --- a/src/pal/pal_noalloc.h +++ b/src/pal/pal_noalloc.h @@ -77,8 +77,7 @@ namespace snmalloc } else { - UNUSED(p); - UNUSED(size); + UNUSED(p, size); } } diff --git a/src/pal/pal_plain.h b/src/pal/pal_plain.h index 57bfe1879..27230f2b7 100644 --- a/src/pal/pal_plain.h +++ b/src/pal/pal_plain.h @@ -24,8 +24,7 @@ namespace snmalloc } else { - UNUSED(p); - UNUSED(size); + UNUSED(p, size); } } }; diff --git a/src/pal/pal_posix.h b/src/pal/pal_posix.h index c4b85ab1e..e75020da0 100644 --- a/src/pal/pal_posix.h +++ b/src/pal/pal_posix.h @@ -194,8 +194,7 @@ namespace snmalloc } else { - UNUSED(p); - UNUSED(size); + UNUSED(p, size); } } @@ -216,8 +215,7 @@ namespace snmalloc mprotect(p, size, PROT_READ | PROT_WRITE); else { - UNUSED(p); - UNUSED(size); + UNUSED(p, size); } if constexpr (zero_mem == YesZero) @@ -238,8 +236,7 @@ namespace snmalloc mprotect(p, size, PROT_READ); else { - UNUSED(p); - UNUSED(size); + UNUSED(p, size); } } diff --git a/src/test/func/bits/bits.cc b/src/test/func/bits/bits.cc index f78be0b37..874c5fbd6 100644 --- a/src/test/func/bits/bits.cc +++ b/src/test/func/bits/bits.cc @@ -35,8 +35,7 @@ void test_clz() int main(int argc, char** argv) { - UNUSED(argc); - UNUSED(argv); + snmalloc::UNUSED(argc, argv); setup(); diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index 6185c5a93..512728553 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -27,8 +27,7 @@ using namespace snmalloc; void test_limited(rlim64_t as_limit, size_t& count) { - UNUSED(as_limit); - UNUSED(count); + UNUSED(as_limit, count); #if false && defined(TEST_LIMITED) auto pid = fork(); if (!pid) @@ -490,8 +489,7 @@ int main(int argc, char** argv) size_t seed = opt.is("--seed", 0); Virtual::systematic_bump_ptr() += seed << 17; #else - UNUSED(argc); - UNUSED(argv); + UNUSED(argc, argv); #endif test_alloc_dealloc_64k(); test_random_allocation(); diff --git a/src/test/func/pagemap/pagemap.cc b/src/test/func/pagemap/pagemap.cc index d9be2838e..410ba8cd2 100644 --- a/src/test/func/pagemap/pagemap.cc +++ b/src/test/func/pagemap/pagemap.cc @@ -120,8 +120,7 @@ void test_pagemap(bool bounded) int main(int argc, char** argv) { - UNUSED(argc); - UNUSED(argv); + UNUSED(argc, argv); setup(); diff --git a/src/test/func/pool/pool.cc b/src/test/func/pool/pool.cc index f479f4ef6..7d63e9e2a 100644 --- a/src/test/func/pool/pool.cc +++ b/src/test/func/pool/pool.cc @@ -124,8 +124,7 @@ int main(int argc, char** argv) size_t seed = opt.is("--seed", 0); Virtual::systematic_bump_ptr() += seed << 17; #else - UNUSED(argc); - UNUSED(argv); + UNUSED(argc, argv); #endif test_alloc(); diff --git a/src/test/func/two_alloc_types/main.cc b/src/test/func/two_alloc_types/main.cc index b722a7ec3..3cea54cb3 100644 --- a/src/test/func/two_alloc_types/main.cc +++ b/src/test/func/two_alloc_types/main.cc @@ -7,14 +7,13 @@ extern "C" void* oe_memset_s(void* p, size_t p_size, int c, size_t size) { - UNUSED(p_size); + snmalloc::UNUSED(p_size); return memset(p, c, size); } extern "C" int oe_random(void* p, size_t p_size) { - UNUSED(p_size); - UNUSED(p); + snmalloc::UNUSED(p_size, p); // Stub for random data. return 0; } diff --git a/src/test/setup.h b/src/test/setup.h index bcf303561..319902ea4 100644 --- a/src/test/setup.h +++ b/src/test/setup.h @@ -63,7 +63,7 @@ void print_stack_trace() void _cdecl error(int signal) { - UNUSED(signal); + snmalloc::UNUSED(signal); puts("*****ABORT******"); print_stack_trace(); @@ -73,7 +73,7 @@ void _cdecl error(int signal) LONG WINAPI VectoredHandler(struct _EXCEPTION_POINTERS* ExceptionInfo) { - UNUSED(ExceptionInfo); + snmalloc::UNUSED(ExceptionInfo); puts("*****UNHANDLED EXCEPTION******"); @@ -100,7 +100,7 @@ void setup() # include void error_handle(int signal) { - UNUSED(signal); + snmalloc::UNUSED(signal); snmalloc::error("Seg Fault"); _exit(1); } From e24130137da56a7902657137b6abb062ac60ba81 Mon Sep 17 00:00:00 2001 From: David CARLIER Date: Mon, 22 Nov 2021 09:47:30 +0000 Subject: [PATCH 156/302] solaris systems build fix (#425) --- src/override/memcpy.cc | 2 +- src/test/func/memory/memory.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/override/memcpy.cc b/src/override/memcpy.cc index 43c27b25d..6c858f422 100644 --- a/src/override/memcpy.cc +++ b/src/override/memcpy.cc @@ -11,7 +11,7 @@ using namespace snmalloc; // glibc lacks snprintf_l #if defined(__linux__) || defined(__OpenBSD__) || defined(__DragonFly__) || \ - defined(__HAIKU__) + defined(__HAIKU__) || defined(__sun) # define snprintf_l(buf, size, loc, msg, ...) \ snprintf(buf, size, msg, __VA_ARGS__) // Windows has it with an underscore prefix diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index 512728553..c6ce764ce 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -4,9 +4,9 @@ #include #include #include -#if defined(__linux__) && !defined(SNMALLOC_QEMU_WORKAROUND) +#if (defined(__linux__) || defined(__sun)) && !defined(SNMALLOC_QEMU_WORKAROUND) /* - * We only test allocations with limited AS on linux for now. + * We only test allocations with limited AS on linux and Solaris for now. * It should be a good representative for POSIX systems. * QEMU `setrlimit64` does not behave as the same as native linux, * so we need to exclude it from such tests. From 9332557bb01efee6793c669a87a565bdc73ac59f Mon Sep 17 00:00:00 2001 From: David CARLIER Date: Wed, 24 Nov 2021 12:12:47 +0000 Subject: [PATCH 157/302] memory unit test android build fix. (#427) --- src/test/func/memory/memory.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index c6ce764ce..a650a7c8d 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -4,7 +4,8 @@ #include #include #include -#if (defined(__linux__) || defined(__sun)) && !defined(SNMALLOC_QEMU_WORKAROUND) +#if ((defined(__linux__) && !defined(__ANDROID__)) || defined(__sun)) && \ + !defined(SNMALLOC_QEMU_WORKAROUND) /* * We only test allocations with limited AS on linux and Solaris for now. * It should be a good representative for POSIX systems. From d725beca9052c285445d6ed130c59fa93bf21cd1 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 24 Nov 2021 14:33:57 +0000 Subject: [PATCH 158/302] Minor improvement to codegen for remaining_bytes --- src/mem/sizeclasstable.h | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index 899d5c2f9..a69760463 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -274,6 +274,11 @@ namespace snmalloc return sizeclass_metadata.fast(sizeclass).size; } + inline static size_t sizeclass_full_to_slab_size(sizeclass_t sizeclass) + { + return sizeclass_metadata.fast(sizeclass).slab_mask + 1; + } + inline static size_t sizeclass_to_slab_size(smallsizeclass_t sizeclass) { return sizeclass_metadata.fast_small(sizeclass).slab_mask + 1; @@ -321,10 +326,10 @@ namespace snmalloc .capacity; } - inline static size_t mod_by_sizeclass(smallsizeclass_t sc, size_t offset) + inline static size_t mod_by_sizeclass(sizeclass_t sc, size_t offset) { // Only works up to certain offsets, exhaustively tested by rounding.cc - auto meta = sizeclass_metadata.fast_small(sc); + auto meta = sizeclass_metadata.fast(sc); // Powers of two should use straigt mask. SNMALLOC_ASSERT(meta.mod_mult != 0); @@ -353,8 +358,8 @@ namespace snmalloc return addr & (sizeclass_metadata.fast(sc).size - 1); } - address_t offset = addr & (sizeclass_to_slab_size(sc.as_small()) - 1); - return mod_by_sizeclass(sc.as_small(), offset); + address_t offset = addr & (sizeclass_full_to_slab_size(sc) - 1); + return mod_by_sizeclass(sc, offset); } inline static size_t remaining_bytes(sizeclass_t sc, address_t addr) From a8ef963ed7405f0770016b3bb532cda7a95f5491 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 24 Nov 2021 14:34:07 +0000 Subject: [PATCH 159/302] Small comment --- src/mem/localalloc.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 21886baee..4568f287f 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -338,6 +338,11 @@ namespace snmalloc public: constexpr LocalAllocator() = default; + /** + * Remove copy constructors and assignment operators. + * Once initialised the CoreAlloc will take references to the internals + * of this allocators, and thus copying/moving it is very unsound. + */ LocalAllocator(const LocalAllocator&) = delete; LocalAllocator& operator=(const LocalAllocator&) = delete; From c299826f582d991a480ac87a608e6c391b5c3437 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 24 Nov 2021 14:34:26 +0000 Subject: [PATCH 160/302] Improve codegen for checks. Use fail fast in release to avoid stack frame for error reporting. Scope check_client macro. --- src/ds/defines.h | 47 +++++++++++++++++++++++++++----------------- src/mem/corealloc.h | 4 ++-- src/mem/freelist.h | 2 +- src/mem/localalloc.h | 4 ++-- 4 files changed, 34 insertions(+), 23 deletions(-) diff --git a/src/ds/defines.h b/src/ds/defines.h index 69b9106a4..10bba4518 100644 --- a/src/ds/defines.h +++ b/src/ds/defines.h @@ -3,6 +3,7 @@ #if defined(_MSC_VER) && !defined(__clang__) // 28 is FAST_FAIL_INVALID_BUFFER_ACCESS. Not using the symbolic constant to // avoid depending on winnt.h +# include // for __fastfail # define SNMALLOC_FAST_FAIL() __fastfail(28) # define ALWAYSINLINE __forceinline # define NOINLINE __declspec(noinline) @@ -141,32 +142,42 @@ namespace snmalloc # endif #endif -inline SNMALLOC_FAST_PATH void check_client_error(const char* const str) +namespace snmalloc { - //[[clang::musttail]] - return snmalloc::error(str); -} + template + SNMALLOC_FAST_PATH_INLINE void UNUSED(Args&&...) + {} -inline SNMALLOC_FAST_PATH void -check_client_impl(bool test, const char* const str) -{ - if (SNMALLOC_UNLIKELY(!test)) - check_client_error(str); -} -#ifdef SNMALLOC_CHECK_CLIENT -# define check_client(test, str) check_client_impl(test, str) + inline SNMALLOC_FAST_PATH void check_client_error(const char* const str) + { + //[[clang::musttail]] + return snmalloc::error(str); + } + + inline SNMALLOC_FAST_PATH void + check_client_impl(bool test, const char* const str) + { + if (SNMALLOC_UNLIKELY(!test)) + { +#ifdef NDEBUG + UNUSED(str); + SNMALLOC_FAST_FAIL(); #else -# define check_client(test, str) + check_client_error(str); #endif + } + } -namespace snmalloc -{ #ifdef SNMALLOC_CHECK_CLIENT static constexpr bool CHECK_CLIENT = true; #else static constexpr bool CHECK_CLIENT = false; #endif - template - SNMALLOC_FAST_PATH_INLINE void UNUSED(Args&&...) - {} } // namespace snmalloc + +#ifdef SNMALLOC_CHECK_CLIENT +# define snmalloc_check_client(test, str) \ + snmalloc::check_client_impl(test, str) +#else +# define snmalloc_check_client(test, str) +#endif diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index ca9937ee2..89adfc81b 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -379,7 +379,7 @@ namespace snmalloc alloc_classes[sizeclass].unused--; // TODO delay the clear to the next user of the slab, or teardown so - // don't touch the cache lines at this point in check_client. + // don't touch the cache lines at this point in snmalloc_check_client. auto chunk_record = clear_slab(meta, sizeclass); ChunkAllocator::dealloc( get_backend_local_state(), @@ -664,7 +664,7 @@ namespace snmalloc SNMALLOC_ASSERT(!meta->is_unused()); - check_client( + snmalloc_check_client( Metaslab::is_start_of_object( entry.get_sizeclass().as_small(), address_cast(p)), "Not deallocating start of an object"); diff --git a/src/mem/freelist.h b/src/mem/freelist.h index 4640534ef..d2e5a1851 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -181,7 +181,7 @@ namespace snmalloc void check_prev(address_t signed_prev) { UNUSED(signed_prev); - check_client( + snmalloc_check_client( signed_prev == this->prev_encoded, "Heap corruption - free list corrupted!"); } diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 4568f287f..b243290ec 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -523,7 +523,7 @@ namespace snmalloc metaentry_chunk_sizeclass_to_slab_sizeclass(entry_sizeclass); // Check for start of allocation. - check_client( + snmalloc_check_client( pointer_align_down(p_tame, size) == p_tame, "Not start of an allocation."); @@ -560,7 +560,7 @@ namespace snmalloc // If p_tame is not null, then dealloc has been call on something // it shouldn't be called on. // TODO: Should this be tested even in the !CHECK_CLIENT case? - check_client(p_tame == nullptr, "Not allocated by snmalloc."); + snmalloc_check_client(p_tame == nullptr, "Not allocated by snmalloc."); # ifdef SNMALLOC_TRACING std::cout << "nullptr deallocation" << std::endl; From 5fd3288997dff464b06fdb17890380a9b6aa8a55 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 25 Nov 2021 13:43:50 +0000 Subject: [PATCH 161/302] Modify heuristic for adding new slabs. (#429) If there is only one slab remaining, then we probabalisticly allocator a new one. If a slab is barely in use, then this could cause us to effectively double the number of slabs in use. This commit checks if the remaining slab has enough remaining elements to provide randomisation. --- src/ds/seqset.h | 8 ++++++++ src/mem/corealloc.h | 13 +++++++++---- src/mem/freelist.h | 14 ++++++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/ds/seqset.h b/src/ds/seqset.h index 13f9fafa9..da5eb46ab 100644 --- a/src/ds/seqset.h +++ b/src/ds/seqset.h @@ -159,5 +159,13 @@ namespace snmalloc v.end = &(item->next); } } + + /** + * Peek at next element in the set. + */ + SNMALLOC_FAST_PATH const T* peek() + { + return v.head; + } }; } // namespace snmalloc diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 89adfc81b..f5f03f896 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -689,11 +689,16 @@ namespace snmalloc { #ifdef SNMALLOC_CHECK_CLIENT // Occassionally don't use the last list. - if ( - SNMALLOC_UNLIKELY(alloc_classes[sizeclass].length == 1) && - (entropy.next_bit() == 0)) + if (SNMALLOC_UNLIKELY(alloc_classes[sizeclass].length == 1)) { - return small_alloc_slow(sizeclass, fast_free_list); + // If the slab has a lot of free space, then we shouldn't allocate a + // new slab. + auto min = alloc_classes[sizeclass] + .available.peek() + ->free_queue.min_list_length(); + if ((min * 2) < threshold_for_waking_slab(sizeclass)) + if (entropy.next_bit() == 0) + return small_alloc_slow(sizeclass, fast_free_list); } #endif diff --git a/src/mem/freelist.h b/src/mem/freelist.h index d2e5a1851..fbd850907 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -740,6 +740,20 @@ namespace snmalloc UNUSED(domesticate); #endif } + + /** + * Returns length of the shorter free list. + * + * This method is only usable if the free list is adding randomisation + * as that is when it has two lists. + */ + template + [[nodiscard]] std::enable_if_t min_list_length() const + { + static_assert(RANDOM_ == RANDOM, "Don't set SFINAE parameter!"); + + return length[0] < length[1] ? length[0] : length[1]; + } }; } // namespace freelist } // namespace snmalloc From b1da339b3e57fe8fdf5dc2a3c0cb30fb2ea993c2 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Wed, 1 Dec 2021 08:12:46 +0000 Subject: [PATCH 162/302] build fix proposal for GCC 11.x (spotted with 11.2.0). build error due to access none attribute on the pthread_setspecific's value argumenti (glibc). --- src/mem/threadalloc.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mem/threadalloc.h b/src/mem/threadalloc.h index 4f6b674e9..c7d23f1b0 100644 --- a/src/mem/threadalloc.h +++ b/src/mem/threadalloc.h @@ -71,7 +71,6 @@ namespace snmalloc # pragma warning(pop) # endif #else - /** * Holds the thread local state for the allocator. The state is constant * initialised, and has no direct dectructor. Instead snmalloc will call @@ -137,7 +136,8 @@ namespace snmalloc Singleton p_key; // We need to set a non-null value, so that the destructor is called, // we never look at the value. - pthread_setspecific(p_key.get(), reinterpret_cast(1)); + static char p_teardown_val = 1; + pthread_setspecific(p_key.get(), &p_teardown_val); # ifdef SNMALLOC_TRACING std::cout << "Using pthread clean up" << std::endl; # endif From 9b548ed3a263f85b5f737236c6cc6e0d2829d6e9 Mon Sep 17 00:00:00 2001 From: Schrodinger ZHU Yifan Date: Thu, 2 Dec 2021 17:18:24 +0800 Subject: [PATCH 163/302] Add ownership checkings for FlagLock under debug (#432) * add ownership checkings for FlagLock under debug Signed-off-by: SchrodingerZhu * fix owner reset ordering Signed-off-by: SchrodingerZhu * addresss CR Signed-off-by: SchrodingerZhu * fix include and constexpr problem Signed-off-by: SchrodingerZhu * use thread_local variable as thread identity Signed-off-by: SchrodingerZhu * change default form for initialization Signed-off-by: SchrodingerZhu * address CR Signed-off-by: SchrodingerZhu * fix typo and format Signed-off-by: SchrodingerZhu * add more assertions Signed-off-by: SchrodingerZhu * adjust flag lock and comments Signed-off-by: SchrodingerZhu * address CR Signed-off-by: SchrodingerZhu --- src/backend/address_space.h | 2 +- src/backend/globalconfig.h | 2 +- src/ds/flaglock.h | 124 ++++++++++++++++++++++++++++++++++-- src/ds/helpers.h | 2 +- src/mem/pool.h | 2 +- 5 files changed, 124 insertions(+), 8 deletions(-) diff --git a/src/backend/address_space.h b/src/backend/address_space.h index 02c48f0b5..d2e29f26f 100644 --- a/src/backend/address_space.h +++ b/src/backend/address_space.h @@ -28,7 +28,7 @@ namespace snmalloc * This is infrequently used code, a spin lock simplifies the code * considerably, and should never be on the fast path. */ - std::atomic_flag spin_lock = ATOMIC_FLAG_INIT; + FlagWord spin_lock{}; public: /** diff --git a/src/backend/globalconfig.h b/src/backend/globalconfig.h index 04646e3b7..b1243a511 100644 --- a/src/backend/globalconfig.h +++ b/src/backend/globalconfig.h @@ -46,7 +46,7 @@ namespace snmalloc inline static std::atomic initialised{false}; SNMALLOC_REQUIRE_CONSTINIT - inline static std::atomic_flag initialisation_lock{}; + inline static FlagWord initialisation_lock{}; public: static ChunkAllocatorState& diff --git a/src/ds/flaglock.h b/src/ds/flaglock.h index 69ec3079e..954adc0ec 100644 --- a/src/ds/flaglock.h +++ b/src/ds/flaglock.h @@ -6,21 +6,137 @@ namespace snmalloc { + /** + * @brief The DebugFlagWord struct + * Wrapper for std::atomic_flag so that we can examine + * the re-entrancy problem at debug mode. + */ + struct DebugFlagWord + { + /** + * @brief flag + * The underlying atomic field. + */ + std::atomic_flag flag = ATOMIC_FLAG_INIT; + + constexpr DebugFlagWord() = default; + + template + constexpr DebugFlagWord(Args&&... args) : flag(std::forward(args)...) + {} + + /** + * @brief set_owner + * Record the identity of the locker. + */ + void set_owner() + { + SNMALLOC_ASSERT(nullptr == owner); + owner = get_thread_identity(); + } + + /** + * @brief clear_owner + * Set the identity to null. + */ + void clear_owner() + { + SNMALLOC_ASSERT(get_thread_identity() == owner); + owner = nullptr; + } + + /** + * @brief assert_not_owned_by_current_thread + * Assert the lock should not be held already by current thread. + */ + void assert_not_owned_by_current_thread() + { + SNMALLOC_ASSERT(get_thread_identity() != owner); + } + + private: + using ThreadIdentity = int const*; + + /** + * @brief owner + * We use a pointer to TLS field as the thread identity. + * std::thread::id can be another solution but it does not + * support `constexpr` initialisation on some platforms. + */ + ThreadIdentity owner = nullptr; + + /** + * @brief get_thread_identity + * @return The identity of current thread. + */ + inline ThreadIdentity get_thread_identity() + { + static thread_local int SNMALLOC_THREAD_IDENTITY = 0; + return &SNMALLOC_THREAD_IDENTITY; + } + }; + + /** + * @brief The ReleaseFlagWord struct + * The shares the same structure with DebugFlagWord but + * all member functions associated with ownership checkings + * are empty so that they can be optimised out at Release mode. + */ + struct ReleaseFlagWord + { + std::atomic_flag flag = ATOMIC_FLAG_INIT; + + constexpr ReleaseFlagWord() = default; + + template + constexpr ReleaseFlagWord(Args&&... args) + : flag(std::forward(args)...) + {} + + void set_owner() {} + void clear_owner() {} + void assert_not_owned_by_current_thread() {} + }; + +#ifdef NDEBUG + using FlagWord = ReleaseFlagWord; +#else + using FlagWord = DebugFlagWord; +#endif + class FlagLock { private: - std::atomic_flag& lock; + FlagWord& lock; public: - FlagLock(std::atomic_flag& lock) : lock(lock) + FlagLock(FlagWord& lock) : lock(lock) { - while (lock.test_and_set(std::memory_order_acquire)) + while (lock.flag.test_and_set(std::memory_order_acquire)) + { + // assert_not_owned_by_current_thread is only called when the first + // acquiring is failed; which means the lock is already held somewhere + // else. + lock.assert_not_owned_by_current_thread(); +#ifdef __cpp_lib_atomic_flag_test + // acquire ordering because we need other thread's release to be + // visible. This loop is better for spin-waiting because it won't issue + // expensive write operation (xchg for example). + while (lock.flag.test(std::memory_order_acquire)) + { + Aal::pause(); + } +#else Aal::pause(); +#endif + } + lock.set_owner(); } ~FlagLock() { - lock.clear(std::memory_order_release); + lock.clear_owner(); + lock.flag.clear(std::memory_order_release); } }; } // namespace snmalloc diff --git a/src/ds/helpers.h b/src/ds/helpers.h index dba61064e..9213294ae 100644 --- a/src/ds/helpers.h +++ b/src/ds/helpers.h @@ -17,7 +17,7 @@ namespace snmalloc template class Singleton { - inline static std::atomic_flag flag; + inline static FlagWord flag; inline static std::atomic initialised{false}; inline static Object obj; diff --git a/src/mem/pool.h b/src/mem/pool.h index c773e89f8..ffb9572b6 100644 --- a/src/mem/pool.h +++ b/src/mem/pool.h @@ -29,7 +29,7 @@ namespace snmalloc private: MPMCStack stack; - std::atomic_flag lock = ATOMIC_FLAG_INIT; + FlagWord lock{}; T* list{nullptr}; public: From 894b0314c9c49d3c2ab6a4336fed2f7678db17b3 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 1 Dec 2021 20:09:48 +0000 Subject: [PATCH 164/302] Add timeout to ci tests. Windows can hang due to assert failures in CI. Add a timeout to get some information of what is happening. --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dd5153dfd..ee4c4a36c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -99,7 +99,7 @@ jobs: - name: Test if: ${{ matrix.build-only != 'yes' }} working-directory: ${{github.workspace}}/build - run: ctest --output-on-failure -j 4 -C ${{ matrix.build-type }} + run: ctest --output-on-failure -j 4 -C ${{ matrix.build-type }} --timeout 400 - name: Selfhost if: ${{ matrix.self-host }} working-directory: ${{github.workspace}}/build @@ -182,7 +182,7 @@ jobs: # QEMU) - name: Test working-directory: ${{github.workspace}}/build - run: ctest --output-on-failure -E '(perf-.*)|(.*-malloc$)' + run: ctest --output-on-failure -E '(perf-.*)|(.*-malloc$)' --timeout 400 timeout-minutes: 30 windows: @@ -222,7 +222,7 @@ jobs: # Run the tests. - name: Test working-directory: ${{ github.workspace }}/build - run: ctest -j 2 --interactive-debug-mode 0 --output-on-failure -C ${{ matrix.build-type }} + run: ctest -j 2 --interactive-debug-mode 0 --output-on-failure -C ${{ matrix.build-type }} --timeout 400 timeout-minutes: 20 From 9329db102fdfb3475c94bd03f9ee76635372cbb3 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 2 Dec 2021 09:13:36 +0000 Subject: [PATCH 165/302] Remove Win2016 as Github Action support ending. --- .github/workflows/main.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ee4c4a36c..88065d914 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -189,19 +189,14 @@ jobs: strategy: matrix: # Build each combination of OS and release/debug variants - os: [ windows-2016, windows-2019 ] + os: [ windows-2019 ] build-type: [ Release, Debug ] arch: [ Win32, x64 ] toolchain: [ "", "-T ClangCL" ] extra-cmake-flags: [ "" ] - # The ClangCL toolchain was added in Visual Studio 2019, the Windows - # 2016 runners have only VS 2017, so skip them for this configuration - exclude: - - os: windows-2016 - toolchain: "-T ClangCL" # Add an extra check for the Windows 8 compatible PAL include: - - os: windows-2016 + - os: windows-2019 build-type: Release arch: x64 toolchain: "" From 71d5bb8756f1f13bd77625d5cc72f4a1cb5b18c9 Mon Sep 17 00:00:00 2001 From: Schrodinger ZHU Yifan Date: Thu, 2 Dec 2021 18:38:50 +0800 Subject: [PATCH 166/302] add SYS_getrandom fallback for posix PAL (#431) * add SYS_getrandom fallback for posix PAL Signed-off-by: SchrodingerZhu * address CR Signed-off-by: SchrodingerZhu * guard and comments Signed-off-by: SchrodingerZhu * more fallback behavior Signed-off-by: SchrodingerZhu * fix random device fd Signed-off-by: SchrodingerZhu * special handling for EAGAIN Signed-off-by: SchrodingerZhu --- CMakeLists.txt | 6 +-- src/pal/pal_linux.h | 125 ++++++++++++++++++++++++++++++++++++++++++-- src/pal/pal_posix.h | 3 -- 3 files changed, 123 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a3c138890..872ca6f33 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -89,7 +89,7 @@ CHECK_CXX_SOURCE_COMPILES(" # include #endif #if __has_include() -#include +# include #endif int main() { int entropy = 0; @@ -97,7 +97,6 @@ int main() { return res; } " SNMALLOC_PLATFORM_HAS_GETENTROPY) - # Provide as function so other projects can reuse # FIXME: This modifies some variables that may or may not be the ones that # provide flags and so is broken by design. It should be removed once Verona @@ -123,7 +122,7 @@ endfunction() function(clangformat_targets) # The clang-format tool is installed under a variety of different names. Try # to find a sensible one. Only look for versions 9 explicitly - we don't - # know whether our clang-format file will work with newer versions of the + # know whether our clang-format file will work with newer versions of the # tool. It does not work with older versions as AfterCaseLabel is not supported # in earlier versions. find_program(CLANG_FORMAT NAMES @@ -196,6 +195,7 @@ add_as_define(USE_SNMALLOC_STATS) add_as_define(SNMALLOC_QEMU_WORKAROUND) add_as_define(SNMALLOC_CI_BUILD) add_as_define(SNMALLOC_PLATFORM_HAS_GETENTROPY) + target_compile_definitions(snmalloc INTERFACE $<$:MALLOC_USABLE_SIZE_QUALIFIER=const>) # In debug and CI builds, link the backtrace library so that we can get stack diff --git a/src/pal/pal_linux.h b/src/pal/pal_linux.h index f597ff110..438a7469c 100644 --- a/src/pal/pal_linux.h +++ b/src/pal/pal_linux.h @@ -4,8 +4,10 @@ # include "../ds/bits.h" # include "pal_posix.h" +# include # include # include +# include extern "C" int puts(const char* str); @@ -18,12 +20,9 @@ namespace snmalloc * Bitmap of PalFeatures flags indicating the optional features that this * PAL supports. * - * Linux does not support any features other than those in a generic POSIX - * platform. This field is declared explicitly to remind anyone who - * extends this PAL that they may need to extend the set of advertised - * features. + * We always make sure that linux has entropy support. */ - static constexpr uint64_t pal_features = PALPOSIX::pal_features; + static constexpr uint64_t pal_features = PALPOSIX::pal_features | Entropy; static constexpr size_t page_size = Aal::aal_name == PowerPC ? 0x10000 : PALPOSIX::page_size; @@ -85,6 +84,122 @@ namespace snmalloc madvise(p, size, MADV_FREE); } } + + static uint64_t get_entropy64() + { + // TODO: If the system call fails then the POSIX PAL calls libc + // functions that can require malloc, which may result in deadlock. + + // SYS_getrandom API stablized since 3.17. + // This fallback implementation is to aid some environments + // where SYS_getrandom is provided in kernel but the libc + // is not providing getentropy interface. + + union + { + uint64_t result; + char buffer[sizeof(uint64_t)]; + }; + ssize_t ret; + + // give a try to SYS_getrandom +# ifdef SYS_getrandom + static std::atomic_bool syscall_not_working = false; + // Relaxed ordering should be fine here. This function will be called + // during early initialisation, which will examine the availability in a + // protected routine. + if (false == syscall_not_working.load(std::memory_order_relaxed)) + { + auto current = std::begin(buffer); + auto target = std::end(buffer); + while (auto length = target - current) + { + // Reading data via syscall from system entropy pool. + // According to both MUSL and GLIBC implementation, getentropy uses + // /dev/urandom (blocking API). + // + // The third argument here indicates: + // 1. `GRND_RANDOM` bit is not set, so the source of entropy will be + // `urandom`. + // 2. `GRND_NONBLOCK` bit is set. Since we are reading from + // `urandom`, this means if the entropy pool is + // not initialised, we will get a EAGAIN. + ret = syscall(SYS_getrandom, current, length, GRND_NONBLOCK); + // check whether are interrupt by a signal + if (SNMALLOC_UNLIKELY(ret < 0)) + { + if (SNMALLOC_UNLIKELY(errno == EAGAIN)) + { + // the system is going through early initialisation: at this stage + // it is very likely that snmalloc is being used in some system + // programs and we do not want to block it. + return reinterpret_cast(&result) ^ + reinterpret_cast(&error); + } + if (errno != EINTR) + { + break; + } + } + else + { + current += ret; + } + } + if (SNMALLOC_UNLIKELY(target != current)) + { + // in this routine, the only possible situations should be ENOSYS + // or EPERM (forbidden by seccomp, for example). + SNMALLOC_ASSERT(errno == ENOSYS || errno == EPERM); + syscall_not_working.store(true, std::memory_order_relaxed); + } + else + { + return result; + } + } +# endif + + // Syscall is not working. + // In this case, it is not a good idea to fallback to std::random_device: + // 1. it may want to use malloc to create a buffer, which causes + // reentrancy problem during initialisation routine. + // 2. some implementations also require libstdc++ to be linked since + // its APIs are not exception-free. + int flags = O_RDONLY; +# if defined(O_CLOEXEC) + flags |= O_CLOEXEC; +# endif + auto fd = open("/dev/urandom", flags, 0); + if (fd > 0) + { + auto current = std::begin(buffer); + auto target = std::end(buffer); + while (auto length = static_cast(target - current)) + { + ret = read(fd, current, length); + if (ret <= 0) + { + if (errno != EAGAIN && errno != EINTR) + { + break; + } + } + else + { + current += ret; + } + } + ret = close(fd); + SNMALLOC_ASSERT(0 == ret); + if (SNMALLOC_LIKELY(target == current)) + { + return result; + } + } + + error("Failed to get system randomness"); + } }; } // namespace snmalloc #endif diff --git a/src/pal/pal_posix.h b/src/pal/pal_posix.h index e75020da0..4c8dff208 100644 --- a/src/pal/pal_posix.h +++ b/src/pal/pal_posix.h @@ -19,9 +19,6 @@ #if __has_include() # include #endif -#if __has_include() -# include -#endif extern "C" int puts(const char* str); From 7f3642e05c6b59aa972ef3b2db3ac68d5475c287 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 2 Dec 2021 10:39:44 +0000 Subject: [PATCH 167/302] Change external thread_alloc example. (#424) On Open Enclave having the `local_alloc` directly in thread-local storage was causing a crash. This changes the `local_alloc` to be indirected, and thus puts less pressure on the thread-local storage. The test also has deals with how to allocate before a thread-local storage has been established. --- .../thread_alloc_external.cc | 37 +++++++++++++++++-- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/src/test/func/thread_alloc_external/thread_alloc_external.cc b/src/test/func/thread_alloc_external/thread_alloc_external.cc index 832d400bd..bc0952e2e 100644 --- a/src/test/func/thread_alloc_external/thread_alloc_external.cc +++ b/src/test/func/thread_alloc_external/thread_alloc_external.cc @@ -20,19 +20,48 @@ using namespace snmalloc; class ThreadAllocExternal { public: - static Alloc& get() + static Alloc*& get_inner() { - static thread_local Alloc alloc; + static thread_local Alloc* alloc; return alloc; } + + static Alloc& get() + { + return *get_inner(); + } }; #include +void allocator_thread_init(void) +{ + void* aptr; + { + // Create bootstrap allocator + auto a = snmalloc::ScopedAllocator(); + // Create storage for the thread-local allocator + aptr = a->alloc(sizeof(snmalloc::Alloc)); + } + // Initialize the thread-local allocator + ThreadAllocExternal::get_inner() = new (aptr) snmalloc::Alloc(); + ThreadAllocExternal::get().init(); +} + +void allocator_thread_cleanup(void) +{ + // Teardown the thread-local allocator + ThreadAllocExternal::get().teardown(); + // Need a bootstrap allocator to deallocate the thread-local allocator + auto a = snmalloc::ScopedAllocator(); + // Deallocate the storage for the thread local allocator + a->dealloc(ThreadAllocExternal::get_inner()); +} + int main() { setup(); - ThreadAlloc::get().init(); + allocator_thread_init(); auto& a = ThreadAlloc::get(); @@ -55,4 +84,6 @@ int main() a2->dealloc(r1); } + + allocator_thread_cleanup(); } From 360efa21231fe8c3f0163ac57940960c8096d402 Mon Sep 17 00:00:00 2001 From: David CARLIER Date: Thu, 2 Dec 2021 14:49:32 +0000 Subject: [PATCH 168/302] export netbsd's reallocarr proposal. (#433) * export netbsd's reallocarr proposal. acts subtly differently from reallocarray, returns an error code and first argument as receiver. * not export by default * ci tests * apply suggestions * doc addition * Apply suggestions from code review Co-authored-by: Matthew Parkinson --- CMakeLists.txt | 8 +++ docs/BUILDING.md | 13 +++++ src/override/malloc.cc | 41 ++++++++++++- src/test/func/malloc/malloc.cc | 103 +++++++++++++++++++++++++++++++++ 4 files changed, 164 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 872ca6f33..354d9be6f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,6 +16,8 @@ option(USE_SNMALLOC_STATS "Track allocation stats" OFF) option(SNMALLOC_CI_BUILD "Disable features not sensible for CI" OFF) option(SNMALLOC_QEMU_WORKAROUND "Disable using madvise(DONT_NEED) to zero memory on Linux" Off) option(SNMALLOC_USE_CXX17 "Build as C++17 for legacy support." OFF) +option(SNMALLOC_NO_REALLOCARRAY "Build without reallocarray exported" ON) +option(SNMALLOC_NO_REALLOCARR "Build without reallocarr exported" ON) # Options that apply only if we're not building the header-only library cmake_dependent_option(SNMALLOC_RUST_SUPPORT "Build static library for rust" OFF "NOT SNMALLOC_HEADER_ONLY_LIBRARY" OFF) cmake_dependent_option(SNMALLOC_STATIC_LIBRARY "Build static libraries" ON "NOT SNMALLOC_HEADER_ONLY_LIBRARY" OFF) @@ -195,6 +197,12 @@ add_as_define(USE_SNMALLOC_STATS) add_as_define(SNMALLOC_QEMU_WORKAROUND) add_as_define(SNMALLOC_CI_BUILD) add_as_define(SNMALLOC_PLATFORM_HAS_GETENTROPY) +if (SNMALLOC_NO_REALLOCARRAY) + add_as_define(SNMALLOC_NO_REALLOCARRAY) +endif() +if (SNMALLOC_NO_REALLOCARR) + add_as_define(SNMALLOC_NO_REALLOCARR) +endif() target_compile_definitions(snmalloc INTERFACE $<$:MALLOC_USABLE_SIZE_QUALIFIER=const>) diff --git a/docs/BUILDING.md b/docs/BUILDING.md index bad4b57a7..849a4c839 100644 --- a/docs/BUILDING.md +++ b/docs/BUILDING.md @@ -113,3 +113,16 @@ You will also need to compile the relevant parts of snmalloc itself. Create a ne #include "snmalloc/src/override/malloc.cc" #include "snmalloc/src/override/new.cc" ``` + +To enable the `reallocarray` symbol export, this can be added to your cmake command line. + +``` +-DSNMALLOC_NO_REALLOCARRAY=OFF +``` + +likewise for `reallocarr`. + +``` +-DSNMALLOC_NO_REALLOCARR=OFF +``` + diff --git a/src/override/malloc.cc b/src/override/malloc.cc index 658e2badf..f40b2d89d 100644 --- a/src/override/malloc.cc +++ b/src/override/malloc.cc @@ -101,7 +101,7 @@ extern "C" return p; } -#if !defined(__FreeBSD__) && !defined(__OpenBSD__) +#if !defined(SNMALLOC_NO_REALLOCARRAY) SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(reallocarray)(void* ptr, size_t nmemb, size_t size) { @@ -116,6 +116,45 @@ extern "C" } #endif +#if !defined(SNMALLOC_NO_REALLOCARR) + SNMALLOC_EXPORT int + SNMALLOC_NAME_MANGLE(reallocarr)(void* ptr_, size_t nmemb, size_t size) + { + int err = errno; + auto& a = ThreadAlloc::get(); + bool overflow = false; + size_t sz = bits::umul(size, nmemb, overflow); + if (sz == 0) + { + errno = err; + return 0; + } + if (overflow) + { + errno = err; + return EOVERFLOW; + } + + void** ptr = reinterpret_cast(ptr_); + void* p = a.alloc(sz); + if (p == nullptr) + { + errno = ENOMEM; + return ENOMEM; + } + + sz = bits::min(sz, a.alloc_size(*ptr)); + // Guard memcpy as GCC is assuming not nullptr for ptr after the memcpy + // otherwise. + if (sz != 0) + memcpy(p, *ptr, sz); + errno = err; + a.dealloc(*ptr); + *ptr = p; + return 0; + } +#endif + SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(memalign)(size_t alignment, size_t size) { diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index 80667f32e..a672a25a1 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -2,6 +2,8 @@ #include #define SNMALLOC_NAME_MANGLE(a) our_##a +#undef SNMALLOC_NO_REALLOCARRAY +#undef SNMALLOC_NO_REALLOCARR #include "../../../override/malloc.cc" using namespace snmalloc; @@ -138,6 +140,60 @@ void test_memalign(size_t size, size_t align, int err, bool null) check_result(size, align, p, err, null); } +void test_reallocarray(void* p, size_t nmemb, size_t size, int err, bool null) +{ + size_t old_size = 0; + size_t tsize = nmemb * size; + if (p != nullptr) + old_size = our_malloc_usable_size(p); + + printf("reallocarray(%p(%zu), %zu)\n", p, old_size, tsize); + errno = SUCCESS; + auto new_p = our_reallocarray(p, nmemb, size); + if (new_p == nullptr && tsize != 0) + our_free(p); + check_result(tsize, 1, new_p, err, null); +} + +void test_reallocarr( + size_t size_old, size_t nmemb, size_t size, int err, bool null) +{ + void* p = nullptr; + + if (size_old != (size_t)~0) + p = our_malloc(size_old); + errno = SUCCESS; + int r = our_reallocarr(&p, nmemb, size); + if (r != err) + { + printf("reallocarr failed! expected %d got %d\n", err, r); + abort(); + } + + printf("reallocarr(%p(%zu), %zu)\n", p, nmemb, size); + check_result(nmemb * size, 1, p, err, null); + p = our_malloc(size); + if (!p) + { + return; + } + for (size_t i = 1; i < size; i++) + static_cast(p)[i] = 1; + our_reallocarr(&p, nmemb, size); + if (r != SUCCESS) + our_free(p); + + for (size_t i = 1; i < size; i++) + { + if (static_cast(p)[i] != 1) + { + printf("data consistency failed! at %zu", i); + abort(); + } + } + our_free(p); +} + int main(int argc, char** argv) { UNUSED(argc); @@ -229,6 +285,53 @@ int main(int argc, char** argv) test_posix_memalign(0, align + 1, EINVAL, true); } + test_reallocarray(nullptr, 1, 0, SUCCESS, false); + for (smallsizeclass_t sc = 0; sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) + { + const size_t size = bits::one_at_bit(sc); + test_reallocarray(our_malloc(size), 1, size, SUCCESS, false); + test_reallocarray(our_malloc(size), 1, 0, SUCCESS, false); + test_reallocarray(nullptr, 1, size, SUCCESS, false); + test_reallocarray(our_malloc(size), 1, ((size_t)-1) / 2, ENOMEM, true); + for (smallsizeclass_t sc2 = 0; sc2 < (MAX_SMALL_SIZECLASS_BITS + 4); sc2++) + { + const size_t size2 = bits::one_at_bit(sc2); + test_reallocarray(our_malloc(size), 1, size2, SUCCESS, false); + test_reallocarray(our_malloc(size + 1), 1, size2, SUCCESS, false); + } + } + + test_reallocarr((size_t)~0, 1, 0, SUCCESS, false); + test_reallocarr((size_t)~0, 1, 16, SUCCESS, false); + + for (smallsizeclass_t sc = 0; sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) + { + const size_t size = bits::one_at_bit(sc); + test_reallocarr(size, 1, size, SUCCESS, false); + test_reallocarr(size, 1, 0, SUCCESS, false); + test_reallocarr(size, 2, size, SUCCESS, false); + void* p = our_malloc(size); + if (p == nullptr) + { + printf("realloc alloc failed with %zu\n", size); + abort(); + } + int r = our_reallocarr(&p, 1, ((size_t)-1) / 2); + if (r != ENOMEM) + { + printf("expected failure on allocation\n"); + abort(); + } + our_free(p); + + for (smallsizeclass_t sc2 = 0; sc2 < (MAX_SMALL_SIZECLASS_BITS + 4); sc2++) + { + const size_t size2 = bits::one_at_bit(sc2); + printf("size1: %zu, size2:%zu\n", size, size2); + test_reallocarr(size, 1, size2, SUCCESS, false); + } + } + if (our_malloc_usable_size(nullptr) != 0) { printf("malloc_usable_size(nullptr) should be zero"); From d0df2d028a45131e5b85dce873c0bcc572f6a57d Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 13 Dec 2021 14:36:12 +0000 Subject: [PATCH 169/302] Fix CI branch name. (#442) --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 88065d914..264a19fcd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,9 +4,9 @@ name: snmalloc CI on: # Triggers the workflow on push or pull request events but only for the master branch push: - branches: [ master ] + branches: [ main ] pull_request: - branches: [ master, snmalloc2 ] + branches: [ main, snmalloc1 ] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: From 37d2ac42af2910c87da0fd635eef8f5c8347ab9a Mon Sep 17 00:00:00 2001 From: Schrodinger ZHU Yifan Date: Wed, 15 Dec 2021 22:01:34 +0800 Subject: [PATCH 170/302] use ::write and ::fsync in error path (#443) * use ::write and ::fsync in error path Signed-off-by: SchrodingerZhu * mark return value unused Signed-off-by: SchrodingerZhu * adjust fsync positions Signed-off-by: SchrodingerZhu --- src/pal/pal_posix.h | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/pal/pal_posix.h b/src/pal/pal_posix.h index 4c8dff208..6f12b0437 100644 --- a/src/pal/pal_posix.h +++ b/src/pal/pal_posix.h @@ -151,10 +151,9 @@ namespace snmalloc constexpr int SIZE = 1024; void* buffer[SIZE]; auto nptrs = backtrace(buffer, SIZE); - fflush(stdout); backtrace_symbols_fd(buffer, nptrs, STDOUT_FILENO); - puts(""); - fflush(stdout); + UNUSED(write(STDOUT_FILENO, "\n", 1)); + UNUSED(fsync(STDOUT_FILENO)); #endif } @@ -163,7 +162,14 @@ namespace snmalloc */ [[noreturn]] static void error(const char* const str) noexcept { - puts(str); + /// by this part, the allocator is failed; so we cannot assume + /// subsequent allocation will work. + /// @attention: since the program is failing, we do not guarantee that + /// previous bytes in stdout will be flushed + UNUSED(write(STDOUT_FILENO, "\n", 1)); + UNUSED(write(STDOUT_FILENO, str, strlen(str))); + UNUSED(write(STDOUT_FILENO, "\n", 1)); + UNUSED(fsync(STDOUT_FILENO)); print_stack_trace(); abort(); } From f398545154c9889123b3c66cf51ca56eef0154dd Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 14 Dec 2021 11:48:45 +0000 Subject: [PATCH 171/302] Fix memalign usage - Alignment should be above sizeof(void*) - Don't call with very large sizes --- src/mem/external_alloc.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/mem/external_alloc.h b/src/mem/external_alloc.h index ae35870fd..13eb2bcec 100644 --- a/src/mem/external_alloc.h +++ b/src/mem/external_alloc.h @@ -52,6 +52,16 @@ namespace snmalloc::external_alloc { inline void* aligned_alloc(size_t alignment, size_t size) { + // TSAN complains if allocation is large than this. + if constexpr (bits::BITS == 64) + { + if (size >= 0x10000000000) + return nullptr; + } + + if (alignment < sizeof(void*)) + alignment = sizeof(void*); + void* result; if (posix_memalign(&result, alignment, size) != 0) { From 7768765fe8928976ab352e71ebf5a53db2bdc95e Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 14 Dec 2021 11:49:39 +0000 Subject: [PATCH 172/302] Fixed locking around notification Deduplication locking had test_and_set incorrect direction. --- src/pal/pal_ds.h | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/pal/pal_ds.h b/src/pal/pal_ds.h index 852d83cbd..543fb8a1c 100644 --- a/src/pal/pal_ds.h +++ b/src/pal/pal_ds.h @@ -146,18 +146,17 @@ namespace snmalloc // Depulicate calls into here, and make single threaded. if (lock.test_and_set()) - { - timers.apply_all([time_ms](PalTimerObject* curr) { - if ( - (curr->last_run == 0) || - ((time_ms - curr->last_run) > curr->repeat)) - { - curr->last_run = time_ms; - curr->pal_notify(curr); - } - }); - lock.clear(); - } + return; + + timers.apply_all([time_ms](PalTimerObject* curr) { + if ( + (curr->last_run == 0) || ((time_ms - curr->last_run) > curr->repeat)) + { + curr->last_run = time_ms; + curr->pal_notify(curr); + } + }); + lock.clear(); } }; } // namespace snmalloc From 0fd5c37563d7f3dd84b49769f496f5144cd92007 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 15 Dec 2021 11:46:39 +0000 Subject: [PATCH 173/302] Change MPSCQ to use acquire TSAN complained that there was a race that after some thoughts appears to be due to this exchange needing to be an `acquire`. I still wonder if the data dependence that is threaded through the exchange induces enough order. --- src/mem/remoteallocator.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index 6e33b58b2..b9b1ff693 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -106,9 +106,12 @@ namespace snmalloc invariant(); freelist::Object::atomic_store_null(last, key); - // exchange needs to be a release, so nullptr in next is visible. + // Exchange needs to be acq_rel. + // * It needs to be a release, so nullptr in next is visible. + // * Needs to be acquire, so linking into the list does not race with + // the other threads nullptr init of the next field. freelist::QueuePtr prev = - back.exchange(capptr_rewild(last), std::memory_order_release); + back.exchange(capptr_rewild(last), std::memory_order_acq_rel); freelist::Object::atomic_store_next(domesticate_head(prev), first, key); } From 4f2d3ebf33645572a696cb4860e528266ecb3f92 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 15 Dec 2021 11:46:55 +0000 Subject: [PATCH 174/302] Enable sanitizers in CI. --- .github/workflows/main.yml | 4 ++++ CMakeLists.txt | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 264a19fcd..5ed35f7c6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -78,6 +78,10 @@ jobs: dependencies: "sudo apt install ninja-build" extra-cmake-flags: "-DCMAKE_CXX_COMPILER=clang++-10 -DCMAKE_C_COMPILER=clang-10 -DCMAKE_CXX_FLAGS=-stdlib=libstdc++" build-only: yes + - os: "ubuntu-latest" + variant: Clang 10 libc++ (TSan + UBSan) + dependencies: "sudo apt install ninja-build" + extra-cmake-flags: "-DCMAKE_CXX_COMPILER=clang++-10 -DCMAKE_C_COMPILER=clang-10 -DCMAKE_CXX_FLAGS=-stdlib=libc++ -DSNMALLOC_SANITIZER=undefined,thread" # Don't abort runners if a single one fails fail-fast: false runs-on: ${{ matrix.os }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 354d9be6f..b23d067ee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,6 +47,11 @@ if (NOT SNMALLOC_CLEANUP STREQUAL CXX11_DESTRUCTORS) set(CMAKE_CXX_IMPLICIT_LINK_LIBRARIES "") endif() +set(SNMALLOC_SANITIZER "" CACHE STRING "Use sanitizer type (undefined|thread|...)") +if (SNMALLOC_SANITIZER) + message(STATUS "Using sanitizer=${SNMALLOC_SANITIZER}") +endif() + # If CheckLinkerFlag doesn't exist then provide a dummy implementation that # always fails. The fallback can be removed when we move to CMake 3.18 as the # baseline. @@ -374,6 +379,12 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) set(TESTNAME "${TEST_CATEGORY}-${TEST}-${FLAVOUR}") add_executable(${TESTNAME} ${SRC}) + + if(SNMALLOC_SANITIZER) + target_compile_options(${TESTNAME} PRIVATE -g -fsanitize=${SNMALLOC_SANITIZER} -fno-omit-frame-pointer) + target_link_libraries(${TESTNAME} -fsanitize=${SNMALLOC_SANITIZER}) + endif() + add_warning_flags(${TESTNAME}) if (${FLAVOUR} STREQUAL "malloc") From 9b60e8256db4018b180d40f1b3d24e649cc3e9db Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Thu, 11 Nov 2021 14:42:51 +0000 Subject: [PATCH 175/302] CHERI: update StrictProvenance.md for new world order --- docs/StrictProvenance.md | 193 ++++++++++++++++++--------------------- 1 file changed, 90 insertions(+), 103 deletions(-) diff --git a/docs/StrictProvenance.md b/docs/StrictProvenance.md index b2b6317cd..2c2fa0676 100644 --- a/docs/StrictProvenance.md +++ b/docs/StrictProvenance.md @@ -1,71 +1,78 @@ # StrictProvenance Architectures -To aid support of novel architectures, such as CHERI, which explicitly track pointer *provenance* and *bounds*, `snmalloc` makes heavy use of a `CapPtr` wrapper type around `T*` values. -You can view the annotation `B` on a `CapPtr` as characterising the set of operations that are supported on this pointer, such as +To aid code auditing and support of novel architectures, such as CHERI, which explicitly track pointer *provenance* and *bounds*, `snmalloc` makes heavy use of a `CapPtr` wrapper type around `T*` values. +You can think of the annotation `B` on a `CapPtr` as capturing something about the role of the pointer, e.g.: -* address arithmetic within a certain range (e.g, a `Superslab` chunk) +* A pointer to a whole chunk or slab, derived from an internal `void*`. +* A pointer to a particular allocation, destined for the user program +* A putative pointer returned from the user program + +You can also view the annotation `B` as characterising the set of operations that are supported on this pointer, such as + +* nothing (because we haven't checked that it's actually a valid pointer) +* memory access within a certain range (e.g, a chunk or an allocation) * requesting manipulation of the virtual memory mappings -Most architectures and platforms cannot enforce these restrictions, but CHERI enables software to constrain its future use of particular pointers and `snmalloc` imposes strong constraints on its *client(s)* use of memory it manages. +Most architectures and platforms cannot enforce these restrictions outside of static constraints, but CHERI enables software to constrain its future use of particular pointers and `snmalloc` imposes strong constraints on its *client(s)* use of memory it manages. The remainder of this document... * gives a "quick start" guide, * provides a summary of the constraints imposed on clients, -* motivates and introduces the internal `ArenaMap` structure and the `capptr_amplify` function, and -* describes the `StrictProvenance` `capptr_*` functions provided by the Architecture Abstraction Layer (AAL) and Platform Abstraction Layer (PAL). +* describes the `StrictProvenance` `capptr_*` functions provided by `ds/ptrwrap.h`, the Architecture Abstraction Layer (AAL), and the Platform Abstraction Layer (PAL). -## Preface +## Limitations The `CapPtr` and `capptr_*` primitives and derived functions are intended to guide developers in useful directions; they are not security mechanisms in and of themselves. -For non-CHERI architectures, the whole edifice crumbles in the face of an overzealous `reinterpret_cast<>` or `unsafe_capptr` member access. -On CHERI, these are likely to elicit capability violations, but may not if all subsequent access happen to be within bounds. +For non-CHERI architectures, the whole edifice crumbles in the face of an overzealous `reinterpret_cast<>` or `unsafe_*ptr` call. +On CHERI, these are likely to elicit capability violations, but may not if all subsequent access happen to be within the architecturally-enforced bounds. ## Quick Start Guide -### How do I safely get an ordinary pointer to reveal to the client? - -If you are... +### What will I see? -* Adding an interface like `external_pointer`, and so you have a `CapPtr`, `e`, whose bounds you want to *inherit* when revealing some other `CapPtr` `p`, use `capptr_rebound(e, p)` to obtain another `CapPtr` with address from `p`, then go to the last step here. +In practice, `CapPtr` and the details of `B` overtly show themselves only in primitive operations or when polymorphism across `B` is required. +(Or, sadly, when looking at compilation errors or demangled names in a debugger.) +All the concrete forms we have found useful have layers of aliasing to keep the verbosity down: `capptr::B` is a `CapPtr` with `capptr::bounds::B` itself an alias for a `capptr::bound<...>` type-level object. +This trend of aliasing continues into higher-level abstractions, such as the freelist, wherein one finds, for example, `freelist::HeadPtr`, which expands to a type involving several `CapPtr`s and associated annotations. -* Otherwise, if your object is... - - * an entire `SUPERSLAB_SIZE` chunk or bigger, you should have in hand a `CapPtr` from the large allocator. Use `capptr_export` to make a `CapPtr`, then use `capptr_chunk_is_alloc` to convert that to a `CapPtr`, and then proceed. (If, instead, you find yourself holding a `CapPtr`, use `capptr_chunk_from_chunkd` first.) +### How do I safely get an ordinary pointer to reveal to the client? - * of size `sz` and smaller than such a chunk, +Neglecting platform-specific details of getting authority to address space and associating memory in the first place, almost all memory manipulated by `snmalloc` comes from the `AddressSpaceManager`. +Its `reserve(size)` method returns a `capptr::Chunk`; this pointer conveys full authority to the region of `size` at which it points. +To derive a pointer that is suitable for client use, we must - * and have a `CapPtr p` in hand, use `Aal::capptr_bound(p, sz)` to get a `CapPtr`, and then proceed. +* further spatially refine the pointer: adjust its offset with `pointer_offset` and use `capptr_bound` and +* shed address space control: use `PAL::capptr_to_user_address_control()` to convert `AllocFull` to `Alloc`. - * an have a `CapPtr p`, `CapPtr p`, or `CapPtr p` in hand, use `Aal::capptr_bound(p, sz)` to get a `CapPtr`, and then proceed. +If no additional spatial refinement is required, because the entire chunk is intended for client use, -* If the above steps left you with a `CapPtr`, apply any platform constraints for its export with `Pal::capptr_export(p)` to obtain a `CapPtr`. +* shed address space control: use `PAL::capptr_to_user_address_control()` to obtain a `ChunkUser`-bounded pointer, then +* use `capptr_chunk_is_alloc` to capture intent, converting `ChunkUser` to `Alloc` without architectural consequence. -* Use `capptr_reveal` to safely convert a `CapPtr` to a `T*` for the client. +At this point, we hold a `capptr::Alloc`; use `capptr_reveal()` to obtain the underlying `T*`. ### How do I safely ingest an ordinary pointer from the client? -For all its majesty, `CapPtr`'s coverage is merely an impediment to, rather than a complete defense against, malicious client behavior even on CHERI-enabled architectures. +First, we must admit that, for all its majesty, `CapPtr`'s coverage is merely an impediment to, rather than a complete defense against, malicious client behavior even on CHERI-enabled architectures. Further protection is an open research project at MSR. -Nevertheless, if adding a new kind of deallocation, we suggest following the existing flows: +Nevertheless, if adding a new kind of deallocation, we suggest following the existing flows when given a `void* p_raw` from the client: -* Begin by wrapping it with `CapPtr` and avoid using the raw `T*` thereafter. +* Begin by calling `p_wild = capptr_from_client(p_raw)` to annotate it as `AllocWild` and avoid using the raw form thereafter. -* An `CapPtr` can be obtained using `large_allocator.capptr_amplify()`. -Note that this pointer and its progeny are *unsafe* beyond merely having elevated authority: it is possible to construct and dereference pointers with types that do not match memory, resulting in **undefined behavior**. +* Check the `Wild` pointer for domestication with `p_tame = capptr_domesticate(state_ptr, p_wild)`; `p_tame` will be a `capptr::Alloc` and will alias `p_wild` or will be `nullptr`. + At this point, we have no more use for `p_wild`. -* Derive the `ChunkMapSuperslabKind` associated with the putative pointer from the client, by reading the `ChunkMap`. -In some flows, the client will have made a *claim* as to the size (class) of the object which may be tentatively used, but should be validated (unless the client is trusted). +* We may now probe the Pagemap; either `p_tame` is a pointer we have given out or `nullptr`, or this access may trap (especially on platforms where domestication is just a rubber stamp). + This will give us access to the associated `MetaEntry` and, if necessary, a `Chunk`-bounded pointer to the entire backing region. -* Based on the above, for non-Large objects, `::get()` the appropriate header structure (`Superslab` or `Mediumslab`). +* If desired, we can now validate other attributes of the provided capability, including its length, base, and permissions. +In fact, we can even go further and *reconstruct* the capability we would have given out for the indicated allocation, allowing for exact comparison. Eventually we would like to reliably detect references to free objects as part of these flows, especially as frees can change the type of metadata found at the head of a chunk. When that is possible, we will add guidance that only reads of non-pointer scalar types are to be performed until after such tests have confirmed the object's liveness. Until then, we have stochastic defenses (e.g., `encode` in `src/mem/freelist.h`) later on. -As alluded to above, `capptr_rebound` can be used to ensure that pointers manipulated within `snmalloc` inherit bounds from client-provided pointers. -In the future, these derived pointers will inherit *temporal bounds* as well as the spatial ones described herein. - ### What happened to my cast operators? Because `CapPtr` are not the kinds of pointers C++ expects to manipulate, `static_cast<>` and `reinterpret_cast<>` are not applicable. @@ -75,10 +82,12 @@ Please use the first viable option from this list, reserving `reinterpret_cast` ## StrictProvenance in More Detail Tracking pointer *provenance* and *bounds* enables software to constrain uses of *particular pointers* in ways that are not available with traditional protection mechanisms. -For example, while code my *have* a pointer that spans its entire C stack, it may construct a pointer that authorizes access only to a particular stack allocation (e.g., a buffer) and use this latter pointer while copying data. +For example, while code may *have* a pointer that spans its entire C stack, it may construct a pointer that authorizes access only to a particular stack allocation (e.g., a buffer) and use this latter pointer while copying data. Even if an attacker is able to control the length of the copy, the bounds imposed upon pointers involved can ensure that an overflow is impossible. -(Of course, if the attacker can influence both the *bounds* and the copy length, an overflow may still be possible; in practice, however, the two concerns are often sufficiently separated.) +(On the other hand, if the attacker can influence both the *bounds* and the copy length, an overflow may still be possible; in practice, however, the two concerns are often sufficiently separated.) For `malloc()` in particular, it is enormously beneficial to be able to impose bounds on returned pointers: it becomes impossible for allocator clients to use a pointer from `malloc()` to access adjacent allocations! +(*Temporal* concerns still apply, in that live allocations can overlap prior, now-dead allocations. +Stochastic defenses are employed within `snmalloc` and deterministic defenses are ongoing research at MSR.) Borrowing terminology from CHERI, we speak of the **authority** (to a subset of the address space) held by a pointer and will justify actions in terms of this authority. While many kinds of authority can be envisioned, herein we will mean either @@ -93,7 +102,7 @@ Dually, given two pointers, one with a subset of the other's authority, we may * `snmalloc` ensures that returned pointers are bounded to no more than the slab entry used to back each allocation. It may be useful, mostly for debugging, to more precisely bound returned pointers to the actual allocation size,[^bounds-precision] but this is not required for security. -The pointers returned from `alloc()` will be stripped of their *vmmap* authority, if supported by the platform, ensuring that clients cannot manipulate the page mapping underlying `snmalloc`'s address space. +The pointers returned from `alloc()` will also be stripped of their *vmmap* authority, if supported by the platform, ensuring that clients cannot manipulate the page mapping underlying `snmalloc`'s address space. `realloc()`-ation has several policies that may be sensible. We choose a fairly simple one for the moment: resizing in ways that do not change the backing allocation's `snmalloc` size class are left in place, while any change to the size class triggers an allocate-copy-deallocate sequence. @@ -101,34 +110,20 @@ Even if `realloc()` leaves the object in place, the returned pointer should have (Notably, this policy is compatible with the existence of size-parameterized deallocation functions: the result of `realloc()` is always associated with the size class corresponding to the requested size. By contrast, shrinking in place in ways that changed the size class would require tracking the largest size ever associated with the allocation.) -## Impact of Constraints On Deallocation, or Introducing the ArenaMap +## Impact of Constraints On Deallocation -Strict provenance and bounded returns from `alloc()` imply that we cannot expect things like +Previous editions of `snmalloc` stored metadata at "superslab" boundaries in the address space and relied on address arithmetic to map from small allocations to their associated metadata. +These operations relied on being able to take pointers out of bounds, and so posed challenges for `StrictProvenance` architectures. +The current edition of `snmalloc` instead follows pointers (starting from TLS or global roots), using address arithmetic only to derive indicies into these metadata pointers. -```c++ -void dealloc(void *p) -{ - Superslab *super = Superslab::get(p); - ... super->foo ... -} -``` - -to work (using the existing `Superslab::get()` implementation). -Architecturally, `dealloc` is no different from any *allocator client* code and `Superslab::get()` is merely some pointer math. -As such, `Superslab::get()` must either fail to construct its return value (e.g., by trapping) or construct a useless return value (e.g., one that traps on dereference). -To proceed, we must take advantage of the fact that `snmalloc` has separate authority to the memory underlying its allocations. - -Ultimately, all address space manipulated by `snmalloc` comes from its Platform's primitive allocator. -An **arena** is a region returned by that provider. -The `AddressSpaceManager` divides arenas into large allocations and manages their life cycles. -On `StrictProvenance` architectures, the ASM further maintains a map of all PAL-provided memory, called the `ArenaMap`, and uses this to implement `capptr_amplify`, copying the address of a low-authority pointer into a copy of the high-authority pointer provided by the PAL. -The resulting pointer can then be used much as on non-`StrictProvenance` architectures, with integer arithmetic being used to make it point anywhere within an arena. -`snmalloc`'s heap layouts ensure that metadata associated with any object are spread across globals and within the same arena as the object itself, and so, assuming access to globals as given, a single amplification suffices. +When the allocator client returns memory (or otherwise refers to an allocation), we will be careful to use the *lower bound* address, not the indicated address per se, for looking up the allocation. +The indicated address may be out of bounds, while `StrictProvenance` architectures should ensure that bounds are monotonically non-increasing, and so the lower bound will always be within the original allocation. ## Object Lookup `snmalloc` extends the traditional allocator interface with the `template void* external_pointer(void*)` family of functions, which generate additional pointers to live allocations. -To ensure that this function is not used as an amplification oracle, it must construct a return pointer with the same validity as its input even as it internally amplifies to access metadata; see `capptr_rebound`. +To ensure that this function is not used as an amplification oracle, it must construct a return pointer with the same validity as its input even as it internally accesses metadata. +We make `external_pointer` use `pointer_offset` on the user-provided pointer, ensuring that the result has no more authority than the client already held. XXX It may be worth requiring that the input pointer authorize the entire object? What are the desired security properties here? @@ -137,80 +132,74 @@ What are the desired security properties here? ## Design Overview -As mentioned, the `AddressSpaceManager` maintains an `ArenaMap`, a cache of pointers that span the entire heap managed by `snmalloc`. -To keep this cache small, we request very large swaths (GiB-scale on >48-bit ASes) of address space at a time, even if we only populate those regions very slowly. - -Within `snmalloc`, there are several data structures that hold free memory: - -* the `LargeAlloc` holds all regions too big to be managed by `MediumSlab`s +For the majority of operations, no `StrictProvenance`-specific reasoning, beyond applying bounds, need be entertained. +However, as regions of memory move into and out of an `AddressSpaceManagerCore` and `ChunkAllocator`, care must be taken to recover (and preserve) the internal, *vmmap*-authorizing pointers from the user's much more tightly bounded pointers. -* `MediumSlab`s hold free lists +We store these internal pointers inside metadata, at different locations for each state: -* `Slab`s hold free lists. +* For free chunks in `AddressSpaceManagerCore`s, the `next` pointers themselves will be internal pointers. + That is, the head of each list in the `AddressSpaceManagerCore` and the (coerced) next pointers in each `MetaEntry` will be suitable for internal use. -* `Slab`s have associated "bump pointer" regions of address space not yet used (facilitating lazy construction of free lists) - -* `Alloc`s themselves also hold, per small size class, up to one free list and up to one bump pointer (so that the complexity of `Slab` manipulation is amortized across many allocations) - -* `Alloc`s have or point to `RemoteAllocator`s, which contain queues of `Remote` objects formed from deallocated memory. - -* `Alloc`s have `RemoteCaches` that also hold `Remote`s. +* Once outside the `AddressSpaceManager`, chunks have a `Metaslab` associated with them, and we can store internal pointers therein (in the `MetaCommon` `chunk` field). +Within each slab, there is one or more free list of objects. We take the position that free list entries should be suitable for return, i.e., with authority bounded to their backing slab entry. (However, the *contents* of free memory may be dangerous to expose to the user and require clearing prior to handing out.) -This means that allocation fast paths are unaffected by the requirement to bound return pointers, but that deallocation paths may need to amplify twice, once on receipt of the pointer from the application and again on receipt of the pointer from another `Allocator` through the `Remote` mechanism. ## Static Pointer Bound Taxonomy -At the moment, we introduce six possible annotations, though the taxonomy is imperfect: +We introduce a multi-dimensional space of bounds. The facets are `enum class`-es in `snmalloc::capptr::dimension`. + +* `Spatial` captures the intended spatial extent / role of the pointer: either `Alloc`-ation or `Chunk`. -* bounded only to an underlying arena without platform constraints, `CBArena`; -* bounded to a `SUPERSLAB_SIZE` or larger chunk without platform constraints, `CBChunk`; -* bounded to a `SUPERSLAB_SIZE` or larger chunk with platform constraints, `CBChunkE`; -* bounded *on debug builds* to a `SUPERSLAB_SIZE` or larger chunk without platform constraints, `CBChunkD`; -* bounded to an allocation but without platform constraints yet applied, `CBAlloc`; -* bounded to an allocation and with platform constraints, `CBAllocE`; +* `AddressSpaceControl` captures whether the pointer conveys control of its address space. -By "platform constraints" we mean, for example, CheriBSD's ability to remove the authority to manage the VM mappings underlying a pointer. -Clients of malloc have no business attempting to manage the backing pages. +* `Wildness` captures whether the pointer has been checked to belong to this allocator. -In practice, we use the pair of the type `T` and the bounds annotation for additional light-weight verification. -For example, we differentiate `CapPtr` from `CapPtr`, with the former being offset (if cache-friendly offsets are in effect) and the latter almost always pointing to the start of the object. -While it is possible to write code which subverts the annotation scheme, in general method signatures should provide the correct affordance. +These `dimension`s are composited using a `capptr::bound<>` type that we use as `B` in `CapPtr`. +This is enforced (loosely) using the `ConceptBound` C++20 concept. + +The namespace `snmalloc::capptr::bounds` contains particular points in the space of `capptr::bound<>` types: + +* bounded to a region of more than `MAX_SIZECLASS_SIZE` bytes with address space control, `Chunk`; +* bounded to a region of more than `MAX_SIZECLASS_SIZE` bytes without address space control, `ChunkUser`; +* bounded to a smaller region but with address space control, `AllocFull`; +* bounded to a smaller region and without address space control, `Alloc`; +* bounded to a smaller region, without address space control, and unverified, `AllocWild`. ## Primitive Architectural Operations Several new functions are introduced to AALs to capture primitives of the Architecture. -* `CapPtr capptr_bound(CapPtr a, size_t sz)` - spatially bounds the pointer `a` to have authority ranging only from its current target to its current target plus `sz` bytes (which must be within `a`'s authority). +* `CapPtr capptr_bound(CapPtr a, size_t sz)` spatially bounds the pointer `a` to have authority ranging only from its current target to its current target plus `sz` bytes (which must be within `a`'s authority). No imprecision in authority is permitted. - The `obounds` annotation is required to be either strictly higher authority than `CBAlloc` or `CBChunkE`, and the bounds annotations must obey `capptr_is_bounds_refinement`. + The bounds annotations must obey `capptr_is_spatial_refinement`. -* `CapPtr capptr_rebound(CapPtr a, CapPtr p)` is the *architectural primitive* enabling the software amplification mechanism. - It combines the authority of `a` and the current target of `p`. - The result may be safely dereferenced iff `a` authorizes access to `p`'s target. - The simplest sufficient (but not necessary) condition to ensure safety is that authority of `a` is a superset of the authority of `p` and `p` points within its authority. +Ultimately, all address space manipulated by `snmalloc` comes from its Platform's primitive allocator. +An **arena** is a region returned by that provider. +The `AddressSpaceManager` divides arenas into large allocations and manages their life cycles. +`snmalloc`'s (new, as of `snmalloc2`) heap layouts ensure that metadata associated with any object are reachable through globals, meaning no explicit amplification is required. ## Primitive Platform Operations -* `CapPtr capptr_export(CapPtr f)` applies any additional platform constraints required before handing permissions out to the client. +* `CapPtr capptr_to_user_address_control(CapPtr f)` sheds authority over the address space from the `CapPtr`, on platforms where that is possible. On CheriBSD, specifically, this strips the `VMMAP` software permission, ensuring that clients cannot have the kernel manipulate heap pages. +The annotation `Bout` is *computed* as a function of `Bin`. In future architectures, this is increasingly likely to be a no-op. -The annotation `BO` is *computed* as a function of `BI`, which must be `CBChunk` or `CBAlloc`. -## Constructed Operators +## Backend-Provided Operations -* `capptr_bound_chunkd` and `capptr_chunk_from_chunkd` manage the construction and elimination of `CapPtr` pointers. +* `CapPtr capptr_domesticate(Backend::LocalState *, CapPtr ptr)` allows the backend to test whether `ptr` is sensible, by some definition thereof. +The annotation `Bout` is *computed* as a function of `Bin`. +`Bin` is required to be `Wild`, and `Bout` is `Tame` but otherwise identical. -* `capptr_chunk_is_alloc` converts a `CapPtr` to a `CapPtr` unsafely; it is intended to ease auditing. +## Constructed Operators -* `capptr_reveal` converts a `CapPtr` to a `void*`. +* `capptr_chunk_is_alloc` converts a `capptr::ChunkUser` to a `capptr::Alloc` with no additional bounding; it is intended to ease auditing. -## Amplification +* `capptr_reveal` converts a `capptr::Alloc` to a `void*`, annotating where we mean to return a pointer to the user. -The `AddressSpaceManager` now exposes a method with signature `CapPtr capptr_amplify(CapPtr p)` which uses `capptr_rebound` to construct a pointer targeting `p`'s target but bearing the authority of the primordial allocation granule (as provided by the kernel) containing this address. -This pointer can be used to reach the `Allocslab` metadata associated with `p` (and a good bit more, besides!). +* `capptr_reveal_wild` converts a `capptr::AllocWild` to a `void*`, annotating where we mean to return a *wild* pointer to the user (in `external_pointer`, e.g., where the result is just an offset of the user's pointer). # Endnotes @@ -225,5 +214,3 @@ There have, in times past, been capability systems with architectural amplificat [^bounds-precision] `StrictProvenance` architectures have historically differed in the precision with which authority can be represented. Notably, it may not be possible to achieve byte-granular authority boundaries at every size scale. In the case of CHERI specifically, `snmalloc`'s size classes and its alignment policies are already much coarser than existing architectural requirements for representable authority on all existing implementations. - - From 13a4b0471e098543052ec37cbcae9788be5b55c5 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Thu, 11 Nov 2021 14:43:16 +0000 Subject: [PATCH 176/302] CHERI: address_space_core should bound more When reserving with leftover space, bound both the reservation and the residual pointer. This may be excessive? --- src/backend/address_space_core.h | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/backend/address_space_core.h b/src/backend/address_space_core.h index d050790ee..a4b1f19ce 100644 --- a/src/backend/address_space_core.h +++ b/src/backend/address_space_core.h @@ -232,7 +232,13 @@ namespace snmalloc size_t length_align_bits = (bits::BITS - 1) - bits::clz(length); size_t align_bits = bits::min(base_align_bits, length_align_bits); size_t align = bits::one_at_bit(align_bits); - auto b = base.as_static(); + + /* + * Now that we have found a maximally-aligned block, we can set bounds + * and be certain that we won't hit representation imprecision. + */ + auto b = + Aal::capptr_bound(base, align); check_block(b, align_bits); add_block(local_state, align_bits, b); @@ -289,7 +295,7 @@ namespace snmalloc * of the block is retained by the address space manager. * * This is useful for allowing the space required for alignment to be - * used, by smaller objects. + * used by smaller objects. */ template< SNMALLOC_CONCEPT(ConceptPAL) PAL, @@ -309,8 +315,14 @@ namespace snmalloc { if (rsize > size) { - add_range( - local_state, pointer_offset(res, size), rsize - size); + /* + * Set bounds on the allocation requested but leave the residual with + * wider bounds for the moment; we'll fix that above in add_range. + */ + size_t residual_size = rsize - size; + auto residual = pointer_offset(res, size); + res = Aal::capptr_bound(res, size); + add_range(local_state, residual, residual_size); } } return res; From 3a509d41f0a875d065a3e9eba6751e61750e1719 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Thu, 11 Nov 2021 14:44:36 +0000 Subject: [PATCH 177/302] CHERI: Avoid traps on nullptr The CHERI-RISC-V `CAndPerm` and `CSetBoundsExact` instructions trap on untagged inputs, so avoid passing `nullptr` to primitives that become those instructions. --- src/aal/aal_cheri.h | 40 ++++++++++++++++++++++++++++++++++++++++ src/pal/pal_freebsd.h | 7 +++++++ 2 files changed, 47 insertions(+) diff --git a/src/aal/aal_cheri.h b/src/aal/aal_cheri.h index 866e0ad14..1a7906e6b 100644 --- a/src/aal/aal_cheri.h +++ b/src/aal/aal_cheri.h @@ -21,6 +21,38 @@ namespace snmalloc static constexpr uint64_t aal_features = (Base::aal_features & ~IntegerPointers) | StrictProvenance; + enum AalCheriFeatures : uint64_t + { + /** + * This CHERI flavor traps if the capability input to a bounds-setting + * instruction has its tag clear, rather than just leaving the output + * untagged. + * + * For example, CHERI-RISC-V's CSetBoundsExact traps in contrast to + * Morello's SCBNDSE. + */ + SetBoundsTrapsUntagged = (1 << 0), + + /** + * This CHERI flavor traps if the capability input to a + * permissions-masking instruction has its tag clear, rather than just + * leaving the output untagged. + * + * For example, CHERI-RISC-V's CAndPerms traps in contrast to Morello's + * CLRPERM. + */ + AndPermsTrapsUntagged = (1 << 0), + }; + + /** + * Specify "features" of the particular CHERI machine we're running on. + */ + static constexpr uint64_t aal_cheri_features = + /* CHERI-RISC-V prefers to trap on untagged inputs. Morello does not. */ + (Base::aal_name == RISCV ? + SetBoundsTrapsUntagged | AndPermsTrapsUntagged : + 0); + /** * On CHERI-aware compilers, ptraddr_t is an integral type that is wide * enough to hold any address that may be contained within a memory @@ -45,6 +77,14 @@ namespace snmalloc "capptr_bound must preserve non-spatial CapPtr dimensions"); SNMALLOC_ASSERT(__builtin_cheri_tag_get(a.unsafe_ptr())); + if constexpr (aal_cheri_features & SetBoundsTrapsUntagged) + { + if (a == nullptr) + { + return nullptr; + } + } + void* pb = __builtin_cheri_bounds_set_exact(a.unsafe_ptr(), size); return CapPtr(static_cast(pb)); } diff --git a/src/pal/pal_freebsd.h b/src/pal/pal_freebsd.h index e17df290d..3cd2e4fae 100644 --- a/src/pal/pal_freebsd.h +++ b/src/pal/pal_freebsd.h @@ -57,6 +57,13 @@ namespace snmalloc static SNMALLOC_FAST_PATH CapPtr> capptr_to_user_address_control(CapPtr p) { + if constexpr (Aal::aal_cheri_features & Aal::AndPermsTrapsUntagged) + { + if (p == nullptr) + { + return nullptr; + } + } return CapPtr>( __builtin_cheri_perms_and( p.unsafe_ptr(), From 22a05b4a3cad6c37e9cce798b30374c085ca6833 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Thu, 11 Nov 2021 14:46:52 +0000 Subject: [PATCH 178/302] CHERI: dealloc() use cap base, not address --- src/mem/localalloc.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index b243290ec..b556d77f1 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -464,6 +464,24 @@ namespace snmalloc // before init, that maps null to a remote_deallocator that will never // be in thread local state. +# ifdef __CHERI_PURE_CAPABILITY__ + /* + * On CHERI platforms, snap the provided pointer to its base, ignoring + * any client-provided offset, which may have taken the pointer out of + * bounds and so appear to designate a different object. The base is + * is guaranteed by monotonicity either... + * * to be within the bounds originally returned by alloc(), or + * * one past the end (in which case, the capability length must be 0). + * + * Setting the offset does not trap on untagged capabilities, so the tag + * might be clear after this, as well. + * + * For a well-behaved client, this is a no-op: the base is already at the + * start of the allocation and so the offset is zero. + */ + p_raw = __builtin_cheri_offset_set(p_raw, 0); +# endif + capptr::AllocWild p_wild = capptr_from_client(p_raw); /* From b7772439812d72e35cce87dcaead943dcfbf5a1a Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Thu, 18 Nov 2021 22:54:13 +0000 Subject: [PATCH 179/302] Additional CHERI client checks --- src/mem/localalloc.h | 137 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index b556d77f1..23d00591f 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -454,6 +454,133 @@ namespace snmalloc return alloc(size); } + /* + * Many of these tests come with an "or is null" branch that they'd need to + * add if we did them up front. Instead, defer them until we're past the + * point where we know, from the pagemap, or by explicitly testing, that the + * pointer under test is not nullptr. + */ +#if defined(__CHERI_PURE_CAPABILITY__) && defined(SNMALLOC_CHECK_CLIENT) + SNMALLOC_SLOW_PATH void dealloc_cheri_checks(void* p) + { + /* + * Enforce the use of an unsealed capability. + * + * TODO In CHERI+MTE, this, is part of the CAmoCDecVersion instruction; + * elide this test in that world. + */ + check_client( + !__builtin_cheri_sealed_get(p), "Sealed capability in deallocation"); + + /* + * Enforce permissions on the returned pointer. These pointers end up in + * free queues and will be cycled out to clients again, so try to catch + * erroneous behavior now, rather than later. + * + * TODO In the CHERI+MTE case, we must reconstruct the pointer for the + * free queues as part of the discovery of the start of the object (so + * that it has the correct version), and the CAmoCDecVersion call imposes + * its own requirements on the permissions (to ensure that it's at least + * not zero). They are somewhat more lax than we might wish, so this test + * may remain, guarded by SNMALLOC_CHECK_CLIENT, but no explicit + * permissions checks are required in the non-SNMALLOC_CHECK_CLIENT case + * to defend ourselves or other clients against a misbehaving client. + */ + static const size_t reqperm = CHERI_PERM_LOAD | CHERI_PERM_STORE | + CHERI_PERM_LOAD_CAP | CHERI_PERM_STORE_CAP; + check_client( + (__builtin_cheri_perms_get(p) & reqperm) == reqperm, + "Insufficient permissions on capability in deallocation"); + + /* + * We check for a valid tag here, rather than in domestication, because + * domestication might be answering a slightly different question, about + * the plausibility of addresses rather than of exact pointers. + * + * TODO Further, in the CHERI+MTE case, the tag check will be implicit in + * a future CAmoCDecVersion instruction, and there should be no harm in + * the lookups we perform along the way to get there. In that world, + * elide this test. + */ + check_client( + __builtin_cheri_tag_get(p), "Untagged capability in deallocation"); + + /* + * Verify that the capability is not zero-length, ruling out the other + * edge case around monotonicity. + */ + check_client( + __builtin_cheri_length_get(p) > 0, + "Zero-length capability in deallocation"); + + /* + * At present we check for the pointer also being the start of an + * allocation closer to dealloc; for small objects, that happens in + * dealloc_local_object_fast, either below or *on the far end of message + * receipt*. For large objects, it happens below by directly rounding to + * power of two rather than using the is_start_of_object helper. + * (XXX This does mean that we might end up threading our remote queue + * state somewhere slightly unexpected rather than at the head of an + * object. That is perhaps fine for now?) + */ + + /* + * TODO + * + * We could enforce other policies here, including that the length exactly + * match the sizeclass. At present, we bound caps we give for allocations + * to the underlying sizeclass, so even malloc(0) will have a non-zero + * length. Monotonicity would then imply that the pointer must be the + * head of an object (modulo, perhaps, temporal aliasing if we somehow + * introduced phase shifts in heap layout like some allocators do). + * + * If we switched to bounding with upwards-rounded representable bounds + * (c.f., CRRL) rather than underlying object size, then we should, + * instead, in general require plausibility of p_raw by checking that its + * length is nonzero and the snmalloc size class associated with its + * length is the one for the slab in question... except for the added + * challenge of malloc(0). Since 0 rounds up to 0, we might end up + * constructing zero-length caps to hand out, which we would then reject + * upon receipt. Instead, as part of introducing CRRL bounds, we should + * introduce a sizeclass for slabs holding zero-size objects. All told, + * we would want to check that + * + * size_to_sizeclass(length) == entry.get_sizeclass() + * + * I believe a relaxed CRRL test of + * + * length > 0 || (length == sizeclass_to_size(entry.get_sizeclass())) + * + * would also suffice and may be slightly less expensive than the test + * above, at the cost of not catching as many misbehaving clients. + * + * In either case, having bounded by CRRL bounds, we would need to be + * *reconstructing* the capabilities headed to our free lists to be given + * out to clients again; there are many more CRRL classes than snmalloc + * sizeclasses (this is the same reason that we can always get away with + * CSetBoundsExact in capptr_bound). Switching to CRRL bounds, if that's + * ever a thing we want to do, will be easier after we've done the + * plumbing for CHERI+MTE. + */ + + /* + * TODO: Unsurprisingly, the CHERI+MTE case once again has something to + * say here. In that world, again, we are certain to be reconstructing + * the capability for the free queue anyway, and so exactly what we wish + * to enforce, length-wise, of the provided capability, is somewhat more + * flexible. Using the provided capability bounds when recoloring memory + * could be a natural way to enforce that it covers the entire object, at + * the cost of a more elaborate recovery story (as we risk aborting with a + * partially recolored object). On non-SNMALLOC_CHECK_CLIENT builds, it + * likely makes sense to just enforce that length > 0 (*not* enforced by + * the CAmoCDecVersion instruction) and say that any authority-bearing + * interior pointer suffices to free the object. I believe that to be an + * acceptable security posture for the allocator and between clients; + * misbehavior is confined to the misbehaving client. + */ + } +#endif + SNMALLOC_FAST_PATH void dealloc(void* p_raw) { #ifdef SNMALLOC_PASS_THROUGH @@ -503,6 +630,9 @@ namespace snmalloc core_alloc->backend_state_ptr(), address_cast(p_tame)); if (SNMALLOC_LIKELY(local_cache.remote_allocator == entry.get_remote())) { +# if defined(__CHERI_PURE_CAPABILITY__) && defined(SNMALLOC_CHECK_CLIENT) + dealloc_cheri_checks(p_tame.unsafe_ptr()); +# endif if (SNMALLOC_LIKELY(CoreAlloc::dealloc_local_object_fast( entry, p_tame, local_cache.entropy))) return; @@ -513,6 +643,9 @@ namespace snmalloc if (SNMALLOC_LIKELY( entry.get_remote() != SharedStateHandle::fake_large_remote)) { +# if defined(__CHERI_PURE_CAPABILITY__) && defined(SNMALLOC_CHECK_CLIENT) + dealloc_cheri_checks(p_tame.unsafe_ptr()); +# endif // Check if we have space for the remote deallocation if (local_cache.remote_dealloc_cache.reserve_space(entry)) { @@ -534,6 +667,10 @@ namespace snmalloc if (SNMALLOC_LIKELY( (p_tame != nullptr) && !entry.get_sizeclass().is_default())) { +# if defined(__CHERI_PURE_CAPABILITY__) && defined(SNMALLOC_CHECK_CLIENT) + dealloc_cheri_checks(p_tame.unsafe_ptr()); +# endif + size_t entry_sizeclass = entry.get_sizeclass().as_large(); size_t size = bits::one_at_bit(entry_sizeclass); From be85d53c205f3ee0864619c16ef5e4302cf901dc Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 17 Dec 2021 09:26:24 +0000 Subject: [PATCH 180/302] Factor out sanitizer run. Don't fail on sanitizer failure and provide separate CI jobs for sanitizer builds. --- .github/workflows/main.yml | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5ed35f7c6..4bda5f9c5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -78,10 +78,6 @@ jobs: dependencies: "sudo apt install ninja-build" extra-cmake-flags: "-DCMAKE_CXX_COMPILER=clang++-10 -DCMAKE_C_COMPILER=clang-10 -DCMAKE_CXX_FLAGS=-stdlib=libstdc++" build-only: yes - - os: "ubuntu-latest" - variant: Clang 10 libc++ (TSan + UBSan) - dependencies: "sudo apt install ninja-build" - extra-cmake-flags: "-DCMAKE_CXX_COMPILER=clang++-10 -DCMAKE_C_COMPILER=clang-10 -DCMAKE_CXX_FLAGS=-stdlib=libc++ -DSNMALLOC_SANITIZER=undefined,thread" # Don't abort runners if a single one fails fail-fast: false runs-on: ${{ matrix.os }} @@ -114,6 +110,36 @@ jobs: ninja clean LD_PRELOAD=/usr/local/lib/libsnmallocshim-checks.so ninja + sanitizer: + strategy: + matrix: + # Build each combination of OS and release/debug variants + os: [ "ubuntu-latest"] + build-type: [ Release, Debug ] + include: + - os: "ubuntu-latest" + continue-on-error: # Don't class as an error if this fails, until we have a more reliablity. + variant: Clang 10 libc++ (TSan + UBSan) + dependencies: "sudo apt install ninja-build" + extra-cmake-flags: "-DCMAKE_CXX_COMPILER=clang++-10 -DCMAKE_CXX_FLAGS=-stdlib=\"libc++ -g\" -DSNMALLOC_SANITIZER=undefined,thread" + # Don't abort runners if a single one fails + fail-fast: false + runs-on: ${{ matrix.os }} + name: ${{ matrix.os }} ${{ matrix.build-type }} ${{ matrix.variant }} + steps: + - uses: actions/checkout@v2 + - name: Install build dependencies + run: ${{ matrix.dependencies }} + - name: Configure CMake + run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{matrix.build-type}} -G Ninja ${{ matrix.cmake-flags }} ${{ matrix.extra-cmake-flags }} + # Build with a nice ninja status line + - name: Build + working-directory: ${{github.workspace}}/build + run: NINJA_STATUS="%p [%f:%s/%t] %o/s, %es" ninja + - name: Test + working-directory: ${{github.workspace}}/build + run: ctest --output-on-failure -j 4 -C ${{ matrix.build-type }} --timeout 400 -E "memcpy|external_pointer" --repeat-until-fail 2 + qemu-crossbuild: strategy: matrix: From 91d500baf89664187c3a98380263a1d77a646148 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 17 Dec 2021 11:46:06 +0000 Subject: [PATCH 181/302] Fix flag lock race condition in Debug. --- src/ds/flaglock.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ds/flaglock.h b/src/ds/flaglock.h index 954adc0ec..6155d2b45 100644 --- a/src/ds/flaglock.h +++ b/src/ds/flaglock.h @@ -63,7 +63,7 @@ namespace snmalloc * std::thread::id can be another solution but it does not * support `constexpr` initialisation on some platforms. */ - ThreadIdentity owner = nullptr; + std::atomic owner = nullptr; /** * @brief get_thread_identity From 84ac360445004127c3869ad8a0f0e292030be4f6 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 17 Dec 2021 12:12:32 +0000 Subject: [PATCH 182/302] Make MPSCQ invariant thread-safe --- src/mem/remoteallocator.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index b9b1ff693..3c060e6b0 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -63,7 +63,6 @@ namespace snmalloc void invariant() { SNMALLOC_ASSERT(back != nullptr); - SNMALLOC_ASSERT(front != nullptr); } void init(freelist::HeadPtr stub) @@ -140,6 +139,7 @@ namespace snmalloc Cb cb) { invariant(); + SNMALLOC_ASSERT(front != nullptr); // Use back to bound, so we don't handle new entries. auto b = back.load(std::memory_order_relaxed); From 16875382c401a1ba03ffb9241f00cda76c21a712 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 17 Dec 2021 12:29:42 +0000 Subject: [PATCH 183/302] Make explicitly racy part of stack The MPMCStack has a race that is necessary for implementing optimistic non-blocking data-structures. This commit makes TSAN ignore this race. --- src/ds/mpmcstack.h | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/ds/mpmcstack.h b/src/ds/mpmcstack.h index b701c052c..48af89354 100644 --- a/src/ds/mpmcstack.h +++ b/src/ds/mpmcstack.h @@ -3,6 +3,12 @@ #include "aba.h" #include "ptrwrap.h" +#if defined(__has_feature) +# if __has_feature(thread_sanitizer) +# define SNMALLOC_THREAD_SANITIZER_ENABLED +# endif +#endif + namespace snmalloc { template @@ -13,6 +19,22 @@ namespace snmalloc private: alignas(CACHELINE_SIZE) ABAT stack; +#ifdef SNMALLOC_THREAD_SANITIZER_ENABLED + __attribute__((no_sanitize("thread"))) static T* + racy_read(std::atomic& ptr) + { + // reinterpret_cast is required as TSAN still instruments + // std::atomic operations, even if you disable TSAN on + // the function. + return *reinterpret_cast(&ptr); + } +#else + static T* racy_read(std::atomic& ptr) + { + return ptr.load(std::memory_order_relaxed); + } +#endif + public: constexpr MPMCStack() = default; @@ -52,7 +74,11 @@ namespace snmalloc if (top == nullptr) break; - next = top->next.load(std::memory_order_acquire); + // The following read can race with non-atomic accesses + // this is undefined behaviour. There is no way to use + // CAS sensibly that conforms to the standard with optimistic + // concurrency. + next = racy_read(top->next); } while (!cmp.store_conditional(next)); return top; From 927575dc86c5361e342c95676c4002ac4199f349 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 17 Dec 2021 12:41:27 +0000 Subject: [PATCH 184/302] Increase barrier strength on ABA. --- src/ds/aba.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ds/aba.h b/src/ds/aba.h index e57c574ee..64b0f9955 100644 --- a/src/ds/aba.h +++ b/src/ds/aba.h @@ -112,7 +112,7 @@ namespace snmalloc std::atomic& addr = parent->linked; auto result = addr.compare_exchange_weak( - old, xchg, std::memory_order_relaxed, std::memory_order_relaxed); + old, xchg, std::memory_order_acq_rel, std::memory_order_relaxed); # endif return result; } From 3fb7c98364c94f98fc346f5e325b22b09e23853a Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Thu, 16 Dec 2021 20:16:41 +0000 Subject: [PATCH 185/302] That's snmalloc_check_client to you. c299826f582d991a480ac87a608e6c391b5c3437 landed as part of #428 while #416 was still outstanding and not all conflicts are textual. Sorry! --- src/mem/localalloc.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 23d00591f..e132d060c 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -469,7 +469,7 @@ namespace snmalloc * TODO In CHERI+MTE, this, is part of the CAmoCDecVersion instruction; * elide this test in that world. */ - check_client( + snmalloc_check_client( !__builtin_cheri_sealed_get(p), "Sealed capability in deallocation"); /* @@ -488,7 +488,7 @@ namespace snmalloc */ static const size_t reqperm = CHERI_PERM_LOAD | CHERI_PERM_STORE | CHERI_PERM_LOAD_CAP | CHERI_PERM_STORE_CAP; - check_client( + snmalloc_check_client( (__builtin_cheri_perms_get(p) & reqperm) == reqperm, "Insufficient permissions on capability in deallocation"); @@ -502,14 +502,14 @@ namespace snmalloc * the lookups we perform along the way to get there. In that world, * elide this test. */ - check_client( + snmalloc_check_client( __builtin_cheri_tag_get(p), "Untagged capability in deallocation"); /* * Verify that the capability is not zero-length, ruling out the other * edge case around monotonicity. */ - check_client( + snmalloc_check_client( __builtin_cheri_length_get(p) > 0, "Zero-length capability in deallocation"); From 61314f2260299577ada91b348b9ed9d4fa34eb9c Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 17 Dec 2021 14:08:08 +0000 Subject: [PATCH 186/302] Post large deallocations to original thread (#441) * Post large deallocations to original thread This change sets all large allocations to be owned by the originating thread. This means they will be messaged back to the original thread before they can be reused. The following reason for making this change: * This will improve producer/consumer apps involving large allocations. * It enables the implementation of a more complex chunk allocator that reassembles chunks. * It addresses an issue with compartmentalisation where the handling of large allocations can result in meta-data ownership changing. --- src/mem/corealloc.h | 30 ++++++++++- src/mem/localalloc.h | 58 ++-------------------- src/mem/localcache.h | 6 ++- src/mem/metaslab.h | 38 +++++++++++--- src/mem/sizeclasstable.h | 19 ++++--- src/test/func/release-rounding/rounding.cc | 2 +- 6 files changed, 78 insertions(+), 75 deletions(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index f5f03f896..9a87d2f89 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -402,6 +402,33 @@ namespace snmalloc // TODO: Handle message queue on this path? Metaslab* meta = entry.get_metaslab(); + + if (meta->is_large()) + { + // Handle large deallocation here. + size_t entry_sizeclass = entry.get_sizeclass().as_large(); + size_t size = bits::one_at_bit(entry_sizeclass); + size_t slab_sizeclass = + metaentry_chunk_sizeclass_to_slab_sizeclass(entry_sizeclass); + +#ifdef SNMALLOC_TRACING + std::cout << "Large deallocation: " << size + << " chunk sizeclass: " << slab_sizeclass << std::endl; +#else + UNUSED(size); +#endif + + auto slab_record = reinterpret_cast(meta); + + ChunkAllocator::dealloc( + get_backend_local_state(), + chunk_local_state, + slab_record, + slab_sizeclass); + + return; + } + smallsizeclass_t sizeclass = entry.get_sizeclass().as_small(); UNUSED(entropy); @@ -665,8 +692,7 @@ namespace snmalloc SNMALLOC_ASSERT(!meta->is_unused()); snmalloc_check_client( - Metaslab::is_start_of_object( - entry.get_sizeclass().as_small(), address_cast(p)), + is_start_of_object(entry.get_sizeclass(), address_cast(p)), "Not deallocating start of an object"); auto cp = p.as_static>(); diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index e132d060c..a1d96dc8f 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -186,7 +186,7 @@ namespace snmalloc size_to_sizeclass_full(size), large_size_to_chunk_sizeclass(size), large_size_to_chunk_size(size), - SharedStateHandle::fake_large_remote); + core_alloc->public_state()); // set up meta data so sizeclass is correct, and hence alloc size, and // external pointer. #ifdef SNMALLOC_TRACING @@ -194,9 +194,9 @@ namespace snmalloc << bits::next_pow2_bits(size) << std::endl; #endif - // Note that meta data is not currently used for large allocs. - // meta->initialise(size_to_sizeclass(size)); - UNUSED(meta); + // Initialise meta data for a successful large allocation. + if (meta != nullptr) + meta->initialise_large(); if (zero_mem == YesZero) { @@ -662,56 +662,6 @@ namespace snmalloc return; } - // Large deallocation or null. - // also checks for managed by page map. - if (SNMALLOC_LIKELY( - (p_tame != nullptr) && !entry.get_sizeclass().is_default())) - { -# if defined(__CHERI_PURE_CAPABILITY__) && defined(SNMALLOC_CHECK_CLIENT) - dealloc_cheri_checks(p_tame.unsafe_ptr()); -# endif - - size_t entry_sizeclass = entry.get_sizeclass().as_large(); - - size_t size = bits::one_at_bit(entry_sizeclass); - size_t slab_sizeclass = - metaentry_chunk_sizeclass_to_slab_sizeclass(entry_sizeclass); - - // Check for start of allocation. - snmalloc_check_client( - pointer_align_down(p_tame, size) == p_tame, - "Not start of an allocation."); - -# ifdef SNMALLOC_TRACING - std::cout << "Large deallocation: " << size - << " chunk sizeclass: " << slab_sizeclass << std::endl; -# else - UNUSED(size); -# endif - - auto slab_record = - static_cast(entry.get_metaslab_no_remote()); - - SNMALLOC_ASSERT( - address_cast(slab_record->meta_common.chunk) == address_cast(p_tame)); - - check_init( - []( - CoreAlloc* core_alloc, - ChunkRecord* slab_record, - size_t slab_sizeclass) { - ChunkAllocator::dealloc( - core_alloc->get_backend_local_state(), - core_alloc->chunk_local_state, - slab_record, - slab_sizeclass); - return nullptr; - }, - slab_record, - slab_sizeclass); - return; - } - // If p_tame is not null, then dealloc has been call on something // it shouldn't be called on. // TODO: Should this be tested even in the !CHECK_CLIENT case? diff --git a/src/mem/localcache.h b/src/mem/localcache.h index caa19c6b3..9b3f174d4 100644 --- a/src/mem/localcache.h +++ b/src/mem/localcache.h @@ -15,7 +15,8 @@ namespace snmalloc inline static SNMALLOC_FAST_PATH capptr::Alloc finish_alloc_no_zero(freelist::HeadPtr p, smallsizeclass_t sizeclass) { - SNMALLOC_ASSERT(Metaslab::is_start_of_object(sizeclass, address_cast(p))); + SNMALLOC_ASSERT(is_start_of_object( + sizeclass_t::from_small_class(sizeclass), address_cast(p))); UNUSED(sizeclass); return p.as_void(); @@ -90,7 +91,8 @@ namespace snmalloc while (!small_fast_free_lists[i].empty()) { auto p = small_fast_free_lists[i].take(key, domesticate); - SNMALLOC_ASSERT(Metaslab::is_start_of_object(i, address_cast(p))); + SNMALLOC_ASSERT(is_start_of_object( + sizeclass_t::from_small_class(i), address_cast(p))); dealloc(p.as_void()); } } diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 8ece19a83..bad2d3c27 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -60,6 +60,12 @@ namespace snmalloc */ bool sleeping_ = false; + /** + * Flag to indicate this is actually a large allocation rather than a slab + * of small allocations. + */ + bool large_ = false; + uint16_t& needed() { return needed_; @@ -82,6 +88,25 @@ namespace snmalloc // allocated from. Hence, the bump allocator slab will never be returned // for use in another size class. set_sleeping(sizeclass, 0); + + large_ = false; + } + + /** + * Make this a chunk represent a large allocation. + * + * Set needed so immediately moves to slow path. + */ + void initialise_large() + { + // We will push to this just to make the fast path clean. + free_queue.init(); + + // Flag to detect that it is a large alloc on the slow path + large_ = true; + + // Jump to slow path on first deallocation. + needed() = 1; } /** @@ -106,6 +131,11 @@ namespace snmalloc return sleeping(); } + bool is_large() + { + return large_; + } + /** * Try to set this metaslab to sleep. If the remaining elements are fewer * than the threshold, then it will actually be set to the sleeping state, @@ -144,14 +174,6 @@ namespace snmalloc sleeping() = false; } - static SNMALLOC_FAST_PATH bool - is_start_of_object(smallsizeclass_t sizeclass, address_t p) - { - return divisible_by_sizeclass( - sizeclass, - p - (bits::align_down(p, sizeclass_to_slab_size(sizeclass)))); - } - /** * Allocates a free list from the meta data. * diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index a69760463..e243fc19c 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -254,10 +254,12 @@ namespace snmalloc for (size_t sizeclass = 1; sizeclass < bits::BITS; sizeclass++) { auto lsc = sizeclass_t::from_large_class(sizeclass); - fast(lsc).size = bits::one_at_bit(lsc.as_large()); - - // Use slab mask as 0 for power of two sizes. - fast(lsc).slab_mask = 0; + auto& meta = fast(lsc); + meta.size = bits::one_at_bit(lsc.as_large()); + meta.slab_mask = meta.size - 1; + // The slab_mask will do all the necessary work, so + // perform identity multiplication for the test. + meta.mod_zero_mult = 1; } } }; @@ -367,23 +369,24 @@ namespace snmalloc return sizeclass_metadata.fast(sc).size - index_in_object(sc, addr); } - inline static bool divisible_by_sizeclass(smallsizeclass_t sc, size_t offset) + inline static bool is_start_of_object(sizeclass_t sc, address_t addr) { - // Only works up to certain offsets, exhaustively tested by rounding.cc + size_t offset = addr & (sizeclass_full_to_slab_size(sc) - 1); + // Only works up to certain offsets, exhaustively tested by rounding.cc if constexpr (sizeof(offset) >= 8) { // Only works for 64 bit multiplication, as the following will overflow in // 32bit. // This is based on: // https://lemire.me/blog/2019/02/20/more-fun-with-fast-remainders-when-the-divisor-is-a-constant/ - auto mod_zero_mult = sizeclass_metadata.fast_small(sc).mod_zero_mult; + auto mod_zero_mult = sizeclass_metadata.fast(sc).mod_zero_mult; return (offset * mod_zero_mult) < mod_zero_mult; } else // Use 32-bit division as considerably faster than 64-bit, and // everything fits into 32bits here. - return static_cast(offset % sizeclass_to_size(sc)) == 0; + return static_cast(offset % sizeclass_full_to_size(sc)) == 0; } inline static size_t large_size_to_chunk_size(size_t size) diff --git a/src/test/func/release-rounding/rounding.cc b/src/test/func/release-rounding/rounding.cc index 72fe59563..7b1a2d765 100644 --- a/src/test/func/release-rounding/rounding.cc +++ b/src/test/func/release-rounding/rounding.cc @@ -36,7 +36,7 @@ int main(int argc, char** argv) failed = true; } - bool opt_mod_0 = divisible_by_sizeclass(size_class, offset); + bool opt_mod_0 = is_start_of_object(sc, offset); if (opt_mod_0 != mod_0) { std::cout << "rsize " << rsize << " offset " << offset From a77890c6ee0a8aacef9967a68fce85c7deaae2d8 Mon Sep 17 00:00:00 2001 From: David CARLIER Date: Fri, 7 Jan 2022 11:24:35 +0000 Subject: [PATCH 187/302] PAL netbsd (temporary until some time 2022) build fix. (#450) std::random_device seems unimplemented on this platform thus falling back to the usual /dev/urandom device for the time being. --- src/pal/pal_linux.h | 35 +---------------------------------- src/pal/pal_netbsd.h | 14 +++++++++++++- src/pal/pal_posix.h | 44 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 35 deletions(-) diff --git a/src/pal/pal_linux.h b/src/pal/pal_linux.h index 438a7469c..39d402d91 100644 --- a/src/pal/pal_linux.h +++ b/src/pal/pal_linux.h @@ -4,7 +4,6 @@ # include "../ds/bits.h" # include "pal_posix.h" -# include # include # include # include @@ -166,39 +165,7 @@ namespace snmalloc // reentrancy problem during initialisation routine. // 2. some implementations also require libstdc++ to be linked since // its APIs are not exception-free. - int flags = O_RDONLY; -# if defined(O_CLOEXEC) - flags |= O_CLOEXEC; -# endif - auto fd = open("/dev/urandom", flags, 0); - if (fd > 0) - { - auto current = std::begin(buffer); - auto target = std::end(buffer); - while (auto length = static_cast(target - current)) - { - ret = read(fd, current, length); - if (ret <= 0) - { - if (errno != EAGAIN && errno != EINTR) - { - break; - } - } - else - { - current += ret; - } - } - ret = close(fd); - SNMALLOC_ASSERT(0 == ret); - if (SNMALLOC_LIKELY(target == current)) - { - return result; - } - } - - error("Failed to get system randomness"); + return dev_urandom(); } }; } // namespace snmalloc diff --git a/src/pal/pal_netbsd.h b/src/pal/pal_netbsd.h index e91c4b278..377c1ccaf 100644 --- a/src/pal/pal_netbsd.h +++ b/src/pal/pal_netbsd.h @@ -3,6 +3,8 @@ #ifdef __NetBSD__ # include "pal_bsd_aligned.h" +# include + namespace snmalloc { /** @@ -26,7 +28,17 @@ namespace snmalloc * As NetBSD does not have the getentropy call, get_entropy64 will * currently fallback to C++ libraries std::random_device. */ - static constexpr uint64_t pal_features = PALBSD_Aligned::pal_features; + static constexpr uint64_t pal_features = + PALBSD_Aligned::pal_features | Entropy; + + /** + * Temporary solution while waiting getrandom support for the next release + * random_device seems unimplemented in clang for this platform + */ + static uint64_t get_entropy64() + { + return PALPOSIX::dev_urandom(); + } }; } // namespace snmalloc #endif diff --git a/src/pal/pal_posix.h b/src/pal/pal_posix.h index 6f12b0437..bc05f9f59 100644 --- a/src/pal/pal_posix.h +++ b/src/pal/pal_posix.h @@ -9,6 +9,7 @@ # include SNMALLOC_BACKTRACE_HEADER #endif #include +#include #include #include #include @@ -363,5 +364,48 @@ namespace snmalloc return (static_cast(ts.tv_sec) * 1000) + (static_cast(ts.tv_nsec) / 1000000); } + + static uint64_t dev_urandom() + { + union + { + uint64_t result; + char buffer[sizeof(uint64_t)]; + }; + ssize_t ret; + int flags = O_RDONLY; +#if defined(O_CLOEXEC) + flags |= O_CLOEXEC; +#endif + auto fd = open("/dev/urandom", flags, 0); + if (fd > 0) + { + auto current = std::begin(buffer); + auto target = std::end(buffer); + while (auto length = static_cast(target - current)) + { + ret = read(fd, current, length); + if (ret <= 0) + { + if (errno != EAGAIN && errno != EINTR) + { + break; + } + } + else + { + current += ret; + } + } + ret = close(fd); + SNMALLOC_ASSERT(0 == ret); + if (SNMALLOC_LIKELY(target == current)) + { + return result; + } + } + + error("Failed to get system randomness"); + } }; } // namespace snmalloc From 74a2da177f8545dce3b17927af76951af5743844 Mon Sep 17 00:00:00 2001 From: Schrodinger ZHU Yifan Date: Fri, 7 Jan 2022 19:24:56 +0800 Subject: [PATCH 188/302] try enabling windows-2022 (#445) * try enable windows-2022 Signed-off-by: SchrodingerZhu * only test MSVC on windows 2022 Signed-off-by: SchrodingerZhu --- .github/workflows/main.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4bda5f9c5..27a528c4b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -232,6 +232,23 @@ jobs: toolchain: "" extra-cmake-flags: -DWIN8COMPAT=TRUE variant: Windows 8 compatible + - os: windows-2022 + build-type: Release + arch: Win32 + toolchain: "" + - os: windows-2022 + build-type: Debug + arch: Win32 + toolchain: "" + - os: windows-2022 + build-type: Release + arch: x64 + toolchain: "" + - os: windows-2022 + build-type: Debug + arch: x64 + toolchain: "" + # Don't abort runners if a single one fails fail-fast: false runs-on: ${{ matrix.os }} From 4ea978b9463706ae58b9724fd502069f1a8caea1 Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Fri, 7 Jan 2022 15:29:34 +0000 Subject: [PATCH 189/302] Remove fake_large_remote Since #441 was merged, pagemap entries are no longer ever set to fake_large_remote. --- src/backend/address_space_core.h | 7 +++---- src/backend/commonconfig.h | 18 ------------------ src/mem/localalloc.h | 3 +-- 3 files changed, 4 insertions(+), 24 deletions(-) diff --git a/src/backend/address_space_core.h b/src/backend/address_space_core.h index a4b1f19ce..92113c8ea 100644 --- a/src/backend/address_space_core.h +++ b/src/backend/address_space_core.h @@ -89,10 +89,9 @@ namespace snmalloc // The pagemap stores `MetaEntry`s; abuse the metaslab field to be the // next block in the stack of blocks. // - // The pagemap entries here have nullptr (i.e., fake_large_remote) as - // their remote, and so other accesses to the pagemap (by - // external_pointer, for example) will not attempt to follow this - // "Metaslab" pointer. + // The pagemap entries here have nullptr as their remote, and so other + // accesses to the pagemap (by external_pointer, for example) will not + // attempt to follow this "Metaslab" pointer. // // dealloc() can reject attempts to free such MetaEntry-s due to the // zero sizeclass. diff --git a/src/backend/commonconfig.h b/src/backend/commonconfig.h index befb2e488..74f949d0b 100644 --- a/src/backend/commonconfig.h +++ b/src/backend/commonconfig.h @@ -104,24 +104,6 @@ namespace snmalloc */ SNMALLOC_REQUIRE_CONSTINIT inline static RemoteAllocator unused_remote; - - /** - * Special remote that is used in meta-data for large allocations. - * - * nullptr is considered a large allocations for this purpose to move - * of the critical path. - * - * Bottom bits of the remote pointer are used for a sizeclass, we need - * size bits to represent the non-large sizeclasses, we can then get - * the large sizeclass by having the fake large_remote considerably - * more aligned. - */ - SNMALLOC_REQUIRE_CONSTINIT - inline static constexpr RemoteAllocator* fake_large_remote{nullptr}; - - static_assert( - &unused_remote != fake_large_remote, - "Compilation should ensure these are different"); }; /** diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index a1d96dc8f..5cd80da90 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -640,8 +640,7 @@ namespace snmalloc return; } - if (SNMALLOC_LIKELY( - entry.get_remote() != SharedStateHandle::fake_large_remote)) + if (SNMALLOC_LIKELY(entry.get_remote() != nullptr)) { # if defined(__CHERI_PURE_CAPABILITY__) && defined(SNMALLOC_CHECK_CLIENT) dealloc_cheri_checks(p_tame.unsafe_ptr()); From 419347ba4aff90ecb9209c3fd18367a2ae7d83dc Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 7 Jan 2022 17:09:13 +0000 Subject: [PATCH 190/302] Optimise guarded memcpy (#449) * Improve testing of memcpy including adding perf test. * Change remaining_bytes to be branch free. Use reciprocal division followed by multiply to remove a branch. --- src/mem/sizeclasstable.h | 80 +++++++-------- src/override/memcpy.cc | 35 ++++--- src/test/measuretime.h | 29 +++++- src/test/perf/memcpy/memcpy.cc | 172 +++++++++++++++++++++++++++++++++ 4 files changed, 263 insertions(+), 53 deletions(-) create mode 100644 src/test/perf/memcpy/memcpy.cc diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index e243fc19c..735598748 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -148,7 +148,7 @@ namespace snmalloc // the slab. size_t slab_mask; // Table of constants for reciprocal division for each sizeclass. - size_t mod_mult; + size_t div_mult; // Table of constants for reciprocal modulus for each sizeclass. size_t mod_zero_mult; }; @@ -168,6 +168,8 @@ namespace snmalloc ModArray fast_; ModArray slow_; + size_t DIV_MULT_SHIFT{0}; + [[nodiscard]] constexpr sizeclass_data_fast& fast(sizeclass_t index) { return fast_[index.raw()]; @@ -199,8 +201,10 @@ namespace snmalloc return slow_[index.raw()]; } - constexpr SizeClassTable() : fast_(), slow_() + constexpr SizeClassTable() : fast_(), slow_(), DIV_MULT_SHIFT() { + size_t max_capacity = 0; + for (sizeclass_compress_t sizeclass = 0; sizeclass < NUM_SMALL_SIZECLASSES; sizeclass++) @@ -225,47 +229,49 @@ namespace snmalloc #else static_cast(bits::min((meta_slow.capacity / 4), 32)); #endif + + if (meta_slow.capacity > max_capacity) + { + max_capacity = meta_slow.capacity; + } } + // Get maximum precision to calculate largest division range. + DIV_MULT_SHIFT = bits::BITS - bits::next_pow2_bits_const(max_capacity); + for (sizeclass_compress_t sizeclass = 0; sizeclass < NUM_SMALL_SIZECLASSES; sizeclass++) { - // Calculate reciprocal modulus constant like reciprocal division, but - // constant is choosen to overflow and only leave the modulus as the - // result. + // Calculate reciprocal division constant. auto& meta = fast_small(sizeclass); - meta.mod_mult = bits::one_at_bit(bits::BITS - 1) / meta.size; - meta.mod_mult *= 2; - - if (bits::is_pow2(meta.size)) - { - // Set to zero, so masking path is taken if power of 2. - meta.mod_mult = 0; - } + meta.div_mult = + ((bits::one_at_bit(DIV_MULT_SHIFT) - 1) / meta.size) + 1; size_t zero = 0; meta.mod_zero_mult = (~zero / meta.size) + 1; } - // Set up table for large classes. - // Note skipping sizeclass == 0 as this is size == 0, so the tables can be - // all zero. - for (size_t sizeclass = 1; sizeclass < bits::BITS; sizeclass++) + for (size_t sizeclass = 0; sizeclass < bits::BITS; sizeclass++) { auto lsc = sizeclass_t::from_large_class(sizeclass); auto& meta = fast(lsc); - meta.size = bits::one_at_bit(lsc.as_large()); + meta.size = sizeclass == 0 ? 0 : bits::one_at_bit(lsc.as_large()); meta.slab_mask = meta.size - 1; // The slab_mask will do all the necessary work, so // perform identity multiplication for the test. meta.mod_zero_mult = 1; + // The slab_mask will do all the necessary work for division + // so collapse the calculated offset. + meta.div_mult = 0; } } }; static inline constexpr SizeClassTable sizeclass_metadata = SizeClassTable(); + static constexpr size_t DIV_MULT_SHIFT = sizeclass_metadata.DIV_MULT_SHIFT; + constexpr static inline size_t sizeclass_to_size(smallsizeclass_t sizeclass) { return sizeclass_metadata.fast_small(sizeclass).size; @@ -328,40 +334,36 @@ namespace snmalloc .capacity; } - inline static size_t mod_by_sizeclass(sizeclass_t sc, size_t offset) + inline static address_t start_of_object(sizeclass_t sc, address_t addr) { - // Only works up to certain offsets, exhaustively tested by rounding.cc auto meta = sizeclass_metadata.fast(sc); + address_t slab_start = addr & ~meta.slab_mask; + size_t offset = addr & meta.slab_mask; + size_t size = meta.size; - // Powers of two should use straigt mask. - SNMALLOC_ASSERT(meta.mod_mult != 0); - - if constexpr (sizeof(offset) >= 8) + if constexpr (sizeof(addr) >= 8) { // Only works for 64 bit multiplication, as the following will overflow in // 32bit. - // Could be made nicer with 128bit multiply (umulh): + // Based on // https://lemire.me/blog/2019/02/20/more-fun-with-fast-remainders-when-the-divisor-is-a-constant/ - auto bits_l = bits::BITS / 2; - auto bits_h = bits::BITS - bits_l; - return ( - ((((offset + 1) * meta.mod_mult) >> (bits_l)) * meta.size) >> bits_h); + // We are using an adaptation of the "indirect" method. By using the + // indirect method we can handle the large power of two classes just with + // the slab_mask by making the `div_mult` zero. The link uses 128 bit + // multiplication, we have shrunk the range of the calculation to remove + // this dependency. + size_t offset_start = ((offset * meta.div_mult) >> DIV_MULT_SHIFT) * size; + return slab_start + offset_start; } else - // Use 32-bit division as considerably faster than 64-bit, and - // everything fits into 32bits here. - return static_cast(offset % meta.size); + { + return slab_start + (offset / size) * size; + } } inline static size_t index_in_object(sizeclass_t sc, address_t addr) { - if (sizeclass_metadata.fast(sc).mod_mult == 0) - { - return addr & (sizeclass_metadata.fast(sc).size - 1); - } - - address_t offset = addr & (sizeclass_full_to_slab_size(sc) - 1); - return mod_by_sizeclass(sc, offset); + return addr - start_of_object(sc, addr); } inline static size_t remaining_bytes(sizeclass_t sc, address_t addr) diff --git a/src/override/memcpy.cc b/src/override/memcpy.cc index 6c858f422..884e1c2d5 100644 --- a/src/override/memcpy.cc +++ b/src/override/memcpy.cc @@ -141,7 +141,7 @@ namespace { if constexpr (FailFast) { - UNUSED(ptr, len, msg); + UNUSED(p, len, msg); SNMALLOC_FAST_FAIL(); } else @@ -193,15 +193,12 @@ namespace return (pointer_align_down(const_cast(src)) == src) && (pointer_align_down(dst) == dst); } -} -extern "C" -{ /** * Snmalloc checked memcpy. */ - SNMALLOC_EXPORT void* - SNMALLOC_NAME_MANGLE(memcpy)(void* dst, const void* src, size_t len) + template + void* memcpy(void* dst, const void* src, size_t len) { // 0 is a very common size for memcpy and we don't need to do external // pointer checks if we hit it. It's also the fastest case, to encourage @@ -210,11 +207,15 @@ extern "C" { return dst; } - // Check the bounds of the arguments. - check_bounds( - dst, len, "memcpy with destination out of bounds of heap allocation"); - check_bounds( - src, len, "memcpy with source out of bounds of heap allocation"); + + if constexpr (checked) + { + // Check the bounds of the arguments. + check_bounds( + dst, len, "memcpy with destination out of bounds of heap allocation"); + check_bounds( + src, len, "memcpy with source out of bounds of heap allocation"); + } // If this is a small size, do byte-by-byte copies. if (len < LargestRegisterSize) { @@ -225,4 +226,16 @@ extern "C" copy_end(dst, src, len); return dst; } +} // namespace + +extern "C" +{ + /** + * Snmalloc checked memcpy. + */ + SNMALLOC_EXPORT void* + SNMALLOC_NAME_MANGLE(memcpy)(void* dst, const void* src, size_t len) + { + return memcpy(dst, src, len); + } } diff --git a/src/test/measuretime.h b/src/test/measuretime.h index a67a65e10..903961cc4 100644 --- a/src/test/measuretime.h +++ b/src/test/measuretime.h @@ -5,17 +5,40 @@ #include #include -class MeasureTime : public std::stringstream +class MeasureTime { + std::stringstream ss; std::chrono::time_point start = std::chrono::high_resolution_clock::now(); + bool quiet = false; + public: ~MeasureTime() { auto finish = std::chrono::high_resolution_clock::now(); auto diff = finish - start; - std::cout << str() << ": " << std::setw(12) << diff.count() << " ns" - << std::endl; + if (!quiet) + { + std::cout << ss.str() << ": " << std::setw(12) << diff.count() << " ns" + << std::endl; + } + } + + MeasureTime(bool quiet = false) : quiet(quiet) {} + + template + MeasureTime& operator<<(const T& s) + { + ss << s; + start = std::chrono::high_resolution_clock::now(); + return *this; + } + + std::chrono::nanoseconds get_time() + { + auto finish = std::chrono::high_resolution_clock::now(); + auto diff = finish - start; + return diff; } }; \ No newline at end of file diff --git a/src/test/perf/memcpy/memcpy.cc b/src/test/perf/memcpy/memcpy.cc new file mode 100644 index 000000000..9bfb98544 --- /dev/null +++ b/src/test/perf/memcpy/memcpy.cc @@ -0,0 +1,172 @@ +#include +#include + +#define SNMALLOC_NAME_MANGLE(a) our_##a +#include "override/memcpy.cc" + +#include + +struct Shape +{ + void* object; + void* dst; +}; + +size_t my_random() +{ + return (size_t)rand(); +} + +std::vector allocs; + +void shape(size_t size) +{ + for (size_t i = 0; i < 1000; i++) + { + auto rsize = size * 2; + auto offset = 0; + // Uncomment the next two lines to introduce some randomness to the start of + // the memcpys. constexpr size_t alignment = 16; offset = (my_random() % + // size / alignment) * alignment; + Shape s; + s.object = ThreadAlloc::get().alloc(rsize); + s.dst = reinterpret_cast(s.object) + offset; + // Bring into cache the destination of the copy. + memset(s.dst, 0xFF, size); + allocs.push_back(s); + } +} + +void unshape() +{ + for (auto& s : allocs) + { + ThreadAlloc::get().dealloc(s.object); + } + allocs.clear(); +} + +template +void test_memcpy(size_t size, void* src, Memcpy mc) +{ + for (auto& s : allocs) + { + auto* dst = reinterpret_cast(s.dst); + mc(dst, src, size); + } +} + +template +void test( + size_t size, + Memcpy mc, + std::vector>& stats) +{ + auto src = ThreadAlloc::get().alloc(size); + shape(size); + for (size_t i = 0; i < 10; i++) + { + MeasureTime m(true); + test_memcpy(size, src, mc); + auto time = m.get_time(); + stats.push_back({size, time}); + } + ThreadAlloc::get().dealloc(src); + unshape(); +} + +NOINLINE +void memcpy_checked(void* dst, const void* src, size_t size) +{ + memcpy(dst, src, size); +} + +NOINLINE +void memcpy_unchecked(void* dst, const void* src, size_t size) +{ + memcpy(dst, src, size); +} + +NOINLINE +void memcpy_platform_checked(void* dst, const void* src, size_t size) +{ + check_bounds(dst, size, ""); + memcpy(dst, src, size); +} + +int main(int argc, char** argv) +{ + opt::Opt opt(argc, argv); +#ifndef SNMALLOC_PASS_THROUGH + bool full_test = opt.has("--full_test"); + + // size_t size = 0; + auto mc1 = [](void* dst, const void* src, size_t len) { + memcpy_platform_checked(dst, src, len); + }; + auto mc2 = [](void* dst, const void* src, size_t len) { + memcpy_unchecked(dst, src, len); + }; + auto mc3 = [](void* dst, const void* src, size_t len) { + memcpy(dst, src, len); + }; + + std::vector sizes; + for (size_t size = 1; size < 64; size++) + { + sizes.push_back(size); + } + for (size_t size = 64; size < 256; size += 16) + { + sizes.push_back(size); + sizes.push_back(size + 5); + } + for (size_t size = 256; size < 1024; size += 64) + { + sizes.push_back(size); + sizes.push_back(size + 5); + } + for (size_t size = 1024; size < 8192; size += 256) + { + sizes.push_back(size); + sizes.push_back(size + 5); + } + for (size_t size = 8192; size < bits::one_at_bit(18); size <<= 1) + { + sizes.push_back(size); + sizes.push_back(size + 5); + } + + std::vector> stats_checked; + std::vector> stats_unchecked; + std::vector> stats_platform; + + printf("size, checked, unchecked, platform\n"); + + size_t repeats = full_test ? 80 : 1; + + for (auto repeat = repeats; 0 < repeat; repeat--) + { + for (auto copy_size : sizes) + { + test(copy_size, mc1, stats_checked); + test(copy_size, mc2, stats_unchecked); + test(copy_size, mc3, stats_platform); + } + for (size_t i = 0; i < stats_checked.size(); i++) + { + auto& s1 = stats_checked[i]; + auto& s2 = stats_unchecked[i]; + auto& s3 = stats_platform[i]; + std::cout << s1.first << ", " << s1.second.count() << ", " + << s2.second.count() << ", " << s3.second.count() << std::endl; + } + stats_checked.clear(); + stats_unchecked.clear(); + stats_platform.clear(); + } +#else + snmalloc::UNUSED(opt); +#endif + return 0; +} From ef64f6c31b6cf7a0f79c52ef25b1fca10d98e2f9 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 10 Jan 2022 10:34:28 +0000 Subject: [PATCH 191/302] Improve check_bounds init check. --- src/backend/backend.h | 6 ++++++ src/backend/pagemap.h | 8 ++++++++ src/mem/localalloc.h | 10 ++++++++++ src/override/memcpy.cc | 2 +- src/test/perf/memcpy/memcpy.cc | 2 +- 5 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/backend/backend.h b/src/backend/backend.h index 1c43dbfa9..42bb76023 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -335,6 +335,12 @@ namespace snmalloc UNUSED(local_state); return concretePagemap.get_bounds(); } + + static bool is_initialised(LocalState* ls) + { + UNUSED(ls); + return concretePagemap.is_initialised(); + } }; private: diff --git a/src/backend/pagemap.h b/src/backend/pagemap.h index 48699cd96..cbd32b98a 100644 --- a/src/backend/pagemap.h +++ b/src/backend/pagemap.h @@ -278,6 +278,14 @@ namespace snmalloc return body[p >> SHIFT]; } + /** + * Check if the pagemap has been initialised. + */ + [[nodiscard]] bool is_initialised() const + { + return body_opt != nullptr; + } + /** * Return the starting address corresponding to a given entry within the * Pagemap. Also checks that the reference actually points to a valid entry. diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 5cd80da90..d8e1b131e 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -772,6 +772,16 @@ namespace snmalloc #endif } + bool check_bounds(const void* p, size_t s) + { + auto ls = core_alloc->backend_state_ptr(); + if (SNMALLOC_LIKELY(SharedStateHandle::Pagemap::is_initialised(ls))) + { + return remaining_bytes(p) >= s; + } + return true; + } + /** * Returns the byte offset into an object. * diff --git a/src/override/memcpy.cc b/src/override/memcpy.cc index 884e1c2d5..6b51bc519 100644 --- a/src/override/memcpy.cc +++ b/src/override/memcpy.cc @@ -137,7 +137,7 @@ namespace auto& alloc = ThreadAlloc::get(); void* p = const_cast(ptr); - if (SNMALLOC_UNLIKELY(alloc.remaining_bytes(ptr) < len)) + if (SNMALLOC_UNLIKELY(!alloc.check_bounds(ptr, len))) { if constexpr (FailFast) { diff --git a/src/test/perf/memcpy/memcpy.cc b/src/test/perf/memcpy/memcpy.cc index 9bfb98544..bfa6cb4be 100644 --- a/src/test/perf/memcpy/memcpy.cc +++ b/src/test/perf/memcpy/memcpy.cc @@ -112,7 +112,7 @@ int main(int argc, char** argv) }; std::vector sizes; - for (size_t size = 1; size < 64; size++) + for (size_t size = 0; size < 64; size++) { sizes.push_back(size); } From f1be609cdbd564018d506a91f02ab26b3dd619af Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Fri, 21 Jan 2022 14:19:26 +0000 Subject: [PATCH 192/302] Small fixes for snmalloc used in FreeBSD libc. (#454) - Mark the hook that we're exporting for the threading library to call to clean up per-thread malloc state as 'used'. It was changed to `inline` to allow duplicate copies of it to be merged but this also means that it isn't emitted at all in compilation units that don't use it (and it isn't used internally at all). - Fix the `__je_bootstrap_*` functions, which are used to bootstrap TLS allocation, for the changes to `ScopedAllocator`. The `__je_bootstrap*` functions weren't being built in CI. They now are for non-PIE targets with a smoke test. --- src/ds/defines.h | 2 ++ src/mem/threadalloc.h | 1 + src/override/malloc.cc | 6 +++--- src/test/func/malloc/malloc.cc | 9 +++++++++ 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/ds/defines.h b/src/ds/defines.h index 10bba4518..b21b19446 100644 --- a/src/ds/defines.h +++ b/src/ds/defines.h @@ -25,6 +25,7 @@ # define SNMALLOC_COLD # define SNMALLOC_REQUIRE_CONSTINIT # define SNMALLOC_UNUSED_FUNCTION +# define SNMALLOC_USED_FUNCTION #else # define SNMALLOC_FAST_FAIL() __builtin_trap() # define SNMALLOC_LIKELY(x) __builtin_expect(!!(x), 1) @@ -47,6 +48,7 @@ # define SNMALLOC_PURE __attribute__((const)) # define SNMALLOC_COLD __attribute__((cold)) # define SNMALLOC_UNUSED_FUNCTION __attribute((unused)) +# define SNMALLOC_USED_FUNCTION __attribute((used)) # ifdef __clang__ # define SNMALLOC_REQUIRE_CONSTINIT \ [[clang::require_constant_initialization]] diff --git a/src/mem/threadalloc.h b/src/mem/threadalloc.h index c7d23f1b0..1ab589285 100644 --- a/src/mem/threadalloc.h +++ b/src/mem/threadalloc.h @@ -169,6 +169,7 @@ namespace snmalloc * Entry point that allows libc to call into the allocator for per-thread * cleanup. */ +SNMALLOC_USED_FUNCTION inline void _malloc_thread_cleanup() { snmalloc::ThreadAlloc::get().teardown(); diff --git a/src/override/malloc.cc b/src/override/malloc.cc index f40b2d89d..82b8eafe9 100644 --- a/src/override/malloc.cc +++ b/src/override/malloc.cc @@ -238,7 +238,7 @@ extern "C" void* __je_bootstrap_malloc(size_t size) { - return get_scoped_allocator().alloc(size); + return get_scoped_allocator()->alloc(size); } void* __je_bootstrap_calloc(size_t nmemb, size_t size) @@ -252,12 +252,12 @@ extern "C" } // Include size 0 in the first sizeclass. sz = ((sz - 1) >> (bits::BITS - 1)) + sz; - return get_scoped_allocator().alloc(sz); + return get_scoped_allocator()->alloc(sz); } void __je_bootstrap_free(void* ptr) { - get_scoped_allocator().dealloc(ptr); + get_scoped_allocator()->dealloc(ptr); } #endif } diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index a672a25a1..6aa8ae9ea 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -4,6 +4,7 @@ #define SNMALLOC_NAME_MANGLE(a) our_##a #undef SNMALLOC_NO_REALLOCARRAY #undef SNMALLOC_NO_REALLOCARR +#define SNMALLOC_BOOTSTRAP_ALLOCATOR #include "../../../override/malloc.cc" using namespace snmalloc; @@ -338,6 +339,14 @@ int main(int argc, char** argv) abort(); } +#ifndef __PIC__ snmalloc::debug_check_empty(); + void* bootstrap = __je_bootstrap_malloc(42); + if (bootstrap == nullptr) + { + printf("Failed to allocate from bootstrap malloc\n"); + } + __je_bootstrap_free(bootstrap); +#endif return 0; } From 3d1b973480a4c0342931e43d4a5d5bc8590f6bef Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 1 Feb 2022 09:11:35 +0000 Subject: [PATCH 193/302] Add DEBUG constexpr Enable checking use of a constexpr rather than ifdef for checking if in DEBUG. --- src/ds/defines.h | 21 +++-- src/ds/dllist.h | 82 +++++++++---------- src/mem/corealloc.h | 29 +++---- src/override/memcpy.cc | 12 +-- src/pal/pal_apple.h | 5 +- src/pal/pal_bsd.h | 6 +- src/pal/pal_linux.h | 6 +- src/pal/pal_posix.h | 6 +- .../perf/external_pointer/externalpointer.cc | 7 +- 9 files changed, 85 insertions(+), 89 deletions(-) diff --git a/src/ds/defines.h b/src/ds/defines.h index b21b19446..78304339e 100644 --- a/src/ds/defines.h +++ b/src/ds/defines.h @@ -95,6 +95,12 @@ namespace snmalloc { +#ifdef NDEBUG + static constexpr bool DEBUG = false; +#else + static constexpr bool DEBUG = true; +#endif + // Forwards reference so that the platform can define how to handle errors. [[noreturn]] SNMALLOC_COLD void error(const char* const str); } // namespace snmalloc @@ -161,12 +167,15 @@ namespace snmalloc { if (SNMALLOC_UNLIKELY(!test)) { -#ifdef NDEBUG - UNUSED(str); - SNMALLOC_FAST_FAIL(); -#else - check_client_error(str); -#endif + if constexpr (DEBUG) + { + UNUSED(str); + SNMALLOC_FAST_FAIL(); + } + else + { + check_client_error(str); + } } } diff --git a/src/ds/dllist.h b/src/ds/dllist.h index d4c20e8bc..95bb8e568 100644 --- a/src/ds/dllist.h +++ b/src/ds/dllist.h @@ -92,9 +92,7 @@ namespace snmalloc void insert(Ptr item) { -#ifndef NDEBUG debug_check_not_contains(item); -#endif item->next = head; item->prev = Terminator(); @@ -105,16 +103,14 @@ namespace snmalloc tail = item; head = item; -#ifndef NDEBUG - debug_check(); -#endif + + if constexpr (DEBUG) + debug_check(); } void insert_back(Ptr item) { -#ifndef NDEBUG debug_check_not_contains(item); -#endif item->prev = tail; item->next = Terminator(); @@ -125,16 +121,13 @@ namespace snmalloc head = item; tail = item; -#ifndef NDEBUG + debug_check(); -#endif } SNMALLOC_FAST_PATH void remove(Ptr item) { -#ifndef NDEBUG debug_check_contains(item); -#endif if (item->next != Terminator()) item->next->prev = item->prev; @@ -146,9 +139,7 @@ namespace snmalloc else head = item->next; -#ifndef NDEBUG debug_check(); -#endif } void clear() @@ -163,49 +154,56 @@ namespace snmalloc void debug_check_contains(Ptr item) { -#ifndef NDEBUG - debug_check(); - Ptr curr = head; - - while (curr != item) + if constexpr (DEBUG) + { + debug_check(); + Ptr curr = head; + + while (curr != item) + { + SNMALLOC_ASSERT(curr != Terminator()); + curr = curr->next; + } + } + else { - SNMALLOC_ASSERT(curr != Terminator()); - curr = curr->next; + UNUSED(item); } -#else - UNUSED(item); -#endif } void debug_check_not_contains(Ptr item) { -#ifndef NDEBUG - debug_check(); - Ptr curr = head; - - while (curr != Terminator()) + if constexpr (DEBUG) { - SNMALLOC_ASSERT(curr != item); - curr = curr->next; + debug_check(); + Ptr curr = head; + + while (curr != Terminator()) + { + SNMALLOC_ASSERT(curr != item); + curr = curr->next; + } + } + else + { + UNUSED(item); } -#else - UNUSED(item); -#endif } void debug_check() { -#ifndef NDEBUG - Ptr item = head; - Ptr prev = Terminator(); - - while (item != Terminator()) + if constexpr (DEBUG) { - SNMALLOC_ASSERT(item->prev == prev); - prev = item; - item = item->next; + Ptr item = head; + Ptr prev = Terminator(); + + while (item != Terminator()) + { + SNMALLOC_ASSERT(item->prev == prev); + prev = item; + item = item->next; + } } -#endif } }; } // namespace snmalloc diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 9a87d2f89..66e6b7b76 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -586,21 +586,22 @@ namespace snmalloc ChunkAllocator::register_local_state( get_backend_local_state(), chunk_local_state); -#ifndef NDEBUG - for (smallsizeclass_t i = 0; i < NUM_SMALL_SIZECLASSES; i++) - { - size_t size = sizeclass_to_size(i); - smallsizeclass_t sc1 = size_to_sizeclass(size); - smallsizeclass_t sc2 = size_to_sizeclass_const(size); - size_t size1 = sizeclass_to_size(sc1); - size_t size2 = sizeclass_to_size(sc2); - - SNMALLOC_ASSERT(sc1 == i); - SNMALLOC_ASSERT(sc1 == sc2); - SNMALLOC_ASSERT(size1 == size); - SNMALLOC_ASSERT(size1 == size2); + if constexpr (DEBUG) + { + for (smallsizeclass_t i = 0; i < NUM_SMALL_SIZECLASSES; i++) + { + size_t size = sizeclass_to_size(i); + smallsizeclass_t sc1 = size_to_sizeclass(size); + smallsizeclass_t sc2 = size_to_sizeclass_const(size); + size_t size1 = sizeclass_to_size(sc1); + size_t size2 = sizeclass_to_size(sc2); + + SNMALLOC_CHECK(sc1 == i); + SNMALLOC_CHECK(sc1 == sc2); + SNMALLOC_CHECK(size1 == size); + SNMALLOC_CHECK(size1 == size2); + } } -#endif } public: diff --git a/src/override/memcpy.cc b/src/override/memcpy.cc index 6b51bc519..acfc6a6a1 100644 --- a/src/override/memcpy.cc +++ b/src/override/memcpy.cc @@ -30,11 +30,7 @@ namespace #ifdef SNMALLOC_CHECK_LOADS SNMALLOC_CHECK_LOADS #else -# ifdef NDEBUG - false -# else - true -# endif + DEBUG #endif ; @@ -50,11 +46,7 @@ namespace #ifdef SNMALLOC_FAIL_FAST SNMALLOC_FAIL_FAST #else -# ifdef NDEBUG - true -# else - false -# endif + !DEBUG #endif ; diff --git a/src/pal/pal_apple.h b/src/pal/pal_apple.h index edf511bdd..69f4e5da0 100644 --- a/src/pal/pal_apple.h +++ b/src/pal/pal_apple.h @@ -113,9 +113,8 @@ namespace snmalloc { SNMALLOC_ASSERT(is_aligned_block(p, size)); -# if !defined(NDEBUG) - memset(p, 0x5a, size); -# endif + if constexpr (DEBUG) + memset(p, 0x5a, size); // `MADV_FREE_REUSABLE` can only be applied to writable pages, // otherwise it's an error. diff --git a/src/pal/pal_bsd.h b/src/pal/pal_bsd.h index cf6424499..a07b7c526 100644 --- a/src/pal/pal_bsd.h +++ b/src/pal/pal_bsd.h @@ -35,9 +35,9 @@ namespace snmalloc { SNMALLOC_ASSERT(is_aligned_block(p, size)); -#if !defined(NDEBUG) - memset(p, 0x5a, size); -#endif + if constexpr (DEBUG) + memset(p, 0x5a, size); + madvise(p, size, MADV_FREE); if constexpr (PalEnforceAccess) diff --git a/src/pal/pal_linux.h b/src/pal/pal_linux.h index 39d402d91..139a6afa2 100644 --- a/src/pal/pal_linux.h +++ b/src/pal/pal_linux.h @@ -70,11 +70,11 @@ namespace snmalloc if constexpr (PalEnforceAccess) { -# if !defined(NDEBUG) // Fill memory so that when we switch the pages back on we don't make // assumptions on the content. - memset(p, 0x5a, size); -# endif + if constexpr (DEBUG) + memset(p, 0x5a, size); + madvise(p, size, MADV_FREE); mprotect(p, size, PROT_NONE); } diff --git a/src/pal/pal_posix.h b/src/pal/pal_posix.h index bc05f9f59..4689f6928 100644 --- a/src/pal/pal_posix.h +++ b/src/pal/pal_posix.h @@ -189,11 +189,11 @@ namespace snmalloc if constexpr (PalEnforceAccess) { -#if !defined(NDEBUG) // Fill memory so that when we switch the pages back on we don't make // assumptions on the content. - memset(p, 0x5a, size); -#endif + if constexpr (DEBUG) + memset(p, 0x5a, size); + mprotect(p, size, PROT_NONE); } else diff --git a/src/test/perf/external_pointer/externalpointer.cc b/src/test/perf/external_pointer/externalpointer.cc index e2fdb06be..b589124cb 100644 --- a/src/test/perf/external_pointer/externalpointer.cc +++ b/src/test/perf/external_pointer/externalpointer.cc @@ -95,11 +95,8 @@ int main(int, char**) setup(); xoroshiro::p128r64 r; -# ifdef NDEBUG - size_t nn = 30; -# else - size_t nn = 3; -# endif + + size_t nn = snmalloc::DEBUG ? 30 : 3; for (size_t n = 0; n < nn; n++) test::test_external_pointer(r); From 63d392868710659c0bb4d3eab5c5aaa61c9089c6 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 24 Jan 2022 13:13:58 +0000 Subject: [PATCH 194/302] Add a red-black tree implementation and testing. --- src/ds/redblacktree.h | 698 +++++++++++++++++++++++++++++ src/test/func/redblack/redblack.cc | 178 ++++++++ 2 files changed, 876 insertions(+) create mode 100644 src/ds/redblacktree.h create mode 100644 src/test/func/redblack/redblack.cc diff --git a/src/ds/redblacktree.h b/src/ds/redblacktree.h new file mode 100644 index 000000000..7ea6e9ae6 --- /dev/null +++ b/src/ds/redblacktree.h @@ -0,0 +1,698 @@ +#pragma once +#include "concept.h" +#include "defines.h" + +#include +#include +#include + +namespace snmalloc +{ +#ifdef __cpp_concepts + template + concept RBRepTypes = requires() + { + typename Rep::Holder; + typename Rep::Contents; + }; + + template + concept RBRepMethods = + requires(typename Rep::Holder* hp, typename Rep::Contents k, bool b) + { + { + Rep::get(hp) + } + ->ConceptSame; + { + Rep::set(hp, k) + } + ->ConceptSame; + { + Rep::is_red(k) + } + ->ConceptSame; + { + Rep::set_red(k, b) + } + ->ConceptSame; + { + Rep::ref(b, k) + } + ->ConceptSame; + }; + + template + concept RBRep = // + RBRepTypes // + && RBRepMethods // + && ConceptSame< + decltype(Rep::null), + std::add_const_t>; +#endif + + /** + * Contains a self balancing binary tree. + * + * The template parameter Rep provides the representation of the nodes as a + * collection of functions and types that are requires. See the associated + * test for an example. + * + * run_checks enables invariant checking on the tree. Enabled in Debug. + * TRACE prints all the sets of the rebalancing operations. Only enabled by + * the test when debugging a specific failure. + */ + template< + SNMALLOC_CONCEPT(RBRep) Rep, + bool run_checks = DEBUG, + bool TRACE = false> + class RBTree + { + using H = typename Rep::Holder; + using K = typename Rep::Contents; + + // Container that behaves like a C++ Ref type to enable assignment + // to treat left, right and root uniformly. + class ChildRef + { + H* ptr; + + public: + ChildRef() : ptr(nullptr) {} + + ChildRef(H& p) : ptr(&p) {} + + operator K() + { + return Rep::get(ptr); + } + + K operator=(K t) + { + // Use representations assigment, so we update the correct bits + // color and other things way also be stored in the Holder. + Rep::set(ptr, t); + return t; + } + + bool operator==(ChildRef& t) + { + return ptr == t.ptr; + } + + bool operator!=(ChildRef& t) + { + return ptr != t.ptr; + } + + H* addr() + { + return ptr; + } + }; + + // Root field of the tree + H root{}; + + static ChildRef get_dir(bool direction, K k) + { + return {Rep::ref(direction, k)}; + } + + ChildRef get_root() + { + return {root}; + } + + void invariant() + { + invariant(get_root()); + } + + /* + * Verify structural invariants. Returns the black depth of the `curr`ent + * node. + */ + int invariant(K curr, K lower = Rep::MinKey, K upper = Rep::MaxKey) + { + if constexpr (!run_checks) + { + UNUSED(curr, lower, upper); + return 0; + } + if (curr == Rep::null) + return 1; + + if (curr < lower || curr > upper) + { + if constexpr (TRACE) + { + std::cout << "Invariant failed: " << curr << " is out of bounds " + << lower << ", " << upper << std::endl; + print(); + } + snmalloc::error("Invariant failed"); + } + + if ( + Rep::is_red(curr) && + (Rep::is_red(get_dir(true, curr)) || Rep::is_red(get_dir(false, curr)))) + { + if constexpr (TRACE) + { + std::cout << "Red invariant failed: " << curr + << " is red and has red children" << std::endl; + print(); + } + snmalloc::error("Invariant failed"); + } + + int left_inv = invariant(get_dir(true, curr), lower, curr); + int right_inv = invariant(get_dir(false, curr), curr, upper); + + if (left_inv != right_inv) + { + if constexpr (TRACE) + { + std::cout << "Balance failed: " << curr + << " has different black depths on left and right" + << std::endl; + print(); + } + snmalloc::error("Invariant failed"); + } + + if (Rep::is_red(curr)) + return left_inv; + else + return left_inv + 1; + } + + struct RBStep + { + ChildRef node; + bool dir = false; + }; + + public: + // Internal representation of a path in the tree. + // Exposed to allow for some composite operations to be defined + // externally. + class RBPath + { + friend class RBTree; + + std::array path; + size_t length = 0; + + RBPath(typename Rep::Holder& root) : path{} + { + path[0] = {root, false}; + length = 1; + } + + ChildRef ith(size_t n) + { + SNMALLOC_ASSERT(length >= n); + return path[length - n - 1].node; + } + + bool ith_dir(size_t n) + { + SNMALLOC_ASSERT(length >= n); + return path[length - n - 1].dir; + } + + ChildRef curr() + { + return ith(0); + } + + bool curr_dir() + { + return ith_dir(0); + } + + ChildRef parent() + { + return ith(1); + } + + bool parent_dir() + { + return ith_dir(1); + } + + ChildRef grand_parent() + { + return ith(2); + } + + // Extend path in `direction`. + // If `direction` contains `Rep::null`, do not extend the path. + // Returns false if path is not extended. + bool move(bool direction) + { + auto next = get_dir(direction, curr()); + if (next == Rep::null) + return false; + path[length] = {next, direction}; + length++; + return true; + } + + // Extend path in `direction`. + // If `direction` contains zero, do not extend the path. + // Returns false if path is extended with null. + bool move_inc_null(bool direction) + { + auto next = get_dir(direction, curr()); + path[length] = {next, direction}; + length++; + return next != Rep::null; + } + + // Remove top element from the path. + void pop() + { + SNMALLOC_ASSERT(length > 0); + length--; + } + + // If a path is changed in place, then some references can be stale. + // This rewalks the updated path, and corrects any internal references. + // `expected` is used to run the update, or if `false` used to check + // that no update is required. + void fixup(bool expected = true) + { + if (!run_checks && !expected) + return; + + // During a splice in remove the path can be invalidated, + // this refreshs the path so that the it refers to the spliced + // nodes fields. + // TODO optimise usage to avoid traversing whole path. + for (size_t i = 1; i < length; i++) + { + auto parent = path[i - 1].node; + auto& curr = path[i].node; + auto dir = path[i].dir; + auto actual = get_dir(dir, parent); + if (actual != curr) + { + if (!expected) + { + snmalloc::error("Performed an unexpected fixup."); + } + curr = actual; + } + } + } + + void print() + { + if constexpr (TRACE) + { + for (size_t i = 0; i < length; i++) + { + std::cout << "->" << K(path[i].node) << "@" << path[i].node.addr() + << " (" << path[i].dir << ") "; + } + std::cout << std::endl; + } + } + }; + + private: + void debug_log(const char* msg, RBPath& path) + { + debug_log(msg, path, get_root()); + } + + void debug_log(const char* msg, RBPath& path, ChildRef base) + { + if constexpr (TRACE) + { + std::cout << "-------" << std::endl; + std::cout << msg << std::endl; + path.print(); + print(base); + } + else + { + UNUSED(msg, path, base); + } + } + + public: + RBTree() {} + + void print() + { + print(get_root()); + } + + void print(ChildRef curr, const char* indent = "", size_t depth = 0) + { + if constexpr (TRACE) + { + std::cout << indent << "\\_"; + + if (curr == Rep::null) + { + std::cout << "null" << std::endl; + return; + } + +#ifdef _MSC_VER + auto colour = Rep::is_red(curr) ? "R-" : "B-"; + auto reset = ""; +#else + auto colour = Rep::is_red(curr) ? "\e[1;31m" : "\e[1;34m"; + auto reset = "\e[0m"; +#endif + + std::cout << colour << curr << reset << curr.addr() << " (" << depth + << ")" << std::endl; + if ((get_dir(true, curr) != 0) || (get_dir(false, curr) != 0)) + { + auto s_indent = std::string(indent); + print(get_dir(true, curr), (s_indent + "|").c_str(), depth + 1); + print(get_dir(false, curr), (s_indent + " ").c_str(), depth + 1); + } + } + } + + bool find(RBPath& path, K value) + { + bool dir; + + if (path.curr() == Rep::null) + return false; + + do + { + if (path.curr() == value) + return true; + dir = path.curr() > value; + } while (path.move_inc_null(dir)); + + return false; + } + + bool remove_path(RBPath& path) + { + ChildRef splice = path.curr(); + SNMALLOC_ASSERT(splice != Rep::null); + + debug_log("Removing", path); + + /* + * Find immediately smaller leaf element (rightmost descendant of left + * child) to serve as the replacement for this node. We may not have a + * left subtree, so this may not move the path at all. + */ + path.move(true); + while (path.move(false)) + { + } + + K curr = path.curr(); + + { + // Locally extract right-child-less replacement, replacing it with its + // left child, if any + K child = get_dir(true, path.curr()); + // Unlink target replacing with possible child. + path.curr() = child; + } + + bool leaf_red = Rep::is_red(curr); + + if (path.curr() != splice) + { + // If we had a left child, replace ourselves with the extracted value + // from above + Rep::set_red(curr, Rep::is_red(splice)); + get_dir(true, curr) = K(get_dir(true, splice)); + get_dir(false, curr) = K(get_dir(false, splice)); + splice = curr; + path.fixup(); + } + + debug_log("Splice done", path); + + // Red leaf removal requires no rebalancing. + if (leaf_red) + return true; + + // Now in the double black case. + // End of path is considered double black, that is, one black element + // shorter than satisfies the invariant. The following algorithm moves up + // the path until it finds a close red element or the root. If we convert + // the tree to one, in which the root is double black, then the algorithm + // is complete, as there is nothing to be out of balance with. Otherwise, + // we are searching for nearby red elements so we can rotate the tree to + // rebalance. The following slides nicely cover the case analysis below + // https://www.cs.purdue.edu/homes/ayg/CS251/slides/chap13c.pdf + while (path.curr() != ChildRef(root)) + { + K parent = path.parent(); + bool cur_dir = path.curr_dir(); + K sibling = get_dir(!cur_dir, parent); + + /* Handle red sibling case. + * This performs a rotation to give a black sibling. + * + * p s(b) + * / \ / \ + * c s(r) --> p(r) m + * / \ / \ + * n m c n + * + * By invariant we know that p, n and m are all initially black. + */ + if (Rep::is_red(sibling)) + { + debug_log("Red sibling", path, path.parent()); + K nibling = get_dir(cur_dir, sibling); + get_dir(!cur_dir, parent) = nibling; + get_dir(cur_dir, sibling) = parent; + Rep::set_red(parent, true); + Rep::set_red(sibling, false); + path.parent() = sibling; + // Manually fix path. Using path.fixup would alter the complexity + // class. + path.pop(); + path.move(cur_dir); + path.move_inc_null(cur_dir); + path.fixup(false); + debug_log("Red sibling - done", path, path.parent()); + continue; + } + + /* Handle red nibling case 1. + *

+ * / \ / \ + * c s --> p rn + * / \ / \ + * on rn c on + */ + if (Rep::is_red(get_dir(!cur_dir, sibling))) + { + debug_log("Red nibling 1", path, path.parent()); + K r_nibling = get_dir(!cur_dir, sibling); + K o_nibling = get_dir(cur_dir, sibling); + get_dir(cur_dir, sibling) = parent; + get_dir(!cur_dir, parent) = o_nibling; + path.parent() = sibling; + Rep::set_red(r_nibling, false); + Rep::set_red(sibling, Rep::is_red(parent)); + Rep::set_red(parent, false); + debug_log("Red nibling 1 - done", path, path.parent()); + break; + } + + /* Handle red nibling case 2. + *

+ * / \ / \ + * c s --> p s + * / \ / \ / \ + * rn on c rno rns on + * / \ + * rno rns + */ + if (Rep::is_red(get_dir(cur_dir, sibling))) + { + debug_log("Red nibling 2", path, path.parent()); + K r_nibling = get_dir(cur_dir, sibling); + K r_nibling_same = get_dir(cur_dir, r_nibling); + K r_nibling_opp = get_dir(!cur_dir, r_nibling); + get_dir(!cur_dir, parent) = r_nibling_same; + get_dir(cur_dir, sibling) = r_nibling_opp; + get_dir(cur_dir, r_nibling) = parent; + get_dir(!cur_dir, r_nibling) = sibling; + path.parent() = r_nibling; + Rep::set_red(r_nibling, Rep::is_red(parent)); + Rep::set_red(parent, false); + debug_log("Red nibling 2 - done", path, path.parent()); + break; + } + + // Handle black sibling and niblings, and red parent. + if (Rep::is_red(parent)) + { + // std::cout << "Black sibling and red parent case" << std::endl; + Rep::set_red(parent, false); + Rep::set_red(sibling, true); + break; + } + // Handle black sibling and niblings and black parent. + debug_log( + "Black sibling, niblings and black parent case", path, path.parent()); + Rep::set_red(sibling, true); + path.pop(); + invariant(path.curr()); + debug_log( + "Black sibling, niblings and black parent case - done", + path, + path.curr()); + } + return true; + } + + // Insert an element at the given path. + void insert_path(RBPath path, K value) + { + SNMALLOC_ASSERT(path.curr() == Rep::null); + path.curr() = value; + get_dir(true, path.curr()) = Rep::null; + get_dir(false, path.curr()) = Rep::null; + Rep::set_red(value, true); + + debug_log("Insert ", path); + + // Propogate double red up to rebalance. + // These notes were particularly clear for explaining insert + // https://www.cs.cmu.edu/~fp/courses/15122-f10/lectures/17-rbtrees.pdf + while (path.curr() != get_root()) + { + SNMALLOC_ASSERT(Rep::is_red(path.curr())); + if (!Rep::is_red(path.parent())) + { + invariant(); + return; + } + bool curr_dir = path.curr_dir(); + K curr = path.curr(); + K parent = path.parent(); + K grand_parent = path.grand_parent(); + SNMALLOC_ASSERT(!Rep::is_red(grand_parent)); + if (path.parent_dir() == curr_dir) + { + debug_log("Insert - double red case 1", path, path.grand_parent()); + /* Same direction case + * G - grand parent + * P - parent + * C - current + * S - sibling + * + * G P + * / \ / \ + * A P --> G C + * / \ / \ + * S C A S + */ + K sibling = get_dir(!curr_dir, parent); + Rep::set_red(curr, false); + get_dir(curr_dir, grand_parent) = sibling; + get_dir(!curr_dir, parent) = grand_parent; + path.grand_parent() = parent; + debug_log( + "Insert - double red case 1 - done", path, path.grand_parent()); + } + else + { + debug_log("Insert - double red case 2", path, path.grand_parent()); + /* G - grand parent + * P - parent + * C - current + * Cg - Current child for grand parent + * Cp - Current child for parent + * + * G C + * / \ / \ + * A P G P + * / \ --> / \ / \ + * C B A Cg Cp B + * / \ + * Cg Cp + */ + K child_g = get_dir(curr_dir, curr); + K child_p = get_dir(!curr_dir, curr); + + Rep::set_red(parent, false); + path.grand_parent() = curr; + get_dir(curr_dir, curr) = grand_parent; + get_dir(!curr_dir, curr) = parent; + get_dir(curr_dir, parent) = child_p; + get_dir(!curr_dir, grand_parent) = child_g; + debug_log( + "Insert - double red case 2 - done", path, path.grand_parent()); + } + + // Move to what replaced grand parent. + path.pop(); + path.pop(); + invariant(path.curr()); + } + Rep::set_red(get_root(), false); + invariant(); + } + + K remove_min() + { + if (get_root() == Rep::null) + return Rep::null; + + auto path = get_root_path(); + while (path.move(true)) + { + } + + K result = path.curr(); + + remove_path(path); + return result; + } + + bool remove_elem(K value) + { + if (get_root() == Rep::null) + return false; + + auto path = get_root_path(); + if (!find(path, value)) + return false; + + remove_path(path); + return true; + } + + bool insert_elem(K value) + { + auto path = get_root_path(); + + if (find(path, value)) + return false; + + insert_path(path, value); + return true; + } + + RBPath get_root_path() + { + return RBPath(root); + } + }; +} diff --git a/src/test/func/redblack/redblack.cc b/src/test/func/redblack/redblack.cc new file mode 100644 index 000000000..185d3c147 --- /dev/null +++ b/src/test/func/redblack/redblack.cc @@ -0,0 +1,178 @@ +#include "test/opt.h" +#include "test/setup.h" +#include "test/usage.h" +#include "test/xoroshiro.h" + +#include +#include +#include +#include + +// Redblack tree needs some libraries with trace enabled. +#include "ds/redblacktree.h" +#include "snmalloc.h" + +struct Wrapper +{ + // The redblack tree is going to be used inside the pagemap, + // and the redblack tree cannot use all the bits. Applying an offset + // to the stored value ensures that we have some abstraction over + // the representation. + static constexpr size_t offset = 10000; + + size_t value = offset << 1; +}; + +// Simple representation that is like the pagemap. +// Bottom bit of left is used to store the colour. +// We shift the fields up to make room for the colour. +struct node +{ + Wrapper left; + Wrapper right; +}; + +inline static node array[2048]; + +class Rep +{ +public: + using key = size_t; + + static constexpr key null = 0; + static constexpr key MinKey = 0; + static constexpr key MaxKey = ~MinKey; + + using Holder = Wrapper; + using Contents = size_t; + + static void set(Holder* ptr, Contents r) + { + ptr->value = ((r + Wrapper::offset) << 1) + (ptr->value & 1); + } + + static Contents get(Holder* ptr) + { + return (ptr->value >> 1) - Wrapper::offset; + } + + static Holder& ref(bool direction, key k) + { + if (direction) + return array[k].left; + else + return array[k].right; + } + + static bool is_red(key k) + { + return (array[k].left.value & 1) == 1; + } + + static void set_red(key k, bool new_is_red) + { + if (new_is_red != is_red(k)) + array[k].left.value ^= 1; + } +}; + +template +void test(size_t size, unsigned int seed) +{ + /// Perform a pseudo-random series of + /// additions and removals from the tree. + + xoroshiro::p64r32 rand(seed); + snmalloc::RBTree tree; + std::vector entries; + + bool first = true; + std::cout << "size: " << size << " seed: " << seed << std::endl; + for (size_t i = 0; i < 20 * size; i++) + { + auto batch = 1 + rand.next() % (3 + (size / 2)); + auto op = rand.next() % 4; + if (op < 2 || first) + { + first = false; + for (auto j = batch; j > 0; j--) + { + auto index = 1 + rand.next() % size; + if (tree.insert_elem(index)) + { + entries.push_back(index); + } + } + } + else if (op == 3) + { + for (auto j = batch; j > 0; j--) + { + if (entries.size() == 0) + continue; + auto index = rand.next() % entries.size(); + auto elem = entries[index]; + if (!tree.remove_elem(elem)) + { + std::cout << "Failed to remove element: " << elem << std::endl; + abort(); + } + entries.erase(entries.begin() + static_cast(index)); + } + } + else + { + for (auto j = batch; j > 0; j--) + { + // print(); + auto min = tree.remove_min(); + auto s = entries.size(); + if (min == 0) + break; + + entries.erase( + std::remove(entries.begin(), entries.end(), min), entries.end()); + if (s != entries.size() + 1) + { + std::cout << "Failed to remove min: " << min << std::endl; + abort(); + } + } + } + if (entries.size() == 0) + { + break; + } + } +} + +int main(int argc, char** argv) +{ + setup(); + + opt::Opt opt(argc, argv); + + auto seed = opt.is("--seed", 0); + auto size = opt.is("--size", 0); + + if (seed == 0 && size == 0) + { + for (size = 1; size <= 300; size = size + 1 + (size >> 3)) + for (seed = 1; seed < 5 + (8 * size); seed++) + { + test(size, seed); + } + + return 0; + } + + if (seed == 0 || size == 0) + { + std::cout << "Set both --seed and --size" << std::endl; + return 1; + } + + // Trace particular example + test(size, seed); + return 0; +} \ No newline at end of file From 539937336d6d0c6a9704b74dbd39ebb9e79bbb90 Mon Sep 17 00:00:00 2001 From: Robert Norton Date: Mon, 7 Feb 2022 16:36:51 +0000 Subject: [PATCH 195/302] Extend malloc test to check CHERI capability length. --- src/test/func/malloc/malloc.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index 6aa8ae9ea..9fa1aef20 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -44,6 +44,15 @@ void check_result(size_t size, size_t align, void* p, int err, bool null) const auto exact_size = false; #else const auto exact_size = align == 1; +#endif +#ifdef __CHERI_PURE_CAPABILITY__ + const auto cheri_size = __builtin_cheri_length_get(p); + if (cheri_size != alloc_size && (size != 0)) + { + printf( + "Cheri size is %zu, but required to be %zu.\n", cheri_size, alloc_size); + failed = true; + } #endif if (exact_size && (alloc_size != expected_size) && (size != 0)) { From 20ddf8a15056ddd1d0030b58356b556e6158c107 Mon Sep 17 00:00:00 2001 From: Robert Norton Date: Mon, 7 Feb 2022 17:07:09 +0000 Subject: [PATCH 196/302] Fix bug setting CHERI bounds in address_space_core.h. When splitting a power of two aligned chunk into two we need to set bounds on both halves and not just the top half. --- src/backend/address_space_core.h | 42 +++++++++++++++++--------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/src/backend/address_space_core.h b/src/backend/address_space_core.h index 92113c8ea..b2396cbc4 100644 --- a/src/backend/address_space_core.h +++ b/src/backend/address_space_core.h @@ -169,28 +169,30 @@ namespace snmalloc // Look for larger block and split up recursively capptr::Chunk bigger = remove_block(local_state, align_bits + 1); - if (bigger != nullptr) + + if (SNMALLOC_UNLIKELY(bigger == nullptr)) + return nullptr; + + // This block is going to be broken up into sub CHUNK_SIZE blocks + // so we need to commit it to enable the next pointers to be used + // inside the block. + if ((align_bits + 1) == MIN_CHUNK_BITS) { - // This block is going to be broken up into sub CHUNK_SIZE blocks - // so we need to commit it to enable the next pointers to be used - // inside the block. - if ((align_bits + 1) == MIN_CHUNK_BITS) - { - commit_block(bigger, MIN_CHUNK_SIZE); - } - - size_t left_over_size = bits::one_at_bit(align_bits); - auto left_over = pointer_offset(bigger, left_over_size); - - add_block( - local_state, - align_bits, - Aal::capptr_bound( - left_over, left_over_size)); - check_block(left_over.as_static(), align_bits); + commit_block(bigger, MIN_CHUNK_SIZE); } - check_block(bigger.as_static(), align_bits + 1); - return bigger; + + size_t half_bigger_size = bits::one_at_bit(align_bits); + auto left_over = pointer_offset(bigger, half_bigger_size); + + add_block( + local_state, + align_bits, + Aal::capptr_bound( + left_over, half_bigger_size)); + check_block(left_over.as_static(), align_bits); + check_block(bigger.as_static(), align_bits); + return Aal::capptr_bound( + bigger, half_bigger_size); } check_block(first, align_bits); From 79ea08779bb8a6b1ded2b86c1ee9bf38c3e95b19 Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Wed, 9 Feb 2022 10:49:02 +0000 Subject: [PATCH 197/302] Add jemalloc compat functions. (#456) This adds the full set of jemalloc functions that FreeBSD's libc exposes, including some (the `*allocm` family) that are gone from newer versions of jemalloc and the `*allocx` family that replaced them. These are not necessarily efficient implementations but they should allow snmalloc to replace jemalloc without any ABI breakage (in the loosest possible sense). Jemalloc provides a very generic sysctl-like mechanism for setting and getting some values. These are all implemented to return the not-supported error code. This may break code that expects that they will succeed. In particular, these APIs are used to register custom backing-store allocators and to manage caches and arenas. These concepts don't map directly onto snmalloc and attempting to do so would almost certainly not provide the same performance characteristics and so it's better to `LD_PRELOAD` jemalloc (or explicitly link to it) for programs that gain a significant speedup from this. --- src/override/jemalloc_compat.cc | 388 +++++++++++++++++++++++++++ src/override/malloc.cc | 45 ---- src/test/func/jemalloc/jemalloc.cc | 406 +++++++++++++++++++++++++++++ src/test/func/malloc/malloc.cc | 8 - 4 files changed, 794 insertions(+), 53 deletions(-) create mode 100644 src/override/jemalloc_compat.cc create mode 100644 src/test/func/jemalloc/jemalloc.cc diff --git a/src/override/jemalloc_compat.cc b/src/override/jemalloc_compat.cc new file mode 100644 index 000000000..a65554630 --- /dev/null +++ b/src/override/jemalloc_compat.cc @@ -0,0 +1,388 @@ +#include "override.h" + +#include +#include + +using namespace snmalloc; +namespace +{ + /** + * Helper for JEMalloc-compatible non-standard APIs. These take a flags + * argument as an `int`. This class provides a wrapper for extracting the + * fields embedded in this API. + */ + class JEMallocFlags + { + /** + * The raw flags. + */ + int flags; + + public: + /** + * Constructor, takes a `flags` parameter from one of the `*allocx()` + * JEMalloc APIs. + */ + constexpr JEMallocFlags(int flags) : flags(flags) {} + + /** + * Jemalloc's *allocx APIs store the alignment in the low 6 bits of the + * flags, allowing any alignment up to 2^63. + */ + constexpr int log2align() + { + return flags & 0x3f; + } + + /** + * Jemalloc's *allocx APIs use bit 6 to indicate whether memory should be + * zeroed. + */ + constexpr bool should_zero() + { + return (flags & 0x40) == 0x40; + } + + /** + * Jemalloc's *allocm APIs use bit 7 to indicate whether reallocation may + * move. This is ignored by the `*allocx` functions. + */ + constexpr bool may_not_move() + { + return (flags & 0x80) == 0x80; + } + + size_t aligned_size(size_t size) + { + return ::aligned_size(bits::one_at_bit(log2align()), size); + } + }; + + /** + * Error codes from Jemalloc 3's experimental API. + */ + enum JEMalloc3Result + { + /** + * Allocation succeeded. + */ + allocm_success = 0, + + /** + * Allocation failed because memory was not available. + */ + allocm_err_oom = 1, + + /** + * Reallocation failed because it would have required moving. + */ + allocm_err_not_moved = 2 + }; +} // namespace + +extern "C" +{ + // Stub implementations for jemalloc compatibility. + // These are called by FreeBSD's libthr (pthreads) to notify malloc of + // various events. They are currently unused, though we may wish to reset + // statistics on fork if built with statistics. + + SNMALLOC_EXPORT SNMALLOC_USED_FUNCTION inline void _malloc_prefork(void) {} + SNMALLOC_EXPORT SNMALLOC_USED_FUNCTION inline void _malloc_postfork(void) {} + SNMALLOC_EXPORT SNMALLOC_USED_FUNCTION inline void _malloc_first_thread(void) + {} + + /** + * Jemalloc API provides a way of avoiding name lookup when calling + * `mallctl`. For now, always return an error. + */ + int SNMALLOC_NAME_MANGLE(mallctlnametomib)(const char*, size_t*, size_t*) + { + return ENOENT; + } + + /** + * Jemalloc API provides a generic entry point for various functions. For + * now, this is always implemented to return an error. + */ + int SNMALLOC_NAME_MANGLE(mallctlbymib)( + const size_t*, size_t, void*, size_t*, void*, size_t) + { + return ENOENT; + } + + /** + * Jemalloc API provides a generic entry point for various functions. For + * now, this is always implemented to return an error. + */ + SNMALLOC_EXPORT int + SNMALLOC_NAME_MANGLE(mallctl)(const char*, void*, size_t*, void*, size_t) + { + return ENOENT; + } + +#ifdef SNMALLOC_JEMALLOC3_EXPERIMENTAL + /** + * Jemalloc 3 experimental API. Allocates at least `size` bytes and returns + * the result in `*ptr`, if `rsize` is not null then writes the allocated size + * into `*rsize`. `flags` controls whether the memory is zeroed and what + * alignment is requested. + */ + int SNMALLOC_NAME_MANGLE(allocm)( + void** ptr, size_t* rsize, size_t size, int flags) + { + auto f = JEMallocFlags(flags); + size = f.aligned_size(size); + if (rsize != nullptr) + { + *rsize = round_size(size); + } + if (f.should_zero()) + { + *ptr = ThreadAlloc::get().alloc(size); + } + else + { + *ptr = ThreadAlloc::get().alloc(size); + } + return (*ptr != nullptr) ? allocm_success : allocm_err_oom; + } + + /** + * Jemalloc 3 experimental API. Reallocates the allocation in `*ptr` to be at + * least `size` bytes and returns the result in `*ptr`, if `rsize` is not null + * then writes the allocated size into `*rsize`. `flags` controls whether the + * memory is zeroed and what alignment is requested and whether reallocation + * is permitted. If reallocating, the size will be at least `size` + `extra` + * bytes. + */ + int SNMALLOC_NAME_MANGLE(rallocm)( + void** ptr, size_t* rsize, size_t size, size_t extra, int flags) + { + auto f = JEMallocFlags(flags); + auto alloc_size = f.aligned_size(size); + + auto& a = ThreadAlloc::get(); + size_t sz = a.alloc_size(*ptr); + // Keep the current allocation if the given size is in the same sizeclass. + if (sz == round_size(alloc_size)) + { + if (rsize != nullptr) + { + *rsize = sz; + } + return allocm_success; + } + + if (f.may_not_move()) + { + return allocm_err_not_moved; + } + + if (std::numeric_limits::max() - size > extra) + { + alloc_size = f.aligned_size(size + extra); + } + + void* p = + f.should_zero() ? a.alloc(alloc_size) : a.alloc(alloc_size); + if (SNMALLOC_LIKELY(p != nullptr)) + { + sz = bits::min(alloc_size, sz); + // Guard memcpy as GCC is assuming not nullptr for ptr after the memcpy + // otherwise. + if (sz != 0) + { + memcpy(p, *ptr, sz); + } + a.dealloc(*ptr); + *ptr = p; + if (rsize != nullptr) + { + *rsize = alloc_size; + } + return allocm_success; + } + return allocm_err_oom; + } + + /** + * Jemalloc 3 experimental API. Sets `*rsize` to the size of the allocation + * at `*ptr`. The third argument contains some flags relating to arenas that + * we ignore. + */ + int SNMALLOC_NAME_MANGLE(sallocm)(const void* ptr, size_t* rsize, int) + { + *rsize = ThreadAlloc::get().alloc_size(ptr); + return allocm_success; + } + + /** + * Jemalloc 3 experimental API. Deallocates the allocation + * at `*ptr`. The second argument contains some flags relating to arenas that + * we ignore. + */ + int SNMALLOC_NAME_MANGLE(dallocm)(void* ptr, int) + { + ThreadAlloc::get().dealloc(ptr); + return allocm_success; + } + + /** + * Jemalloc 3 experimental API. Returns in `*rsize` the size of the + * allocation that would be returned if `size` and `flags` are passed to + * `allocm`. + */ + int SNMALLOC_NAME_MANGLE(nallocm)(size_t* rsize, size_t size, int flags) + { + *rsize = round_size(JEMallocFlags(flags).aligned_size(size)); + return allocm_success; + } +#endif + +#ifdef SNMALLOC_JEMALLOC_NONSTANDARD + /** + * Jemalloc function that provides control over alignment and zeroing + * behaviour via the `flags` argument. This argument also includes control + * over the thread cache and arena to use. These don't translate directly to + * snmalloc and so are ignored. + */ + SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(mallocx)(size_t size, int flags) + { + auto f = JEMallocFlags(flags); + size = f.aligned_size(size); + if (f.should_zero()) + { + return ThreadAlloc::get().alloc(size); + } + return ThreadAlloc::get().alloc(size); + } + + /** + * Jemalloc non-standard function that is similar to `realloc`. This can + * request zeroed memory for any newly allocated memory, though only if the + * object grows (which, for snmalloc, means if it's copied). The flags + * controlling the thread cache and arena are ignored. + */ + SNMALLOC_EXPORT void* + SNMALLOC_NAME_MANGLE(rallocx)(void* ptr, size_t size, int flags) + { + auto f = JEMallocFlags(flags); + size = f.aligned_size(size); + + auto& a = ThreadAlloc::get(); + size_t sz = round_size(a.alloc_size(ptr)); + // Keep the current allocation if the given size is in the same sizeclass. + if (sz == size) + { + return ptr; + } + + if (size == (size_t)-1) + { + return nullptr; + } + + // We have a choice here of either asking for zeroed memory, or trying to + // zero the remainder. The former is *probably* faster for large + // allocations, because we get zeroed memory from the PAL and don't zero it + // twice. This is not profiled and so should be considered for refactoring + // if anyone cares about the performance of these APIs. + void* p = f.should_zero() ? a.alloc(size) : a.alloc(size); + if (SNMALLOC_LIKELY(p != nullptr)) + { + sz = bits::min(size, sz); + // Guard memcpy as GCC is assuming not nullptr for ptr after the memcpy + // otherwise. + if (sz != 0) + memcpy(p, ptr, sz); + a.dealloc(ptr); + } + return p; + } + + /** + * Jemalloc non-standard API that performs a `realloc` only if it can do so + * without copying and returns the size of the underlying object. With + * snmalloc, this simply returns the size of the sizeclass backing the + * object. + */ + size_t SNMALLOC_NAME_MANGLE(xallocx)(void* ptr, size_t, size_t, int) + { + auto& a = ThreadAlloc::get(); + return a.alloc_size(ptr); + } + + /** + * Jemalloc non-standard API that queries the underlying size of the + * allocation. + */ + size_t SNMALLOC_NAME_MANGLE(sallocx)(const void* ptr, int) + { + auto& a = ThreadAlloc::get(); + return a.alloc_size(ptr); + } + + /** + * Jemalloc non-standard API that frees `ptr`. The second argument allows + * specifying a thread cache or arena but this is currently unused in + * snmalloc. + */ + void SNMALLOC_NAME_MANGLE(dallocx)(void* ptr, int) + { + ThreadAlloc::get().dealloc(ptr); + } + + /** + * Jemalloc non-standard API that frees `ptr`. The second argument specifies + * a size, which is intended to speed up the operation. This could improve + * performance for snmalloc, if we could guarantee that this is allocated by + * the current thread but is otherwise not helpful. The third argument allows + * specifying a thread cache or arena but this is currently unused in + * snmalloc. + */ + void SNMALLOC_NAME_MANGLE(sdallocx)(void* ptr, size_t, int) + { + ThreadAlloc::get().dealloc(ptr); + } + + /** + * Jemalloc non-standard API that returns the size of memory that would be + * allocated if the same arguments were passed to `mallocx`. + */ + size_t SNMALLOC_NAME_MANGLE(nallocx)(size_t size, int flags) + { + return round_size(JEMallocFlags(flags).aligned_size(size)); + } +#endif + +#if !defined(__PIC__) && defined(SNMALLOC_BOOTSTRAP_ALLOCATOR) + // The following functions are required to work before TLS is set up, in + // statically-linked programs. These temporarily grab an allocator from the + // pool and return it. + + void* __je_bootstrap_malloc(size_t size) + { + return get_scoped_allocator()->alloc(size); + } + + void* __je_bootstrap_calloc(size_t nmemb, size_t size) + { + bool overflow = false; + size_t sz = bits::umul(size, nmemb, overflow); + if (overflow) + { + errno = ENOMEM; + return nullptr; + } + // Include size 0 in the first sizeclass. + sz = ((sz - 1) >> (bits::BITS - 1)) + sz; + return get_scoped_allocator()->alloc(sz); + } + + void __je_bootstrap_free(void* ptr) + { + get_scoped_allocator()->dealloc(ptr); + } +#endif +} diff --git a/src/override/malloc.cc b/src/override/malloc.cc index 82b8eafe9..a04e1dc45 100644 --- a/src/override/malloc.cc +++ b/src/override/malloc.cc @@ -215,49 +215,4 @@ extern "C" return SNMALLOC_NAME_MANGLE(memalign)( OS_PAGE_SIZE, (size + OS_PAGE_SIZE - 1) & ~(OS_PAGE_SIZE - 1)); } - - // Stub implementations for jemalloc compatibility. - // These are called by FreeBSD's libthr (pthreads) to notify malloc of - // various events. They are currently unused, though we may wish to reset - // statistics on fork if built with statistics. - - SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(_malloc_prefork)(void) {} - SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(_malloc_postfork)(void) {} - SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(_malloc_first_thread)(void) {} - - SNMALLOC_EXPORT int - SNMALLOC_NAME_MANGLE(mallctl)(const char*, void*, size_t*, void*, size_t) - { - return ENOENT; - } - -#if !defined(__PIC__) && defined(SNMALLOC_BOOTSTRAP_ALLOCATOR) - // The following functions are required to work before TLS is set up, in - // statically-linked programs. These temporarily grab an allocator from the - // pool and return it. - - void* __je_bootstrap_malloc(size_t size) - { - return get_scoped_allocator()->alloc(size); - } - - void* __je_bootstrap_calloc(size_t nmemb, size_t size) - { - bool overflow = false; - size_t sz = bits::umul(size, nmemb, overflow); - if (overflow) - { - errno = ENOMEM; - return nullptr; - } - // Include size 0 in the first sizeclass. - sz = ((sz - 1) >> (bits::BITS - 1)) + sz; - return get_scoped_allocator()->alloc(sz); - } - - void __je_bootstrap_free(void* ptr) - { - get_scoped_allocator()->dealloc(ptr); - } -#endif } diff --git a/src/test/func/jemalloc/jemalloc.cc b/src/test/func/jemalloc/jemalloc.cc new file mode 100644 index 000000000..83c437708 --- /dev/null +++ b/src/test/func/jemalloc/jemalloc.cc @@ -0,0 +1,406 @@ +#include +#include +#include + +#define SNMALLOC_NAME_MANGLE(a) our_##a +#undef SNMALLOC_NO_REALLOCARRAY +#undef SNMALLOC_NO_REALLOCARR +#define SNMALLOC_BOOTSTRAP_ALLOCATOR +#define SNMALLOC_JEMALLOC3_EXPERIMENTAL +#define SNMALLOC_JEMALLOC_NONSTANDARD +#include "../../../override/jemalloc_compat.cc" +#include "../../../override/malloc.cc" + +#if __has_include() +# include +#endif + +#ifdef __FreeBSD__ +/** + * Enable testing against the versions that we get from libc or elsewhere. + * Enabled by default on FreeBSD where all of the jemalloc functions are + * exported from libc. + */ +# define TEST_JEMALLOC_MALLOCX +#endif + +#define OUR_MALLOCX_LG_ALIGN(la) (static_cast(la)) +#define OUR_MALLOCX_ZERO (one_at_bit(6)) + +#define OUR_ALLOCM_NO_MOVE (one_at_bit(7)) + +#define OUR_ALLOCM_SUCCESS 0 +#define OUR_ALLOCM_ERR_OOM 1 +#define OUR_ALLOCM_ERR_NOT_MOVED 2 + +#ifndef MALLOCX_LG_ALIGN +# define MALLOCX_LG_ALIGN(la) OUR_MALLOCX_LG_ALIGN(la) +#endif +#ifndef MALLOCX_ZERO +# define MALLOCX_ZERO OUR_MALLOCX_ZERO +#endif + +#ifndef ALLOCM_LG_ALIGN +# define ALLOCM_LG_ALIGN(la) OUR_MALLOCX_LG_ALIGN(la) +#endif +#ifndef ALLOCM_ZERO +# define ALLOCM_ZERO OUR_MALLOCX_ZERO +#endif +#ifndef ALLOCM_NO_MOVE +# define ALLOCM_NO_MOVE OUR_ALLOCM_NO_MOVE +#endif +#ifndef ALLOCM_SUCCESS +# define ALLOCM_SUCCESS OUR_ALLOCM_SUCCESS +#endif +#ifndef ALLOCM_ERR_OOM +# define ALLOCM_ERR_OOM OUR_ALLOCM_ERR_OOM +#endif +#ifndef ALLOCM_ERR_NOT_MOVED +# define ALLOCM_ERR_NOT_MOVED OUR_ALLOCM_ERR_NOT_MOVED +#endif + +#ifdef _MSC_VER +# define __PRETTY_FUNCTION__ __FUNCSIG__ +#endif + +using namespace snmalloc; +using namespace snmalloc::bits; + +namespace +{ + /** + * Test whether the MALLOCX_LG_ALIGN macro is defined correctly. This test + * will pass trivially if we don't have the malloc_np.h header from + * jemalloc, but at least the FreeBSD action runners in CI do have this + * header. + */ + template + void check_lg_align_macro() + { + static_assert( + OUR_MALLOCX_LG_ALIGN(Size) == MALLOCX_LG_ALIGN(Size), + "Our definition of MALLOCX_LG_ALIGN is wrong"); + static_assert( + OUR_MALLOCX_LG_ALIGN(Size) == ALLOCM_LG_ALIGN(Size), + "Our definition of ALLOCM_LG_ALIGN is wrong"); + static_assert( + JEMallocFlags(Size).log2align() == Size, "Out log2 align mask is wrong"); + if constexpr (Size > 0) + { + check_lg_align_macro(); + } + } + + /** + * The name of the function under test. This is set in the START_TEST macro + * and used for error reporting in EXPECT. + */ + const char* function = nullptr; + + /** + * Log that the test started. + */ +#define START_TEST(msg) \ + function = __PRETTY_FUNCTION__; \ + fprintf(stderr, "Starting test: " msg "\n"); + + /** + * An assertion that fires even in debug builds. Uses the value set by + * START_TEST. + */ +#define EXPECT(x, msg, ...) \ + if (!(x)) \ + { \ + fprintf( \ + stderr, \ + "%s:%d in %s: " msg "\n", \ + __FILE__, \ + __LINE__, \ + function, \ + ##__VA_ARGS__); \ + fflush(stderr); \ + abort(); \ + } + + /** + * The default maximum number of bits of address space to use for tests. + * This is clamped on platforms without lazy commit because this much RAM + * (or, at least, commit charge) will be used on such systems. + * + * Thread sanitizer makes these tests *very* slow, so reduce the size + * significantly when it's enabled. + */ + constexpr size_t DefaultMax = 22; + + /** + * Run a test with a range of sizes and alignments. The `test` argument is + * called with a size and log2 alignment as parameters. + */ + template + void test_sizes_and_alignments(std::function test) + { + constexpr size_t low = 5; + for (size_t base = low; base < Log2MaxSize; base++) + { + fprintf(stderr, "\tTrying 0x%zx-byte allocations\n", one_at_bit(base)); + for (size_t i = 0; i < one_at_bit(low); i++) + { + for (int align = 1; align < 20; align++) + { + test(one_at_bit(base) + (i << (base - low)), align); + } + } + } + } + + /** + * Test that the size reported by nallocx corresponds to the size reported by + * sallocx on the return value from mallocx. + */ + template< + void*(Mallocx)(size_t, int), + void(Dallocx)(void*, int), + size_t(Sallocx)(const void*, int), + size_t(Nallocx)(size_t, int)> + void test_size() + { + START_TEST("nallocx and mallocx return the same size"); + test_sizes_and_alignments([](size_t size, int align) { + int flags = MALLOCX_LG_ALIGN(align); + size_t expected = Nallocx(size, flags); + void* ptr = Mallocx(size, flags); + EXPECT( + ptr != nullptr, + "Failed to allocate 0x%zx bytes with %d bit alignment", + size, + align); + size_t allocated = Sallocx(ptr, 0); + EXPECT( + allocated == expected, + "Expected to have allocated 0x%zx bytes, got 0x%zx bytes", + expected, + allocated); + Dallocx(ptr, 0); + }); + } + + /** + * Test that, when we request zeroing in rallocx, we get zeroed memory. + */ + template< + void*(Mallocx)(size_t, int), + void(Dallocx)(void*, int), + void*(Rallocx)(void*, size_t, int)> + void test_zeroing() + { + START_TEST("rallocx can zero the remaining space."); + // The Rallocx call will copy everything in the first malloc, so stay + // fairly small. + auto test = [](size_t size, int align) { + int flags = MALLOCX_LG_ALIGN(align) | MALLOCX_ZERO; + char* ptr = static_cast(Mallocx(size, flags)); + ptr = static_cast(Rallocx(ptr, size * 2, flags)); + EXPECT( + ptr != nullptr, + "Failed to reallocate for 0x%zx byte allocation", + size * 2); + EXPECT( + ptr[size] == 0, + "Memory not zero initialised for 0x%zx byte reallocation from 0x%zx " + "byte allocation", + size * 2, + size); + // The second time we run this test, we if we're allocating from a free + // list then we will reuse this, so make sure it requires explicit + // zeroing. + ptr[size] = 12; + Dallocx(ptr, 0); + }; + test_sizes_and_alignments<22>(test); + test_sizes_and_alignments<22>(test); + } + + /** + * Test that xallocx reports a size that is at least the requested amount. + */ + template< + void*(Mallocx)(size_t, int), + void(Dallocx)(void*, int), + size_t(Xallocx)(void*, size_t, size_t, int)> + void test_xallocx() + { + START_TEST("xallocx returns a sensible value."); + // The Rallocx call will copy all of these, so stay fairly small. + auto test = [](size_t size, int align) { + int flags = MALLOCX_LG_ALIGN(align); + void* ptr = Mallocx(size, flags); + EXPECT( + ptr != nullptr, "Failed to allocate for 0x%zx byte allocation", size); + size_t sz = Xallocx(ptr, size, 1024, flags); + EXPECT( + sz >= size, "xalloc returned 0x%zx, expected at least 0x%zx", sz, size); + Dallocx(ptr, 0); + }; + test_sizes_and_alignments(test); + } + + template< + int(Allocm)(void**, size_t*, size_t, int), + int(Sallocm)(const void*, size_t*, int), + int(Dallocm)(void*, int), + int(Nallocm)(size_t*, size_t, int)> + void test_nallocm_size() + { + START_TEST("nallocm and allocm return the same size"); + test_sizes_and_alignments([](size_t size, int align) { + int flags = ALLOCM_LG_ALIGN(align); + size_t expected; + int ret = Nallocm(&expected, size, flags); + EXPECT( + (ret == ALLOCM_SUCCESS), + "nallocm(%zx, %d) failed with error %d", + size, + flags, + ret); + void* ptr; + size_t allocated; + ret = Allocm(&ptr, &allocated, size, flags); + EXPECT( + (ptr != nullptr) && (ret == ALLOCM_SUCCESS), + "Failed to allocate 0x%zx bytes with %d bit alignment", + size, + align); + EXPECT( + allocated == expected, + "Expected to have allocated 0x%zx bytes, got 0x%zx bytes", + expected, + allocated); + ret = Sallocm(ptr, &expected, 0); + EXPECT( + (ret == ALLOCM_SUCCESS) && (allocated == expected), + "Expected to have allocated 0x%zx bytes, got 0x%zx bytes", + expected, + allocated); + + Dallocm(ptr, 0); + }); + } + + template< + int(Allocm)(void**, size_t*, size_t, int), + int(Rallocm)(void**, size_t*, size_t, size_t, int), + int(Dallocm)(void*, int)> + void test_rallocm_nomove() + { + START_TEST("rallocm non-moving behaviour"); + test_sizes_and_alignments([](size_t size, int align) { + int flags = ALLOCM_LG_ALIGN(align); + void* ptr; + size_t allocated; + int ret = Allocm(&ptr, &allocated, size, flags); + void* orig = ptr; + EXPECT( + (ptr != nullptr) && (ret == ALLOCM_SUCCESS), + "Failed to allocate 0x%zx bytes with %d bit alignment", + size, + align); + ret = Rallocm(&ptr, nullptr, allocated + 1, 12, flags | ALLOCM_NO_MOVE); + EXPECT( + (ret == ALLOCM_ERR_NOT_MOVED) || (ptr == orig), + "Expected rallocm not to be able to move or reallocate, but return was " + "%d\n", + ret); + Dallocm(ptr, 0); + }); + } + + template< + int(Allocm)(void**, size_t*, size_t, int), + int(Rallocm)(void**, size_t*, size_t, size_t, int), + int(Sallocm)(const void*, size_t*, int), + int(Dallocm)(void*, int), + int(Nallocm)(size_t*, size_t, int)> + void test_legacy_experimental_apis() + { + START_TEST("allocm out-of-memory behaviour"); + void* ptr = nullptr; + int ret = Allocm(&ptr, nullptr, std::numeric_limits::max() / 2, 0); + EXPECT( + (ptr == nullptr) && (ret == OUR_ALLOCM_ERR_OOM), + "Expected massive allocation to fail with out of memory (%d), received " + "allocation %p, return code %d", + OUR_ALLOCM_ERR_OOM, + ptr, + ret); + test_nallocm_size(); + test_rallocm_nomove(); + } +} + +extern "C" +{ + /** + * The jemalloc 3.x experimental APIs are gone from the headers in newer + * versions, but are still present in FreeBSD libc, so declare them here + * for testing. + */ + int allocm(void**, size_t*, size_t, int); + int rallocm(void**, size_t*, size_t, size_t, int); + int sallocm(const void*, size_t*, int); + int dallocm(void*, int); + int nallocm(size_t*, size_t, int); +} + +int main() +{ +#ifdef SNMALLOC_PASS_THROUGH + return 0; +#endif + check_lg_align_macro<63>(); + static_assert( + OUR_MALLOCX_ZERO == MALLOCX_ZERO, "Our MALLOCX_ZERO macro is wrong"); + static_assert( + OUR_MALLOCX_ZERO == ALLOCM_ZERO, "Our ALLOCM_ZERO macro is wrong"); + static_assert( + OUR_ALLOCM_NO_MOVE == ALLOCM_NO_MOVE, "Our ALLOCM_NO_MOVE macro is wrong"); + static_assert( + JEMallocFlags(MALLOCX_ZERO).should_zero(), + "Our MALLOCX_ZERO is not the value that we are using"); + static_assert( + !JEMallocFlags(~MALLOCX_ZERO).should_zero(), + "Our MALLOCX_ZERO is not the value that we are using"); + static_assert( + JEMallocFlags(ALLOCM_NO_MOVE).may_not_move(), + "Our ALLOCM_NO_MOVE is not the value that we are using"); + static_assert( + !JEMallocFlags(~ALLOCM_NO_MOVE).may_not_move(), + "Our ALLOCM_NO_MOVE is not the value that we are using"); + test_size(); + test_zeroing(); + test_xallocx(); + test_legacy_experimental_apis< + our_allocm, + our_rallocm, + our_sallocm, + our_dallocm, + our_nallocm>(); + +#ifndef __PIC__ + void* bootstrap = __je_bootstrap_malloc(42); + if (bootstrap == nullptr) + { + printf("Failed to allocate from bootstrap malloc\n"); + } + __je_bootstrap_free(bootstrap); +#endif + + // These tests are for jemalloc compatibility and so should work with + // jemalloc's implementation of these functions. If TEST_JEMALLOC is + // defined then we try +#ifdef TEST_JEMALLOC_MALLOCX + test_size(); + test_zeroing(); + test_xallocx(); + test_legacy_experimental_apis(); +#endif +} diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index 9fa1aef20..5ff3f83bd 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -348,14 +348,6 @@ int main(int argc, char** argv) abort(); } -#ifndef __PIC__ snmalloc::debug_check_empty(); - void* bootstrap = __je_bootstrap_malloc(42); - if (bootstrap == nullptr) - { - printf("Failed to allocate from bootstrap malloc\n"); - } - __je_bootstrap_free(bootstrap); -#endif return 0; } From 79486b63315753272afa0d0998da89cda6ca9d1f Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 3 Feb 2022 17:17:58 +0000 Subject: [PATCH 198/302] Rearrange templates for buddy allocator. --- src/backend/address_space.h | 25 +++++----- src/backend/address_space_core.h | 42 ++++++---------- src/backend/backend.h | 85 ++++++++++++++++---------------- src/test/func/pagemap/pagemap.cc | 2 - 4 files changed, 71 insertions(+), 83 deletions(-) diff --git a/src/backend/address_space.h b/src/backend/address_space.h index d2e29f26f..1b3ce4051 100644 --- a/src/backend/address_space.h +++ b/src/backend/address_space.h @@ -19,10 +19,12 @@ namespace snmalloc * It cannot unreserve memory, so this does not require the * usual complexity of a buddy allocator. */ - template + template< + SNMALLOC_CONCEPT(ConceptPAL) PAL, + SNMALLOC_CONCEPT(ConceptBackendMetaRange) Pagemap> class AddressSpaceManager { - AddressSpaceManagerCore core; + AddressSpaceManagerCore core; /** * This is infrequently used code, a spin lock simplifies the code @@ -42,7 +44,7 @@ namespace snmalloc * part of satisfying the request will be registered with the provided * arena_map for use in subsequent amplification. */ - template + template capptr::Chunk reserve(typename Pagemap::LocalState* local_state, size_t size) { @@ -70,7 +72,7 @@ namespace snmalloc capptr::Chunk res; { FlagLock lock(spin_lock); - res = core.template reserve(local_state, size); + res = core.template reserve(local_state, size); if (res == nullptr) { // Allocation failed ask OS for more memory @@ -134,10 +136,10 @@ namespace snmalloc Pagemap::register_range(local_state, address_cast(block), block_size); - core.template add_range(local_state, block, block_size); + core.template add_range(local_state, block, block_size); // still holding lock so guaranteed to succeed. - res = core.template reserve(local_state, size); + res = core.template reserve(local_state, size); } } @@ -155,7 +157,7 @@ namespace snmalloc * This is useful for allowing the space required for alignment to be * used, by smaller objects. */ - template + template capptr::Chunk reserve_with_left_over( typename Pagemap::LocalState* local_state, size_t size) { @@ -165,19 +167,19 @@ namespace snmalloc size_t rsize = bits::next_pow2(size); - auto res = reserve(local_state, rsize); + auto res = reserve(local_state, rsize); if (res != nullptr) { if (rsize > size) { FlagLock lock(spin_lock); - core.template add_range( + core.template add_range( local_state, pointer_offset(res, size), rsize - size); } if constexpr (committed) - core.commit_block(res, size); + core.template commit_block(res, size); } return res; } @@ -193,14 +195,13 @@ namespace snmalloc * Add a range of memory to the address space. * Divides blocks into power of two sizes with natural alignment */ - template void add_range( typename Pagemap::LocalState* local_state, capptr::Chunk base, size_t length) { FlagLock lock(spin_lock); - core.add_range(local_state, base, length); + core.template add_range(local_state, base, length); } }; } // namespace snmalloc diff --git a/src/backend/address_space_core.h b/src/backend/address_space_core.h index b2396cbc4..df26b3915 100644 --- a/src/backend/address_space_core.h +++ b/src/backend/address_space_core.h @@ -31,6 +31,7 @@ namespace snmalloc * to Metaslab objects in perpetuity; it could also make {set,get}_next less * scary. */ + template class AddressSpaceManagerCore { struct FreeChunk @@ -77,7 +78,6 @@ namespace snmalloc * to store the next pointer for the list of unused address space of a * particular size. */ - template void set_next( typename Pagemap::LocalState* local_state, size_t align_bits, @@ -112,7 +112,6 @@ namespace snmalloc * to store the next pointer for the list of unused address space of a * particular size. */ - template capptr::Chunk get_next( typename Pagemap::LocalState* local_state, size_t align_bits, @@ -132,9 +131,7 @@ namespace snmalloc /** * Adds a block to `ranges`. */ - template< - SNMALLOC_CONCEPT(ConceptPAL) PAL, - SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> + template void add_block( typename Pagemap::LocalState* local_state, size_t align_bits, @@ -143,17 +140,15 @@ namespace snmalloc check_block(base, align_bits); SNMALLOC_ASSERT(align_bits < 64); - set_next(local_state, align_bits, base, ranges[align_bits]); - ranges[align_bits] = base.as_static(); + set_next(local_state, align_bits, base, ranges[align_bits]); + ranges[align_bits] = base.template as_static(); } /** * Find a block of the correct size. May split larger blocks * to satisfy this request. */ - template< - SNMALLOC_CONCEPT(ConceptPAL) PAL, - SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> + template capptr::Chunk remove_block(typename Pagemap::LocalState* local_state, size_t align_bits) { @@ -168,7 +163,7 @@ namespace snmalloc // Look for larger block and split up recursively capptr::Chunk bigger = - remove_block(local_state, align_bits + 1); + remove_block(local_state, align_bits + 1); if (SNMALLOC_UNLIKELY(bigger == nullptr)) return nullptr; @@ -184,7 +179,7 @@ namespace snmalloc size_t half_bigger_size = bits::one_at_bit(align_bits); auto left_over = pointer_offset(bigger, half_bigger_size); - add_block( + add_block( local_state, align_bits, Aal::capptr_bound( @@ -196,7 +191,7 @@ namespace snmalloc } check_block(first, align_bits); - ranges[align_bits] = get_next(local_state, align_bits, first); + ranges[align_bits] = get_next(local_state, align_bits, first); return first.as_void(); } @@ -205,9 +200,7 @@ namespace snmalloc * Add a range of memory to the address space. * Divides blocks into power of two sizes with natural alignment */ - template< - SNMALLOC_CONCEPT(ConceptPAL) PAL, - SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> + template void add_range( typename Pagemap::LocalState* local_state, capptr::Chunk base, @@ -242,7 +235,7 @@ namespace snmalloc Aal::capptr_bound(base, align); check_block(b, align_bits); - add_block(local_state, align_bits, b); + add_block(local_state, align_bits, b); base = pointer_offset(base, align); length -= align; @@ -274,9 +267,7 @@ namespace snmalloc * part of satisfying the request will be registered with the provided * arena_map for use in subsequent amplification. */ - template< - SNMALLOC_CONCEPT(ConceptPAL) PAL, - SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> + template capptr::Chunk reserve(typename Pagemap::LocalState* local_state, size_t size) { @@ -287,8 +278,7 @@ namespace snmalloc SNMALLOC_ASSERT(bits::is_pow2(size)); SNMALLOC_ASSERT(size >= sizeof(void*)); - return remove_block( - local_state, bits::next_pow2_bits(size)); + return remove_block(local_state, bits::next_pow2_bits(size)); } /** @@ -298,9 +288,7 @@ namespace snmalloc * This is useful for allowing the space required for alignment to be * used by smaller objects. */ - template< - SNMALLOC_CONCEPT(ConceptPAL) PAL, - SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap> + template capptr::Chunk reserve_with_left_over( typename Pagemap::LocalState* local_state, size_t size) { @@ -310,7 +298,7 @@ namespace snmalloc size_t rsize = bits::next_pow2(size); - auto res = reserve(local_state, rsize); + auto res = reserve(local_state, rsize); if (res != nullptr) { @@ -323,7 +311,7 @@ namespace snmalloc size_t residual_size = rsize - size; auto residual = pointer_offset(res, size); res = Aal::capptr_bound(res, size); - add_range(local_state, residual, residual_size); + add_range(local_state, residual, residual_size); } } return res; diff --git a/src/backend/backend.h b/src/backend/backend.h index 42bb76023..232445efd 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -62,7 +62,9 @@ namespace snmalloc * address space to protect this from corruption. */ static capptr::Chunk alloc_meta_data( - AddressSpaceManager& global, LocalState* local_state, size_t size) + AddressSpaceManager& global, + LocalState* local_state, + size_t size) { return reserve(global, local_state, size); } @@ -77,7 +79,7 @@ namespace snmalloc * where metaslab, is the second element of the pair return. */ static std::pair, Metaslab*> alloc_chunk( - AddressSpaceManager& global, + AddressSpaceManager& global, LocalState* local_state, size_t size, RemoteAllocator* remote, @@ -123,7 +125,9 @@ namespace snmalloc */ template static capptr::Chunk reserve( - AddressSpaceManager& global, LocalState* local_state, size_t size) + AddressSpaceManager& global, + LocalState* local_state, + size_t size) { #ifdef SNMALLOC_META_PROTECTED constexpr auto MAX_CACHED_SIZE = @@ -142,16 +146,14 @@ namespace snmalloc auto& local = local_state->local_address_space; #endif - p = local.template reserve_with_left_over( - local_state, size); + p = local.template reserve_with_left_over(local_state, size); if (p != nullptr) { return p; } auto refill_size = LOCAL_CACHE_BLOCK; - auto refill = - global.template reserve(local_state, refill_size); + auto refill = global.template reserve(local_state, refill_size); if (refill == nullptr) return nullptr; @@ -163,12 +165,10 @@ namespace snmalloc } #endif PAL::template notify_using(refill.unsafe_ptr(), refill_size); - local.template add_range( - local_state, refill, refill_size); + local.template add_range(local_state, refill, refill_size); // This should succeed - return local.template reserve_with_left_over( - local_state, size); + return local.template reserve_with_left_over(local_state, size); } #ifdef SNMALLOC_META_PROTECTED @@ -179,7 +179,7 @@ namespace snmalloc size_t rsize = bits::max(OS_PAGE_SIZE, bits::next_pow2(size)); size_t size_request = rsize * 64; - p = global.template reserve(local_state, size_request); + p = global.template reserve(local_state, size_request); if (p == nullptr) return nullptr; @@ -195,8 +195,7 @@ namespace snmalloc SNMALLOC_ASSERT(!is_meta); #endif - p = global.template reserve_with_left_over( - local_state, size); + p = global.template reserve_with_left_over(local_state, size); return p; } @@ -242,33 +241,7 @@ namespace snmalloc public: using Pal = PAL; - /** - * Local state for the backend allocator. - * - * This class contains thread local structures to make the implementation - * of the backend allocator more efficient. - */ - class LocalState - { - template< - SNMALLOC_CONCEPT(ConceptPAL) PAL2, - typename LocalState, - SNMALLOC_CONCEPT(ConceptBackendMetaRange) Pagemap> - friend class AddressSpaceAllocatorCommon; - - AddressSpaceManagerCore local_address_space; - -#ifdef SNMALLOC_META_PROTECTED - /** - * Secondary local address space, so we can apply some randomisation - * and guard pages to protect the meta-data. - */ - AddressSpaceManagerCore local_meta_address_space; -#endif - }; - - SNMALLOC_REQUIRE_CONSTINIT - static inline AddressSpaceManager address_space; + class LocalState; class Pagemap { @@ -343,6 +316,34 @@ namespace snmalloc } }; + /** + * Local state for the backend allocator. + * + * This class contains thread local structures to make the implementation + * of the backend allocator more efficient. + */ + class LocalState + { + template< + SNMALLOC_CONCEPT(ConceptPAL) PAL2, + typename LocalState, + SNMALLOC_CONCEPT(ConceptBackendMetaRange) Pagemap2> + friend class AddressSpaceAllocatorCommon; + + AddressSpaceManagerCore local_address_space; + +#ifdef SNMALLOC_META_PROTECTED + /** + * Secondary local address space, so we can apply some randomisation + * and guard pages to protect the meta-data. + */ + AddressSpaceManagerCore local_meta_address_space; +#endif + }; + + SNMALLOC_REQUIRE_CONSTINIT + static inline AddressSpaceManager address_space; + private: using AddressSpaceAllocator = AddressSpaceAllocatorCommon; @@ -364,7 +365,7 @@ namespace snmalloc auto [heap_base, heap_length] = Pagemap::concretePagemap.init(base, length); - address_space.template add_range( + address_space.add_range( local_state, capptr::Chunk(heap_base), heap_length); } diff --git a/src/test/func/pagemap/pagemap.cc b/src/test/func/pagemap/pagemap.cc index 410ba8cd2..74f14c45f 100644 --- a/src/test/func/pagemap/pagemap.cc +++ b/src/test/func/pagemap/pagemap.cc @@ -21,8 +21,6 @@ struct T T() {} }; -AddressSpaceManager address_space; - FlatPagemap pagemap_test_unbound; FlatPagemap pagemap_test_bound; From 69c824d54db5cf2702828fbdb8bc7836a4b68d88 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 3 Feb 2022 17:22:21 +0000 Subject: [PATCH 199/302] Expose mutable reference to pagemap --- src/backend/backend.h | 14 ++++++++++++++ src/backend/pagemap.h | 20 ++++++++++++++++---- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/backend/backend.h b/src/backend/backend.h index 232445efd..e529ad8df 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -272,6 +272,20 @@ namespace snmalloc return concretePagemap.template get(p); } + /** + * Get the metadata associated with a chunk. + * + * Set template parameter to true if it not an error + * to access a location that is not backed by a chunk. + */ + template + SNMALLOC_FAST_PATH static MetaEntry& + get_metaentry_mut(LocalState* ls, address_t p) + { + UNUSED(ls); + return concretePagemap.template get_mut(p); + } + /** * Set the metadata associated with a chunk. */ diff --git a/src/backend/pagemap.h b/src/backend/pagemap.h index cbd32b98a..18463b9b5 100644 --- a/src/backend/pagemap.h +++ b/src/backend/pagemap.h @@ -236,12 +236,12 @@ namespace snmalloc * read/write. */ template - const T& get(address_t p) + T& get_mut(address_t p) { if constexpr (potentially_out_of_range) { if (SNMALLOC_UNLIKELY(body_opt == nullptr)) - return default_value; + return const_cast(default_value); } if constexpr (has_bounds) @@ -250,14 +250,14 @@ namespace snmalloc { if constexpr (potentially_out_of_range) { - return default_value; + return const_cast(default_value); } else { // Out of range null should // still return the default value. if (p == 0) - return default_value; + return const_cast(default_value); PAL::error("Internal error: Pagemap read access out of range."); } } @@ -278,6 +278,18 @@ namespace snmalloc return body[p >> SHIFT]; } + /** + * If the location has not been used before, then + * `potentially_out_of_range` should be set to true. + * This will ensure there is a location for the + * read/write. + */ + template + const T& get(address_t p) + { + return get_mut(p); + } + /** * Check if the pagemap has been initialised. */ From be98a27bf17218b29768db0e021a98434500fde8 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 4 Feb 2022 11:58:07 +0000 Subject: [PATCH 200/302] Improve RBTree * Logging improvments * No longer need for the representation to have a min and max. --- src/ds/redblacktree.h | 94 +++++++++++++++++++++++------- src/test/func/redblack/redblack.cc | 3 +- 2 files changed, 75 insertions(+), 22 deletions(-) diff --git a/src/ds/redblacktree.h b/src/ds/redblacktree.h index 7ea6e9ae6..80325b3b2 100644 --- a/src/ds/redblacktree.h +++ b/src/ds/redblacktree.h @@ -8,6 +8,37 @@ namespace snmalloc { + template + class debug_out + { + public: + template + static void msg(A a, Args... args) + { + if constexpr (TRACE) + { +#ifdef SNMALLOC_TRACING + std::cout << a; +#else + UNUSED(a); +#endif + + msg(args...); + } + } + + template + static void msg() + { + if constexpr (TRACE && new_line) + { +#ifdef SNMALLOC_TRACING + std::cout << std::endl; +#endif + } + } + }; + #ifdef __cpp_concepts template concept RBRepTypes = requires() @@ -133,7 +164,7 @@ namespace snmalloc * Verify structural invariants. Returns the black depth of the `curr`ent * node. */ - int invariant(K curr, K lower = Rep::MinKey, K upper = Rep::MaxKey) + int invariant(K curr, K lower = Rep::null, K upper = Rep::null) { if constexpr (!run_checks) { @@ -143,12 +174,19 @@ namespace snmalloc if (curr == Rep::null) return 1; - if (curr < lower || curr > upper) + if ( + ((lower != Rep::null) && curr < lower) || + ((upper != Rep::null) && curr > upper)) { if constexpr (TRACE) { - std::cout << "Invariant failed: " << curr << " is out of bounds " - << lower << ", " << upper << std::endl; + debug_out::msg( + "Invariant failed: ", + curr, + " is out of bounds ", + lower, + ", ", + upper); print(); } snmalloc::error("Invariant failed"); @@ -160,8 +198,8 @@ namespace snmalloc { if constexpr (TRACE) { - std::cout << "Red invariant failed: " << curr - << " is red and has red children" << std::endl; + debug_out::msg( + "Red invariant failed: ", curr, " is red and has red children"); print(); } snmalloc::error("Invariant failed"); @@ -174,9 +212,10 @@ namespace snmalloc { if constexpr (TRACE) { - std::cout << "Balance failed: " << curr - << " has different black depths on left and right" - << std::endl; + debug_out::msg( + "Balance failed: ", + curr, + " has different black depths on left and right"); print(); } snmalloc::error("Invariant failed"); @@ -315,10 +354,16 @@ namespace snmalloc { for (size_t i = 0; i < length; i++) { - std::cout << "->" << K(path[i].node) << "@" << path[i].node.addr() - << " (" << path[i].dir << ") "; + debug_out::msg( + "->", + K(path[i].node), + "@", + path[i].node.addr(), + " (", + path[i].dir, + ") "); } - std::cout << std::endl; + debug_out::msg(); } } }; @@ -333,8 +378,8 @@ namespace snmalloc { if constexpr (TRACE) { - std::cout << "-------" << std::endl; - std::cout << msg << std::endl; + debug_out::msg("-------"); + debug_out::msg(msg); path.print(); print(base); } @@ -356,11 +401,9 @@ namespace snmalloc { if constexpr (TRACE) { - std::cout << indent << "\\_"; - if (curr == Rep::null) { - std::cout << "null" << std::endl; + debug_out::msg(indent, "\\_", "null"); return; } @@ -372,8 +415,17 @@ namespace snmalloc auto reset = "\e[0m"; #endif - std::cout << colour << curr << reset << curr.addr() << " (" << depth - << ")" << std::endl; + debug_out::msg( + indent, + "\\_", + colour, + curr, + reset, + "@", + curr.addr(), + " (", + depth, + ")"); if ((get_dir(true, curr) != 0) || (get_dir(false, curr) != 0)) { auto s_indent = std::string(indent); @@ -542,9 +594,11 @@ namespace snmalloc // Handle black sibling and niblings, and red parent. if (Rep::is_red(parent)) { - // std::cout << "Black sibling and red parent case" << std::endl; + debug_log("Black sibling and red parent case", path, path.parent()); Rep::set_red(parent, false); Rep::set_red(sibling, true); + debug_log( + "Black sibling and red parent case - done", path, path.parent()); break; } // Handle black sibling and niblings and black parent. diff --git a/src/test/func/redblack/redblack.cc b/src/test/func/redblack/redblack.cc index 185d3c147..2c96541e8 100644 --- a/src/test/func/redblack/redblack.cc +++ b/src/test/func/redblack/redblack.cc @@ -8,6 +8,7 @@ #include #include +#define SNMALLOC_TRACING // Redblack tree needs some libraries with trace enabled. #include "ds/redblacktree.h" #include "snmalloc.h" @@ -40,8 +41,6 @@ class Rep using key = size_t; static constexpr key null = 0; - static constexpr key MinKey = 0; - static constexpr key MaxKey = ~MinKey; using Holder = Wrapper; using Contents = size_t; From eb00f3184f3e1c2f1651bfe1d7c0021446bd03ae Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 10 Feb 2022 11:57:13 +0000 Subject: [PATCH 201/302] Remove local state from Pagemap representation. --- src/backend/address_space.h | 28 +++++++----------- src/backend/address_space_core.h | 48 +++++++++++-------------------- src/backend/backend.h | 49 ++++++++++---------------------- src/backend/backend_concept.h | 17 +++++------ src/backend/fixedglobalconfig.h | 5 ++-- src/mem/chunkallocator.h | 2 +- src/mem/corealloc.h | 10 +++---- src/mem/localalloc.h | 19 ++++++------- src/mem/remotecache.h | 8 +++--- 9 files changed, 72 insertions(+), 114 deletions(-) diff --git a/src/backend/address_space.h b/src/backend/address_space.h index 1b3ce4051..15c1ae3eb 100644 --- a/src/backend/address_space.h +++ b/src/backend/address_space.h @@ -45,8 +45,7 @@ namespace snmalloc * arena_map for use in subsequent amplification. */ template - capptr::Chunk - reserve(typename Pagemap::LocalState* local_state, size_t size) + capptr::Chunk reserve(size_t size) { #ifdef SNMALLOC_TRACING std::cout << "ASM reserve request:" << size << std::endl; @@ -64,7 +63,7 @@ namespace snmalloc { auto base = capptr::Chunk(PAL::template reserve_aligned(size)); - Pagemap::register_range(local_state, address_cast(base), size); + Pagemap::register_range(address_cast(base), size); return base; } } @@ -72,7 +71,7 @@ namespace snmalloc capptr::Chunk res; { FlagLock lock(spin_lock); - res = core.template reserve(local_state, size); + res = core.template reserve(size); if (res == nullptr) { // Allocation failed ask OS for more memory @@ -134,12 +133,12 @@ namespace snmalloc return nullptr; } - Pagemap::register_range(local_state, address_cast(block), block_size); + Pagemap::register_range(address_cast(block), block_size); - core.template add_range(local_state, block, block_size); + core.template add_range(block, block_size); // still holding lock so guaranteed to succeed. - res = core.template reserve(local_state, size); + res = core.template reserve(size); } } @@ -158,8 +157,7 @@ namespace snmalloc * used, by smaller objects. */ template - capptr::Chunk reserve_with_left_over( - typename Pagemap::LocalState* local_state, size_t size) + capptr::Chunk reserve_with_left_over(size_t size) { SNMALLOC_ASSERT(size >= sizeof(void*)); @@ -167,15 +165,14 @@ namespace snmalloc size_t rsize = bits::next_pow2(size); - auto res = reserve(local_state, rsize); + auto res = reserve(rsize); if (res != nullptr) { if (rsize > size) { FlagLock lock(spin_lock); - core.template add_range( - local_state, pointer_offset(res, size), rsize - size); + core.template add_range(pointer_offset(res, size), rsize - size); } if constexpr (committed) @@ -195,13 +192,10 @@ namespace snmalloc * Add a range of memory to the address space. * Divides blocks into power of two sizes with natural alignment */ - void add_range( - typename Pagemap::LocalState* local_state, - capptr::Chunk base, - size_t length) + void add_range(capptr::Chunk base, size_t length) { FlagLock lock(spin_lock); - core.template add_range(local_state, base, length); + core.template add_range(base, length); } }; } // namespace snmalloc diff --git a/src/backend/address_space_core.h b/src/backend/address_space_core.h index df26b3915..fa3aa2592 100644 --- a/src/backend/address_space_core.h +++ b/src/backend/address_space_core.h @@ -79,7 +79,6 @@ namespace snmalloc * particular size. */ void set_next( - typename Pagemap::LocalState* local_state, size_t align_bits, capptr::Chunk base, capptr::Chunk next) @@ -96,7 +95,7 @@ namespace snmalloc // dealloc() can reject attempts to free such MetaEntry-s due to the // zero sizeclass. MetaEntry t(reinterpret_cast(next.unsafe_ptr()), nullptr); - Pagemap::set_metaentry(local_state, address_cast(base), 1, t); + Pagemap::set_metaentry(address_cast(base), 1, t); return; } @@ -112,15 +111,13 @@ namespace snmalloc * to store the next pointer for the list of unused address space of a * particular size. */ - capptr::Chunk get_next( - typename Pagemap::LocalState* local_state, - size_t align_bits, - capptr::Chunk base) + capptr::Chunk + get_next(size_t align_bits, capptr::Chunk base) { if (align_bits >= MIN_CHUNK_BITS) { - const MetaEntry& t = Pagemap::template get_metaentry( - local_state, address_cast(base)); + const MetaEntry& t = + Pagemap::template get_metaentry(address_cast(base)); return capptr::Chunk( reinterpret_cast(t.get_metaslab_no_remote())); } @@ -132,15 +129,12 @@ namespace snmalloc * Adds a block to `ranges`. */ template - void add_block( - typename Pagemap::LocalState* local_state, - size_t align_bits, - capptr::Chunk base) + void add_block(size_t align_bits, capptr::Chunk base) { check_block(base, align_bits); SNMALLOC_ASSERT(align_bits < 64); - set_next(local_state, align_bits, base, ranges[align_bits]); + set_next(align_bits, base, ranges[align_bits]); ranges[align_bits] = base.template as_static(); } @@ -149,8 +143,7 @@ namespace snmalloc * to satisfy this request. */ template - capptr::Chunk - remove_block(typename Pagemap::LocalState* local_state, size_t align_bits) + capptr::Chunk remove_block(size_t align_bits) { capptr::Chunk first = ranges[align_bits]; if (first == nullptr) @@ -162,8 +155,7 @@ namespace snmalloc } // Look for larger block and split up recursively - capptr::Chunk bigger = - remove_block(local_state, align_bits + 1); + capptr::Chunk bigger = remove_block(align_bits + 1); if (SNMALLOC_UNLIKELY(bigger == nullptr)) return nullptr; @@ -180,7 +172,6 @@ namespace snmalloc auto left_over = pointer_offset(bigger, half_bigger_size); add_block( - local_state, align_bits, Aal::capptr_bound( left_over, half_bigger_size)); @@ -191,7 +182,7 @@ namespace snmalloc } check_block(first, align_bits); - ranges[align_bits] = get_next(local_state, align_bits, first); + ranges[align_bits] = get_next(align_bits, first); return first.as_void(); } @@ -201,10 +192,7 @@ namespace snmalloc * Divides blocks into power of two sizes with natural alignment */ template - void add_range( - typename Pagemap::LocalState* local_state, - capptr::Chunk base, - size_t length) + void add_range(capptr::Chunk base, size_t length) { // For start and end that are not chunk sized, we need to // commit the pages to track the allocations. @@ -235,7 +223,7 @@ namespace snmalloc Aal::capptr_bound(base, align); check_block(b, align_bits); - add_block(local_state, align_bits, b); + add_block(align_bits, b); base = pointer_offset(base, align); length -= align; @@ -268,8 +256,7 @@ namespace snmalloc * arena_map for use in subsequent amplification. */ template - capptr::Chunk - reserve(typename Pagemap::LocalState* local_state, size_t size) + capptr::Chunk reserve(size_t size) { #ifdef SNMALLOC_TRACING std::cout << "ASM Core reserve request:" << size << std::endl; @@ -278,7 +265,7 @@ namespace snmalloc SNMALLOC_ASSERT(bits::is_pow2(size)); SNMALLOC_ASSERT(size >= sizeof(void*)); - return remove_block(local_state, bits::next_pow2_bits(size)); + return remove_block(bits::next_pow2_bits(size)); } /** @@ -289,8 +276,7 @@ namespace snmalloc * used by smaller objects. */ template - capptr::Chunk reserve_with_left_over( - typename Pagemap::LocalState* local_state, size_t size) + capptr::Chunk reserve_with_left_over(size_t size) { SNMALLOC_ASSERT(size >= sizeof(void*)); @@ -298,7 +284,7 @@ namespace snmalloc size_t rsize = bits::next_pow2(size); - auto res = reserve(local_state, rsize); + auto res = reserve(rsize); if (res != nullptr) { @@ -311,7 +297,7 @@ namespace snmalloc size_t residual_size = rsize - size; auto residual = pointer_offset(res, size); res = Aal::capptr_bound(res, size); - add_range(local_state, residual, residual_size); + add_range(residual, residual_size); } } return res; diff --git a/src/backend/backend.h b/src/backend/backend.h index e529ad8df..e7cff28da 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -114,7 +114,7 @@ namespace snmalloc meta->meta_common.chunk = p; MetaEntry t(meta, remote, sizeclass); - Pagemap::set_metaentry(local_state, address_cast(p), size, t); + Pagemap::set_metaentry(address_cast(p), size, t); return {p, meta}; } @@ -146,14 +146,14 @@ namespace snmalloc auto& local = local_state->local_address_space; #endif - p = local.template reserve_with_left_over(local_state, size); + p = local.template reserve_with_left_over(size); if (p != nullptr) { return p; } auto refill_size = LOCAL_CACHE_BLOCK; - auto refill = global.template reserve(local_state, refill_size); + auto refill = global.template reserve(refill_size); if (refill == nullptr) return nullptr; @@ -165,10 +165,10 @@ namespace snmalloc } #endif PAL::template notify_using(refill.unsafe_ptr(), refill_size); - local.template add_range(local_state, refill, refill_size); + local.template add_range(refill, refill_size); // This should succeed - return local.template reserve_with_left_over(local_state, size); + return local.template reserve_with_left_over(size); } #ifdef SNMALLOC_META_PROTECTED @@ -179,7 +179,7 @@ namespace snmalloc size_t rsize = bits::max(OS_PAGE_SIZE, bits::next_pow2(size)); size_t size_request = rsize * 64; - p = global.template reserve(local_state, size_request); + p = global.template reserve(size_request); if (p == nullptr) return nullptr; @@ -195,7 +195,7 @@ namespace snmalloc SNMALLOC_ASSERT(!is_meta); #endif - p = global.template reserve_with_left_over(local_state, size); + p = global.template reserve_with_left_over(size); return p; } @@ -241,8 +241,6 @@ namespace snmalloc public: using Pal = PAL; - class LocalState; - class Pagemap { friend class BackendAllocator; @@ -252,12 +250,6 @@ namespace snmalloc concretePagemap; public: - /** - * Provide a type alias for LocalState so that we can refer to it without - * needing the whole BackendAllocator type at hand. - */ - using LocalState = BackendAllocator::LocalState; - /** * Get the metadata associated with a chunk. * @@ -265,10 +257,8 @@ namespace snmalloc * to access a location that is not backed by a chunk. */ template - SNMALLOC_FAST_PATH static const MetaEntry& - get_metaentry(LocalState* ls, address_t p) + SNMALLOC_FAST_PATH static const MetaEntry& get_metaentry(address_t p) { - UNUSED(ls); return concretePagemap.template get(p); } @@ -279,10 +269,8 @@ namespace snmalloc * to access a location that is not backed by a chunk. */ template - SNMALLOC_FAST_PATH static MetaEntry& - get_metaentry_mut(LocalState* ls, address_t p) + SNMALLOC_FAST_PATH static MetaEntry& get_metaentry_mut(address_t p) { - UNUSED(ls); return concretePagemap.template get_mut(p); } @@ -290,19 +278,16 @@ namespace snmalloc * Set the metadata associated with a chunk. */ SNMALLOC_FAST_PATH - static void - set_metaentry(LocalState* ls, address_t p, size_t size, MetaEntry t) + static void set_metaentry(address_t p, size_t size, MetaEntry t) { - UNUSED(ls); for (address_t a = p; a < p + size; a += MIN_CHUNK_SIZE) { concretePagemap.set(a, t); } } - static void register_range(LocalState* ls, address_t p, size_t sz) + static void register_range(address_t p, size_t sz) { - UNUSED(ls); concretePagemap.register_range(p, sz); } @@ -314,18 +299,16 @@ namespace snmalloc template static SNMALLOC_FAST_PATH std::enable_if_t> - get_bounds(LocalState* local_state) + get_bounds() { static_assert( fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); - UNUSED(local_state); return concretePagemap.get_bounds(); } - static bool is_initialised(LocalState* ls) + static bool is_initialised() { - UNUSED(ls); return concretePagemap.is_initialised(); } }; @@ -372,15 +355,13 @@ namespace snmalloc } template - static std::enable_if_t - init(LocalState* local_state, void* base, size_t length) + static std::enable_if_t init(void* base, size_t length) { static_assert(fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); auto [heap_base, heap_length] = Pagemap::concretePagemap.init(base, length); - address_space.add_range( - local_state, capptr::Chunk(heap_base), heap_length); + address_space.add_range(capptr::Chunk(heap_base), heap_length); } /** diff --git a/src/backend/backend_concept.h b/src/backend/backend_concept.h index e4cbc91e0..9aac9ffb2 100644 --- a/src/backend/backend_concept.h +++ b/src/backend/backend_concept.h @@ -18,17 +18,16 @@ namespace snmalloc template concept ConceptBackendMeta = requires( - typename Meta::LocalState* ls, address_t addr, size_t sz, MetaEntry t) { - { Meta::set_metaentry(ls, addr, sz, t) } -> ConceptSame; + { Meta::set_metaentry(addr, sz, t) } -> ConceptSame; - { Meta::template get_metaentry(ls, addr) } + { Meta::template get_metaentry(addr) } -> ConceptSame; - { Meta::template get_metaentry(ls, addr) } + { Meta::template get_metaentry(addr) } -> ConceptSame; }; @@ -41,9 +40,9 @@ namespace snmalloc */ template concept ConceptBackendMeta_Range = - requires(typename Meta::LocalState* ls, address_t addr, size_t sz) + requires(address_t addr, size_t sz) { - { Meta::register_range(ls, addr, sz) } -> ConceptSame; + { Meta::register_range(addr, sz) } -> ConceptSame; }; /** @@ -66,7 +65,8 @@ namespace snmalloc */ template concept ConceptBackendDomestication = - requires(typename Globals::LocalState* ls, + requires( + typename Globals::LocalState* ls, capptr::AllocWild ptr) { { Globals::capptr_domesticate(ls, ptr) } @@ -96,9 +96,6 @@ namespace snmalloc std::is_base_of::value && ConceptPAL && ConceptBackendMetaRange && - ConceptSame< - typename Globals::LocalState, - typename Globals::Pagemap::LocalState> && requires() { typename Globals::LocalState; diff --git a/src/backend/fixedglobalconfig.h b/src/backend/fixedglobalconfig.h index b3290dab6..a0fe819da 100644 --- a/src/backend/fixedglobalconfig.h +++ b/src/backend/fixedglobalconfig.h @@ -49,7 +49,8 @@ namespace snmalloc static void init(typename Backend::LocalState* local_state, void* base, size_t length) { - Backend::init(local_state, base, length); + UNUSED(local_state); + Backend::init(base, length); } /* Verify that a pointer points into the region managed by this config */ @@ -66,7 +67,7 @@ namespace snmalloc UNUSED(ls); auto address = address_cast(p); - auto [base, length] = Backend::Pagemap::get_bounds(nullptr); + auto [base, length] = Backend::Pagemap::get_bounds(); if ((address - base > (length - sz)) || (length < sz)) { return nullptr; diff --git a/src/mem/chunkallocator.h b/src/mem/chunkallocator.h index 9f4de928a..3965dbd10 100644 --- a/src/mem/chunkallocator.h +++ b/src/mem/chunkallocator.h @@ -234,7 +234,7 @@ namespace snmalloc #endif MetaEntry entry{meta, remote, sizeclass}; SharedStateHandle::Pagemap::set_metaentry( - &local_state, address_cast(slab), slab_size, entry); + address_cast(slab), slab_size, entry); return {slab, meta}; } diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 66e6b7b76..760ca21ec 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -486,14 +486,14 @@ namespace snmalloc [local_state](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { return capptr_domesticate(local_state, p); }; - auto cb = [this, local_state, &need_post](freelist::HeadPtr msg) + auto cb = [this, &need_post](freelist::HeadPtr msg) SNMALLOC_FAST_PATH_LAMBDA { #ifdef SNMALLOC_TRACING std::cout << "Handling remote" << std::endl; #endif auto& entry = SharedStateHandle::Pagemap::get_metaentry( - local_state, snmalloc::address_cast(msg)); + snmalloc::address_cast(msg)); handle_dealloc_remote(entry, msg.as_void(), need_post); @@ -675,8 +675,8 @@ namespace snmalloc SNMALLOC_FAST_PATH void dealloc_local_object(CapPtr p) { - auto entry = SharedStateHandle::Pagemap::get_metaentry( - backend_state_ptr(), snmalloc::address_cast(p)); + auto entry = + SharedStateHandle::Pagemap::get_metaentry(snmalloc::address_cast(p)); if (SNMALLOC_LIKELY(dealloc_local_object_fast(entry, p, entropy))) return; @@ -848,7 +848,7 @@ namespace snmalloc bool need_post = true; // Always going to post, so ignore. auto n_tame = p_tame->atomic_read_next(key_global, domesticate); auto& entry = SharedStateHandle::Pagemap::get_metaentry( - backend_state_ptr(), snmalloc::address_cast(p_tame)); + snmalloc::address_cast(p_tame)); handle_dealloc_remote(entry, p_tame.as_void(), need_post); p_tame = n_tame; } diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index d8e1b131e..f9082e21a 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -268,8 +268,8 @@ namespace snmalloc std::cout << "Remote dealloc post" << p.unsafe_ptr() << " size " << alloc_size(p.unsafe_ptr()) << std::endl; #endif - MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry( - core_alloc->backend_state_ptr(), address_cast(p)); + MetaEntry entry = + SharedStateHandle::Pagemap::get_metaentry(address_cast(p)); local_cache.remote_dealloc_cache.template dealloc( entry.get_remote()->trunc_id(), p, key_global); post_remote_cache(); @@ -626,8 +626,8 @@ namespace snmalloc capptr::Alloc p_tame = capptr_domesticate( core_alloc->backend_state_ptr(), p_wild); - const MetaEntry& entry = SharedStateHandle::Pagemap::get_metaentry( - core_alloc->backend_state_ptr(), address_cast(p_tame)); + const MetaEntry& entry = + SharedStateHandle::Pagemap::get_metaentry(address_cast(p_tame)); if (SNMALLOC_LIKELY(local_cache.remote_allocator == entry.get_remote())) { # if defined(__CHERI_PURE_CAPABILITY__) && defined(SNMALLOC_CHECK_CLIENT) @@ -716,8 +716,8 @@ namespace snmalloc // To handle this case we require the uninitialised pagemap contain an // entry for the first chunk of memory, that states it represents a // large object, so we can pull the check for null off the fast path. - MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry( - core_alloc->backend_state_ptr(), address_cast(p_raw)); + MetaEntry entry = + SharedStateHandle::Pagemap::get_metaentry(address_cast(p_raw)); return sizeclass_full_to_size(entry.get_sizeclass()); #endif @@ -763,7 +763,7 @@ namespace snmalloc #ifndef SNMALLOC_PASS_THROUGH MetaEntry entry = SharedStateHandle::Pagemap::template get_metaentry( - core_alloc->backend_state_ptr(), address_cast(p)); + address_cast(p)); auto sizeclass = entry.get_sizeclass(); return snmalloc::remaining_bytes(sizeclass, address_cast(p)); @@ -774,8 +774,7 @@ namespace snmalloc bool check_bounds(const void* p, size_t s) { - auto ls = core_alloc->backend_state_ptr(); - if (SNMALLOC_LIKELY(SharedStateHandle::Pagemap::is_initialised(ls))) + if (SNMALLOC_LIKELY(SharedStateHandle::Pagemap::is_initialised())) { return remaining_bytes(p) >= s; } @@ -793,7 +792,7 @@ namespace snmalloc #ifndef SNMALLOC_PASS_THROUGH MetaEntry entry = SharedStateHandle::Pagemap::template get_metaentry( - core_alloc->backend_state_ptr(), address_cast(p)); + address_cast(p)); auto sizeclass = entry.get_sizeclass(); return snmalloc::index_in_object(sizeclass, address_cast(p)); diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index 2ecb05655..cafe15835 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -101,8 +101,8 @@ namespace snmalloc if (!list[i].empty()) { auto [first, last] = list[i].extract_segment(key); - MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry( - local_state, address_cast(first)); + MetaEntry entry = + SharedStateHandle::Pagemap::get_metaentry(address_cast(first)); if constexpr (SharedStateHandle::Options.QueueHeadsAreTame) { auto domesticate_nop = [](freelist::QueuePtr p) { @@ -134,8 +134,8 @@ namespace snmalloc // Use the next N bits to spread out remote deallocs in our own // slot. auto r = resend.take(key, domesticate); - MetaEntry entry = SharedStateHandle::Pagemap::get_metaentry( - local_state, address_cast(r)); + MetaEntry entry = + SharedStateHandle::Pagemap::get_metaentry(address_cast(r)); auto i = entry.get_remote()->trunc_id(); size_t slot = get_slot(i, post_round); list[slot].add(r, key); From baf35cc80efe5826a5f6794ca4c95c6483a63086 Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Fri, 11 Feb 2022 15:11:15 +0000 Subject: [PATCH 202/302] Return a small allocation from realloc(ptr, 0). (#460) An annoying amount of real-world code (e.g. mandoc, BSD sort) treats a NULL return from `realloc` as a failure, even when requesting a size of 0. This code is wrong (the standard explicitly permits a return of NULL from realloc when given a size 0) but working around it in snmalloc is easier than fixing it everywhere. --- src/mem/sizeclasstable.h | 7 ++++++- src/test/func/malloc/malloc.cc | 9 ++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index 735598748..1569bc80e 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -480,9 +480,14 @@ namespace snmalloc { return bits::next_pow2(size); } + // If realloc(ptr, 0) returns nullptr, some consumers treat this as a + // reallocation failure and abort. To avoid this, we round up the size of + // requested allocations to the smallest size class. This can be changed + // on any platform that's happy to return nullptr from realloc(ptr,0) and + // should eventually become a configuration option. if (size == 0) { - return 0; + return sizeclass_to_size(size_to_sizeclass(1)); } return sizeclass_to_size(size_to_sizeclass(size)); } diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index 5ff3f83bd..43fda4a0e 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -36,12 +36,19 @@ void check_result(size_t size, size_t align, void* p, int err, bool null) failed = true; } const auto alloc_size = our_malloc_usable_size(p); - const auto expected_size = round_size(size); + auto expected_size = round_size(size); #ifdef SNMALLOC_PASS_THROUGH // Calling system allocator may allocate a larger block than // snmalloc. Note, we have called the system allocator with // the size snmalloc would allocate, so it won't be smaller. const auto exact_size = false; + // We allocate MIN_ALLOC_SIZE byte for 0-sized allocations (and so round_size + // will tell us that the minimum size is MIN_ALLOC_SIZE), but the system + // allocator may return a 0-sized allocation. + if (size == 0) + { + expected_size = 0; + } #else const auto exact_size = align == 1; #endif From af8ab2daf6455fa1f9d9daedeb65f6abc2d2d50e Mon Sep 17 00:00:00 2001 From: Robert Norton Date: Wed, 9 Feb 2022 16:56:53 +0000 Subject: [PATCH 203/302] Clear freelist pointers on allocation for CHERI or CHECK_CLIENT builds. This is especially important on CHERI to avoid leaking capabilities to the freelist. In the CHERI case we also zero in clear_slab (see comment). Also add a check in the malloc functional test that there are no valid capabilities in the returned allocation. --- src/mem/corealloc.h | 11 +++++++++++ src/mem/freelist.h | 17 ++++++++++++++++- src/test/func/malloc/malloc.cc | 18 ++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 760ca21ec..f621e0c3d 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -339,6 +339,17 @@ namespace snmalloc address_cast(start_of_slab) == address_cast(chunk_record->meta_common.chunk)); +#if defined(__CHERI_PURE_CAPABILITY__) && !defined(SNMALLOC_CHECK_CLIENT) + // Zero the whole slab. For CHERI we at least need to clear the freelist + // pointers to avoid leaking capabilities but we do not need to do it in + // the freelist order as for SNMALLOC_CHECK_CLIENT. Zeroing the whole slab + // may be more friendly to hw because it does not involve pointer chasing + // and is amenable to prefetching. + SharedStateHandle::Pal::zero( + chunk_record->meta_common.chunk.unsafe_ptr(), + snmalloc::sizeclass_to_slab_size(sizeclass)); +#endif + #ifdef SNMALLOC_TRACING std::cout << "Slab " << start_of_slab.unsafe_ptr() << " is unused, Object sizeclass " << sizeclass << std::endl; diff --git a/src/mem/freelist.h b/src/mem/freelist.h index fbd850907..bac781cc7 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -185,6 +185,21 @@ namespace snmalloc signed_prev == this->prev_encoded, "Heap corruption - free list corrupted!"); } + + /** + * Clean up this object when removing it from the list. This is + * important on CHERI to avoid leaking capabilities. On CHECK_CLIENT + * builds it might increase the difficulty to bypass the checks. + */ + void cleanup() + { +#if defined(__CHERI_PURE_CAPABILITY__) || defined(SNMALLOC_CHECK_CLIENT) + this->next_object = nullptr; +# ifdef SNMALLOC_CHECK_CLIENT + this->prev_encoded = 0; +# endif +#endif + } }; // Note the inverted template argument order, since BView is inferable. @@ -467,7 +482,7 @@ namespace snmalloc #else UNUSED(key); #endif - + c->cleanup(); return c; } }; diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index 43fda4a0e..367be7ca8 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -60,6 +60,24 @@ void check_result(size_t size, size_t align, void* p, int err, bool null) "Cheri size is %zu, but required to be %zu.\n", cheri_size, alloc_size); failed = true; } + if (p != nullptr) + { + /* + * Scan the allocation for any tagged capabilities. Since this test doesn't + * use the allocated memory if there is a valid cap it must have leaked from + * the allocator, which is bad. + */ + void** vp = static_cast(p); + for (size_t n = 0; n < alloc_size / sizeof(*vp); vp++, n++) + { + void* c = *vp; + if (__builtin_cheri_tag_get(c)) + { + printf("Found cap tag set in alloc: %#p at %#p\n", c, vp); + failed = true; + } + } + } #endif if (exact_size && (alloc_size != expected_size) && (size != 0)) { From 86aa28644cbab2d6abab140f52561038a84daa78 Mon Sep 17 00:00:00 2001 From: Robert Norton Date: Thu, 24 Feb 2022 10:09:29 +0000 Subject: [PATCH 204/302] Errno fix (#463) Correctly set errno on failure and improve the related test. Previously the malloc test would emit an error message but not abort if the errno was not as expected on failure. This was because the return in the null == true case prevented the check for failed == true at the end of check_result from being reached. To resolve this just abort immediately as in the null case. Also add tests of allocations that are expected to fail for calloc and malloc. To make the tests pass we need to set errno in several places, making sure to keep this off the fast path. We must also take care not to attempt to zero nullptr in case of calloc failure. See microsoft/snmalloc#461 and microsoft/snmalloc#463. --- src/backend/address_space_core.h | 1 + src/mem/chunkallocator.h | 1 + src/mem/external_alloc.h | 7 ++++++- src/mem/localalloc.h | 4 ++-- src/override/malloc.cc | 4 ++++ src/pal/pal_windows.h | 7 ++++++- src/test/func/malloc/malloc.cc | 23 ++++++++++++++++------- 7 files changed, 36 insertions(+), 11 deletions(-) diff --git a/src/backend/address_space_core.h b/src/backend/address_space_core.h index fa3aa2592..533b17356 100644 --- a/src/backend/address_space_core.h +++ b/src/backend/address_space_core.h @@ -151,6 +151,7 @@ namespace snmalloc if (align_bits == (bits::BITS - 1)) { // Out of memory + errno = ENOMEM; return nullptr; } diff --git a/src/mem/chunkallocator.h b/src/mem/chunkallocator.h index 3965dbd10..88a9bcf8e 100644 --- a/src/mem/chunkallocator.h +++ b/src/mem/chunkallocator.h @@ -194,6 +194,7 @@ namespace snmalloc if (slab_sizeclass >= NUM_SLAB_SIZES) { // Your address space is not big enough for this allocation! + errno = ENOMEM; return {nullptr, nullptr}; } diff --git a/src/mem/external_alloc.h b/src/mem/external_alloc.h index 13eb2bcec..4eecfa96f 100644 --- a/src/mem/external_alloc.h +++ b/src/mem/external_alloc.h @@ -56,15 +56,20 @@ namespace snmalloc::external_alloc if constexpr (bits::BITS == 64) { if (size >= 0x10000000000) + { + errno = ENOMEM; return nullptr; + } } if (alignment < sizeof(void*)) alignment = sizeof(void*); void* result; - if (posix_memalign(&result, alignment, size) != 0) + int err = posix_memalign(&result, alignment, size); + if (err != 0) { + errno = err; result = nullptr; } return result; diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index f9082e21a..6fb378ebd 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -198,7 +198,7 @@ namespace snmalloc if (meta != nullptr) meta->initialise_large(); - if (zero_mem == YesZero) + if (zero_mem == YesZero && chunk.unsafe_ptr() != nullptr) { SharedStateHandle::Pal::template zero( chunk.unsafe_ptr(), size); @@ -427,7 +427,7 @@ namespace snmalloc // would guarantee. void* result = external_alloc::aligned_alloc( natural_alignment(size), round_size(size)); - if constexpr (zero_mem == YesZero) + if (zero_mem == YesZero && result != nullptr) memset(result, 0, size); return result; #else diff --git a/src/override/malloc.cc b/src/override/malloc.cc index a04e1dc45..de0eadc9d 100644 --- a/src/override/malloc.cc +++ b/src/override/malloc.cc @@ -98,6 +98,10 @@ extern "C" { a.dealloc(ptr); } + else + { + errno = ENOMEM; + } return p; } diff --git a/src/pal/pal_windows.h b/src/pal/pal_windows.h index 1667dce9f..6f884203f 100644 --- a/src/pal/pal_windows.h +++ b/src/pal/pal_windows.h @@ -186,13 +186,18 @@ namespace snmalloc void* ret = VirtualAlloc2FromApp( nullptr, nullptr, size, flags, PAGE_READWRITE, ¶m, 1); + if (ret == nullptr) + errno = ENOMEM; return ret; } # endif static void* reserve(size_t size) noexcept { - return VirtualAlloc(nullptr, size, MEM_RESERVE, PAGE_READWRITE); + void* ret = VirtualAlloc(nullptr, size, MEM_RESERVE, PAGE_READWRITE); + if (ret == nullptr) + errno = ENOMEM; + return ret; } /** diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index 367be7ca8..a5b0acbc8 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -16,8 +16,9 @@ void check_result(size_t size, size_t align, void* p, int err, bool null) bool failed = false; if (errno != err && err != SUCCESS) { + // Note: successful calls are allowed to spuriously set errno printf("Expected error: %d but got %d\n", err, errno); - failed = true; + abort(); } if (null) @@ -238,6 +239,11 @@ int main(int argc, char** argv) our_free(nullptr); + /* A very large allocation size that we expect to fail. */ + const size_t too_big_size = ((size_t)-1) / 2; + check_result(too_big_size, 1, our_malloc(too_big_size), ENOMEM, true); + errno = SUCCESS; + for (smallsizeclass_t sc = 0; sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) { const size_t size = bits::one_at_bit(sc); @@ -252,6 +258,9 @@ int main(int argc, char** argv) our_free(nullptr); + test_calloc(1, too_big_size, ENOMEM, true); + errno = SUCCESS; + for (smallsizeclass_t sc = 0; sc < NUM_SMALL_SIZECLASSES; sc++) { const size_t size = sizeclass_to_size(sc); @@ -275,7 +284,7 @@ int main(int argc, char** argv) const size_t size = sizeclass_to_size(sc); test_realloc(our_malloc(size), size, SUCCESS, false); test_realloc(nullptr, size, SUCCESS, false); - test_realloc(our_malloc(size), ((size_t)-1) / 2, ENOMEM, true); + test_realloc(our_malloc(size), too_big_size, ENOMEM, true); for (smallsizeclass_t sc2 = 0; sc2 < NUM_SMALL_SIZECLASSES; sc2++) { const size_t size2 = sizeclass_to_size(sc2); @@ -289,7 +298,7 @@ int main(int argc, char** argv) const size_t size = bits::one_at_bit(sc); test_realloc(our_malloc(size), size, SUCCESS, false); test_realloc(nullptr, size, SUCCESS, false); - test_realloc(our_malloc(size), ((size_t)-1) / 2, ENOMEM, true); + test_realloc(our_malloc(size), too_big_size, ENOMEM, true); for (smallsizeclass_t sc2 = 0; sc2 < (MAX_SMALL_SIZECLASS_BITS + 4); sc2++) { const size_t size2 = bits::one_at_bit(sc2); @@ -302,7 +311,7 @@ int main(int argc, char** argv) test_realloc(our_malloc(64), 4194304, SUCCESS, false); test_posix_memalign(0, 0, EINVAL, true); - test_posix_memalign(((size_t)-1) / 2, 0, EINVAL, true); + test_posix_memalign(too_big_size, 0, EINVAL, true); test_posix_memalign(OS_PAGE_SIZE, sizeof(uintptr_t) / 2, EINVAL, true); for (size_t align = sizeof(uintptr_t); align < MAX_SMALL_SIZECLASS_SIZE * 8; @@ -316,7 +325,7 @@ int main(int argc, char** argv) test_memalign(size, align, SUCCESS, false); } test_posix_memalign(0, align, SUCCESS, false); - test_posix_memalign(((size_t)-1) / 2, align, ENOMEM, true); + test_posix_memalign(too_big_size, align, ENOMEM, true); test_posix_memalign(0, align + 1, EINVAL, true); } @@ -327,7 +336,7 @@ int main(int argc, char** argv) test_reallocarray(our_malloc(size), 1, size, SUCCESS, false); test_reallocarray(our_malloc(size), 1, 0, SUCCESS, false); test_reallocarray(nullptr, 1, size, SUCCESS, false); - test_reallocarray(our_malloc(size), 1, ((size_t)-1) / 2, ENOMEM, true); + test_reallocarray(our_malloc(size), 1, too_big_size, ENOMEM, true); for (smallsizeclass_t sc2 = 0; sc2 < (MAX_SMALL_SIZECLASS_BITS + 4); sc2++) { const size_t size2 = bits::one_at_bit(sc2); @@ -351,7 +360,7 @@ int main(int argc, char** argv) printf("realloc alloc failed with %zu\n", size); abort(); } - int r = our_reallocarr(&p, 1, ((size_t)-1) / 2); + int r = our_reallocarr(&p, 1, too_big_size); if (r != ENOMEM) { printf("expected failure on allocation\n"); From 93efbb4807acf9e3d3995f891670582a152242a3 Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Fri, 25 Feb 2022 09:59:51 +0000 Subject: [PATCH 205/302] Refactor error reporting for bounds checks. (#464) This introduces a very limited formatter that can embed strings and hex representations of pointers / integers in an internal buffer. This is used to format error strings for passing to `Pal::error`. This is used, in turn, by a wrapper for reporting bounds checks, which can be used by external functions to implement bounds checks. This removes the sprintf_l usage from the bounds checks. This provides enough of a format implementation that the tests introduced in #465 can be refactored to use this, instead of their custom `printf` wrapper and that can be used by SNMALLOC_CHECK. This will be a follow-on PR. --- src/ds/helpers.h | 141 +++++++++++++++++++++++++++++++++ src/mem/bounds_checks.h | 108 +++++++++++++++++++++++++ src/mem/chunkallocator.h | 1 + src/override/memcpy.cc | 105 +----------------------- src/pal/pal.h | 26 ++++++ src/test/func/malloc/malloc.cc | 18 +++++ 6 files changed, 296 insertions(+), 103 deletions(-) create mode 100644 src/mem/bounds_checks.h diff --git a/src/ds/helpers.h b/src/ds/helpers.h index 9213294ae..095bbed98 100644 --- a/src/ds/helpers.h +++ b/src/ds/helpers.h @@ -5,6 +5,7 @@ #include #include +#include #include namespace snmalloc @@ -249,4 +250,144 @@ namespace snmalloc static_assert(sizeof(TrivialInitAtomic) == sizeof(char)); static_assert(alignof(TrivialInitAtomic) == alignof(char)); + + /** + * Helper class for building fatal errors. Used by `report_fatal_error` to + * build an on-stack buffer containing the formatted string. + */ + template + class FatalErrorBuilder + { + /** + * The buffer that is used to store the formatted output. + */ + std::array buffer; + + /** + * Space in the buffer, excluding a trailing null terminator. + */ + static constexpr size_t SafeLength = BufferSize - 1; + + /** + * The insert position within `buffer`. + */ + size_t insert = 0; + + /** + * Add argument `i` from the tuple `args` to the output. This is + * implemented recursively because the different tuple elements can have + * different types and so the code for dispatching will depend on the type + * at the index. The compiler will lower this to a jump table in optimised + * builds. + */ + template + void add_tuple_arg(size_t i, const std::tuple& args) + { + if (i == I) + { + append(std::get(args)); + } + else if constexpr (I != 0) + { + add_tuple_arg(i, args); + } + } + + /** + * Append a single character into the buffer. This is the single primitive + * operation permitted on the buffer and performs bounds checks to ensure + * that there is space for the character and for a null terminator. + */ + void append_char(char c) + { + if (insert < SafeLength) + { + buffer[insert++] = c; + } + } + + /** + * Append a string to the buffer. + */ + void append(std::string_view sv) + { + for (auto c : sv) + { + append_char(c); + } + } + + /** + * Append a raw pointer to the buffer as a hex string. + */ + void append(void* ptr) + { + append(static_cast(reinterpret_cast(ptr))); + } + + /** + * Append a size to the buffer, as a hex string. + */ + void append(size_t s) + { + append_char('0'); + append_char('x'); + std::array buf; + const char hexdigits[] = "0123456789abcdef"; + // Length of string including null terminator + static_assert(sizeof(hexdigits) == 0x11); + for (long i = long(buf.size() - 1); i >= 0; i--) + { + buf[static_cast(i)] = hexdigits[s & 0xf]; + s >>= 4; + } + bool skipZero = true; + for (auto c : buf) + { + if (skipZero && (c == '0')) + { + continue; + } + skipZero = false; + append_char(c); + } + if (skipZero) + { + append_char('0'); + } + } + + public: + /** + * Constructor. Takes a format string and the arguments to output. + */ + template + SNMALLOC_FAST_PATH FatalErrorBuilder(const char* fmt, Args... args) + { + buffer[SafeLength] = 0; + size_t arg = 0; + auto args_tuple = std::forward_as_tuple(args...); + for (const char* s = fmt; *s != 0; ++s) + { + if (s[0] == '{' && s[1] == '}') + { + add_tuple_arg(arg++, args_tuple); + ++s; + } + else + { + append_char(*s); + } + } + append_char('\0'); + } + + /** + * Return the error buffer. + */ + const char* get_message() + { + return buffer.data(); + } + }; } // namespace snmalloc diff --git a/src/mem/bounds_checks.h b/src/mem/bounds_checks.h new file mode 100644 index 000000000..bbd1d7bd0 --- /dev/null +++ b/src/mem/bounds_checks.h @@ -0,0 +1,108 @@ +#pragma once +#include "../snmalloc.h" + +namespace snmalloc +{ + /** + * Should we check loads? This defaults to on in debug builds, off in + * release (store-only checks) and can be overridden by defining the macro + * `SNMALLOC_CHECK_LOADS` to true or false. + */ + static constexpr bool CheckReads = +#ifdef SNMALLOC_CHECK_LOADS + SNMALLOC_CHECK_LOADS +#else + DEBUG +#endif + ; + + /** + * Should we fail fast when we encounter an error? With this set to true, we + * just issue a trap instruction and crash the process once we detect an + * error. With it set to false we print a helpful error message and then crash + * the process. The process may be in an undefined state by the time the + * check fails, so there are potentially security implications to turning this + * off. It defaults to true for debug builds, false for release builds and + * can be overridden by defining the macro `SNMALLOC_FAIL_FAST` to true or + * false. + */ + static constexpr bool FailFast = +#ifdef SNMALLOC_FAIL_FAST + SNMALLOC_FAIL_FAST +#else + !DEBUG +#endif + ; + + /** + * Report an error message for a failed bounds check and then abort the + * program. + * `p` is the input pointer and `len` is the offset from this pointer of the + * bounds. `msg` is the message that will be reported along with the + * start and end of the real object's bounds. + */ + SNMALLOC_SLOW_PATH SNMALLOC_UNUSED_FUNCTION inline void + report_fatal_bounds_error [[noreturn]] ( + void* p, size_t len, const char* msg, decltype(ThreadAlloc::get())& alloc) + { + report_fatal_error( + "{}: {} is in allocation {}--{}, offset {} is past the end\n", + msg, + p, + alloc.template external_pointer(p), + alloc.template external_pointer(p), + len); + } + + /** + * The direction for a bounds check. + */ + enum class CheckDirection + { + /** + * A read bounds check, performed only when read checks are enabled. + */ + Read, + + /** + * A write bounds check, performed unconditionally. + */ + Write + }; + + /** + * Check whether a pointer + length is in the same object as the pointer. + * Fail with the error message from the third argument if not. + * + * The template parameter indicates whether this is a read. If so, this + * function is a no-op when `CheckReads` is false. + */ + template + SNMALLOC_FAST_PATH_INLINE void + check_bounds(const void* ptr, size_t len, const char* msg = "") + { + if constexpr ((Direction == CheckDirection::Write) || CheckReads) + { + auto& alloc = ThreadAlloc::get(); + void* p = const_cast(ptr); + + if (SNMALLOC_UNLIKELY(!alloc.check_bounds(ptr, len))) + { + if constexpr (FailFast) + { + UNUSED(p, len, msg); + SNMALLOC_FAST_FAIL(); + } + else + { + report_fatal_bounds_error(p, len, msg, alloc); + } + } + } + else + { + UNUSED(ptr, len, msg); + } + } + +} diff --git a/src/mem/chunkallocator.h b/src/mem/chunkallocator.h index 88a9bcf8e..bd2103426 100644 --- a/src/mem/chunkallocator.h +++ b/src/mem/chunkallocator.h @@ -1,5 +1,6 @@ #pragma once +#include "../backend/backend_concept.h" #include "../ds/mpmcstack.h" #include "../ds/spmcstack.h" #include "../mem/metaslab.h" diff --git a/src/override/memcpy.cc b/src/override/memcpy.cc index acfc6a6a1..9d96eba9e 100644 --- a/src/override/memcpy.cc +++ b/src/override/memcpy.cc @@ -1,55 +1,10 @@ +#include "../mem/bounds_checks.h" #include "override.h" -#include -#include -#include -#if __has_include() -# include -#endif - using namespace snmalloc; -// glibc lacks snprintf_l -#if defined(__linux__) || defined(__OpenBSD__) || defined(__DragonFly__) || \ - defined(__HAIKU__) || defined(__sun) -# define snprintf_l(buf, size, loc, msg, ...) \ - snprintf(buf, size, msg, __VA_ARGS__) -// Windows has it with an underscore prefix -#elif defined(_MSC_VER) -# define snprintf_l(buf, size, loc, msg, ...) \ - _snprintf_s_l(buf, size, _TRUNCATE, msg, loc, __VA_ARGS__) -#endif - namespace { - /** - * Should we check loads? This defaults to on in debug builds, off in - * release (store-only checks) - */ - static constexpr bool CheckReads = -#ifdef SNMALLOC_CHECK_LOADS - SNMALLOC_CHECK_LOADS -#else - DEBUG -#endif - ; - - /** - * Should we fail fast when we encounter an error? With this set to true, we - * just issue a trap instruction and crash the process once we detect an - * error. With it set to false we print a helpful error message and then crash - * the process. The process may be in an undefined state by the time the - * check fails, so there are potentially security implications to turning this - * off. It defaults to true for debug builds, false for release builds. - */ - static constexpr bool FailFast = -#ifdef SNMALLOC_FAIL_FAST - SNMALLOC_FAIL_FAST -#else - !DEBUG -#endif - ; - /** * The largest register size that we can use for loads and stores. These * types are expected to work for overlapping copies: we can always load them @@ -92,62 +47,6 @@ namespace #endif } - SNMALLOC_SLOW_PATH SNMALLOC_UNUSED_FUNCTION void crashWithMessage - [[noreturn]] ( - void* p, size_t len, const char* msg, decltype(ThreadAlloc::get())& alloc) - { - // We're going to crash the program now, but try to avoid heap - // allocations if possible, since the heap may be in an undefined - // state. - std::array buffer; - snprintf_l( - buffer.data(), - buffer.size(), - /* Force C locale */ nullptr, - "%s: %p is in allocation %p--%p, offset 0x%zx is past the end.\n", - msg, - p, - alloc.template external_pointer(p), - alloc.template external_pointer(p), - len); - Pal::error(buffer.data()); - } - - /** - * Check whether a pointer + length is in the same object as the pointer. - * Fail with the error message from the third argument if not. - * - * The template parameter indicates whether this is a read. If so, this - * function is a no-op when `CheckReads` is false. - */ - template - SNMALLOC_FAST_PATH_INLINE void - check_bounds(const void* ptr, size_t len, const char* msg = "") - { - if constexpr (!IsRead || CheckReads) - { - auto& alloc = ThreadAlloc::get(); - void* p = const_cast(ptr); - - if (SNMALLOC_UNLIKELY(!alloc.check_bounds(ptr, len))) - { - if constexpr (FailFast) - { - UNUSED(p, len, msg); - SNMALLOC_FAST_FAIL(); - } - else - { - crashWithMessage(p, len, msg, alloc); - } - } - } - else - { - UNUSED(ptr, len, msg); - } - } - /** * Copy a block using the specified size. This copies as many complete * chunks of size `Size` as are possible from `len`. @@ -205,7 +104,7 @@ namespace // Check the bounds of the arguments. check_bounds( dst, len, "memcpy with destination out of bounds of heap allocation"); - check_bounds( + check_bounds( src, len, "memcpy with source out of bounds of heap allocation"); } // If this is a small size, do byte-by-byte copies. diff --git a/src/pal/pal.h b/src/pal/pal.h index 096df25bf..c18c16521 100644 --- a/src/pal/pal.h +++ b/src/pal/pal.h @@ -142,4 +142,30 @@ namespace snmalloc # endif "Page size from system header does not match snmalloc config page size."); #endif + + /** + * Report a fatal error via a PAL-specific error reporting mechanism. This + * takes a format string and a set of arguments. The format string indicates + * the remaining arguments with "{}". This could be extended later to + * support indexing fairly easily, if we ever want to localise these error + * messages. + * + * The following are supported as arguments: + * + * - Characters (`char`), printed verbatim. + * - Strings (anything convertible to `std::string_view`), typically string + * literals because nothing on this path should be performing heap + * allocations. Printed verbatim. + * - Raw pointers (void*), printed as hex strings. + * - Integers (convertible to `size_t`), printed as hex strings. + * + * These types should be sufficient for allocator-related error messages. + */ + template + [[noreturn]] inline void report_fatal_error(Args... args) + { + FatalErrorBuilder msg{std::forward(args)...}; + Pal::error(msg.get_message()); + } + } // namespace snmalloc diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index a5b0acbc8..542695642 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -237,6 +237,24 @@ int main(int argc, char** argv) setup(); + // Smoke test the fatal error builder. Check that it can generate strings + // including all of the kinds of things that it expects to be able to format. + void* fakeptr = reinterpret_cast(static_cast(0x42)); + FatalErrorBuilder<1024> b{ + "testing pointer {} size_t {} message, {} world, null is {}", + fakeptr, + size_t(42), + "hello", + nullptr}; + if ( + strcmp( + "testing pointer 0x42 size_t 0x2a message, hello world, null is 0x0", + b.get_message()) != 0) + { + printf("Incorrect rendering of fatal error message: %s\n", b.get_message()); + abort(); + } + our_free(nullptr); /* A very large allocation size that we expect to fail. */ From 95bd974fb0c858b2a8b7ea736af773a81a79999a Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Fri, 25 Feb 2022 15:57:28 +0000 Subject: [PATCH 206/302] Add test helper macros. (#465) - Refactor the existing SNMALLOC_ASSERT and SNMALLOC_CHECK. These now use the FatalErrorBuilder to format the output if a format string is provided. - Extend the FatalErrorBuilder to print decimal integers for signed values. - Rename FatalErrorBuilder to MessageBuilder. - Rewrite the macros used in the jemalloc tests to use FatalErrorBuilder and move them into a header. - Refactor some of the tests to use the new macros. --- src/ds/defines.h | 39 +++++++-- src/ds/helpers.h | 91 +++++++++++++++++++-- src/pal/pal.h | 4 +- src/test/func/jemalloc/jemalloc.cc | 68 ++++------------ src/test/func/malloc/malloc.cc | 122 +++++++++++----------------- src/test/func/memcpy/func-memcpy.cc | 18 +++- src/test/helpers.h | 39 +++++++++ 7 files changed, 234 insertions(+), 147 deletions(-) create mode 100644 src/test/helpers.h diff --git a/src/ds/defines.h b/src/ds/defines.h index 78304339e..0503a858f 100644 --- a/src/ds/defines.h +++ b/src/ds/defines.h @@ -1,4 +1,5 @@ #pragma once +#include #if defined(_MSC_VER) && !defined(__clang__) // 28 is FAST_FAIL_INVALID_BUFFER_ACCESS. Not using the symbolic constant to @@ -109,27 +110,40 @@ namespace snmalloc #define TOSTRING2(expr) #expr #ifdef NDEBUG -# define SNMALLOC_ASSERT(expr) \ +# define SNMALLOC_ASSERT_MSG(...) \ {} #else -# define SNMALLOC_ASSERT(expr) \ +# define SNMALLOC_ASSERT_MSG(expr, fmt, ...) \ + do \ { \ if (!(expr)) \ { \ - snmalloc::error("assert fail: " #expr " in " __FILE__ \ - " on " TOSTRING(__LINE__)); \ + snmalloc::report_fatal_error( \ + "assert fail: {} in {} on {} " fmt "\n", \ + #expr, \ + __FILE__, \ + TOSTRING(__LINE__), \ + ##__VA_ARGS__); \ } \ - } + } while (0) #endif +#define SNMALLOC_ASSERT(expr) SNMALLOC_ASSERT_MSG(expr, "") -#define SNMALLOC_CHECK(expr) \ +#define SNMALLOC_CHECK_MSG(expr, fmt, ...) \ + do \ { \ if (!(expr)) \ { \ - snmalloc::error("Check fail: " #expr " in " __FILE__ \ - " on " TOSTRING(__LINE__)); \ + snmalloc::report_fatal_error( \ + "Check fail: {} in {} on {} " fmt "\n", \ + #expr, \ + __FILE__, \ + TOSTRING(__LINE__), \ + ##__VA_ARGS__); \ } \ - } + } while (0) + +#define SNMALLOC_CHECK(expr) SNMALLOC_CHECK_MSG(expr, "") #ifndef NDEBUG # define SNMALLOC_ASSUME(x) SNMALLOC_ASSERT(x) @@ -184,6 +198,13 @@ namespace snmalloc #else static constexpr bool CHECK_CLIENT = false; #endif + + /** + * Forward declaration so that this can be called before the pal header is + * included. + */ + template + [[noreturn]] inline void report_fatal_error(Args... args); } // namespace snmalloc #ifdef SNMALLOC_CHECK_CLIENT diff --git a/src/ds/helpers.h b/src/ds/helpers.h index 095bbed98..dd4d557d2 100644 --- a/src/ds/helpers.h +++ b/src/ds/helpers.h @@ -256,7 +256,7 @@ namespace snmalloc * build an on-stack buffer containing the formatted string. */ template - class FatalErrorBuilder + class MessageBuilder { /** * The buffer that is used to store the formatted output. @@ -322,17 +322,51 @@ namespace snmalloc */ void append(void* ptr) { - append(static_cast(reinterpret_cast(ptr))); + append(static_cast(reinterpret_cast(ptr))); + // TODO: CHERI bits. + } + + /** + * Append a signed integer to the buffer, as a decimal string. + */ + void append(long long s) + { + if (s < 0) + { + append_char('-'); + s = 0 - s; + } + std::array buf; + const char digits[] = "0123456789"; + for (long i = long(buf.size() - 1); i >= 0; i--) + { + buf[static_cast(i)] = digits[s % 10]; + s /= 10; + } + bool skipZero = true; + for (auto c : buf) + { + if (skipZero && (c == '0')) + { + continue; + } + skipZero = false; + append_char(c); + } + if (skipZero) + { + append_char('0'); + } } /** * Append a size to the buffer, as a hex string. */ - void append(size_t s) + void append(unsigned long long s) { append_char('0'); append_char('x'); - std::array buf; + std::array buf; const char hexdigits[] = "0123456789abcdef"; // Length of string including null terminator static_assert(sizeof(hexdigits) == 0x11); @@ -357,12 +391,44 @@ namespace snmalloc } } + /** + * Overload to force `long` to be promoted to `long long`. + */ + void append(long x) + { + append(static_cast(x)); + } + + /** + * Overload to force `unsigned long` to be promoted to `unsigned long long`. + */ + void append(unsigned long x) + { + append(static_cast(x)); + } + + /** + * Overload to force `int` to be promoted to `long long`. + */ + void append(int x) + { + append(static_cast(x)); + } + + /** + * Overload to force `unsigned int` to be promoted to `unsigned long long`. + */ + void append(unsigned int x) + { + append(static_cast(x)); + } + public: /** * Constructor. Takes a format string and the arguments to output. */ template - SNMALLOC_FAST_PATH FatalErrorBuilder(const char* fmt, Args... args) + SNMALLOC_FAST_PATH MessageBuilder(const char* fmt, Args... args) { buffer[SafeLength] = 0; size_t arg = 0; @@ -382,6 +448,21 @@ namespace snmalloc append_char('\0'); } + /** + * Constructor for trivial format strings (no arguments). This exists to + * allow `MessageBuilder` to be used with macros without special casing + * the single-argument version. + */ + SNMALLOC_FAST_PATH MessageBuilder(const char* fmt) + { + buffer[SafeLength] = 0; + for (const char* s = fmt; *s != 0; ++s) + { + append_char(*s); + } + append_char('\0'); + } + /** * Return the error buffer. */ diff --git a/src/pal/pal.h b/src/pal/pal.h index c18c16521..c4dfd0ca0 100644 --- a/src/pal/pal.h +++ b/src/pal/pal.h @@ -161,10 +161,10 @@ namespace snmalloc * * These types should be sufficient for allocator-related error messages. */ - template + template [[noreturn]] inline void report_fatal_error(Args... args) { - FatalErrorBuilder msg{std::forward(args)...}; + MessageBuilder msg{std::forward(args)...}; Pal::error(msg.get_message()); } diff --git a/src/test/func/jemalloc/jemalloc.cc b/src/test/func/jemalloc/jemalloc.cc index 83c437708..1d0271e29 100644 --- a/src/test/func/jemalloc/jemalloc.cc +++ b/src/test/func/jemalloc/jemalloc.cc @@ -1,5 +1,6 @@ #include #include +#include #include #define SNMALLOC_NAME_MANGLE(a) our_##a @@ -59,10 +60,6 @@ # define ALLOCM_ERR_NOT_MOVED OUR_ALLOCM_ERR_NOT_MOVED #endif -#ifdef _MSC_VER -# define __PRETTY_FUNCTION__ __FUNCSIG__ -#endif - using namespace snmalloc; using namespace snmalloc::bits; @@ -91,37 +88,6 @@ namespace } } - /** - * The name of the function under test. This is set in the START_TEST macro - * and used for error reporting in EXPECT. - */ - const char* function = nullptr; - - /** - * Log that the test started. - */ -#define START_TEST(msg) \ - function = __PRETTY_FUNCTION__; \ - fprintf(stderr, "Starting test: " msg "\n"); - - /** - * An assertion that fires even in debug builds. Uses the value set by - * START_TEST. - */ -#define EXPECT(x, msg, ...) \ - if (!(x)) \ - { \ - fprintf( \ - stderr, \ - "%s:%d in %s: " msg "\n", \ - __FILE__, \ - __LINE__, \ - function, \ - ##__VA_ARGS__); \ - fflush(stderr); \ - abort(); \ - } - /** * The default maximum number of bits of address space to use for tests. * This is clamped on platforms without lazy commit because this much RAM @@ -142,7 +108,7 @@ namespace constexpr size_t low = 5; for (size_t base = low; base < Log2MaxSize; base++) { - fprintf(stderr, "\tTrying 0x%zx-byte allocations\n", one_at_bit(base)); + INFO("\tTrying {}-byte allocations\n", one_at_bit(base)); for (size_t i = 0; i < one_at_bit(low); i++) { for (int align = 1; align < 20; align++) @@ -171,13 +137,13 @@ namespace void* ptr = Mallocx(size, flags); EXPECT( ptr != nullptr, - "Failed to allocate 0x%zx bytes with %d bit alignment", + "Failed to allocate {} bytes with {}-bit alignment", size, align); size_t allocated = Sallocx(ptr, 0); EXPECT( allocated == expected, - "Expected to have allocated 0x%zx bytes, got 0x%zx bytes", + "Expected to have allocated {} bytes, got {} bytes", expected, allocated); Dallocx(ptr, 0); @@ -202,11 +168,11 @@ namespace ptr = static_cast(Rallocx(ptr, size * 2, flags)); EXPECT( ptr != nullptr, - "Failed to reallocate for 0x%zx byte allocation", + "Failed to reallocate for {} byte allocation", size * 2); EXPECT( ptr[size] == 0, - "Memory not zero initialised for 0x%zx byte reallocation from 0x%zx " + "Memory not zero initialised for {} byte reallocation from {} " "byte allocation", size * 2, size); @@ -234,11 +200,9 @@ namespace auto test = [](size_t size, int align) { int flags = MALLOCX_LG_ALIGN(align); void* ptr = Mallocx(size, flags); - EXPECT( - ptr != nullptr, "Failed to allocate for 0x%zx byte allocation", size); + EXPECT(ptr != nullptr, "Failed to allocate for zx byte allocation", size); size_t sz = Xallocx(ptr, size, 1024, flags); - EXPECT( - sz >= size, "xalloc returned 0x%zx, expected at least 0x%zx", sz, size); + EXPECT(sz >= size, "xalloc returned {}, expected at least {}", sz, size); Dallocx(ptr, 0); }; test_sizes_and_alignments(test); @@ -258,7 +222,7 @@ namespace int ret = Nallocm(&expected, size, flags); EXPECT( (ret == ALLOCM_SUCCESS), - "nallocm(%zx, %d) failed with error %d", + "nallocm({}, {}) failed with error {}", size, flags, ret); @@ -267,18 +231,18 @@ namespace ret = Allocm(&ptr, &allocated, size, flags); EXPECT( (ptr != nullptr) && (ret == ALLOCM_SUCCESS), - "Failed to allocate 0x%zx bytes with %d bit alignment", + "Failed to allocate {} bytes with {} bit alignment", size, align); EXPECT( allocated == expected, - "Expected to have allocated 0x%zx bytes, got 0x%zx bytes", + "Expected to have allocated {} bytes, got {} bytes", expected, allocated); ret = Sallocm(ptr, &expected, 0); EXPECT( (ret == ALLOCM_SUCCESS) && (allocated == expected), - "Expected to have allocated 0x%zx bytes, got 0x%zx bytes", + "Expected to have allocated {} bytes, got {} bytes", expected, allocated); @@ -301,14 +265,14 @@ namespace void* orig = ptr; EXPECT( (ptr != nullptr) && (ret == ALLOCM_SUCCESS), - "Failed to allocate 0x%zx bytes with %d bit alignment", + "Failed to allocate {} bytes with {} bit alignment", size, align); ret = Rallocm(&ptr, nullptr, allocated + 1, 12, flags | ALLOCM_NO_MOVE); EXPECT( (ret == ALLOCM_ERR_NOT_MOVED) || (ptr == orig), "Expected rallocm not to be able to move or reallocate, but return was " - "%d\n", + "{}\n", ret); Dallocm(ptr, 0); }); @@ -327,8 +291,8 @@ namespace int ret = Allocm(&ptr, nullptr, std::numeric_limits::max() / 2, 0); EXPECT( (ptr == nullptr) && (ret == OUR_ALLOCM_ERR_OOM), - "Expected massive allocation to fail with out of memory (%d), received " - "allocation %p, return code %d", + "Expected massive allocation to fail with out of memory ({}), received " + "allocation {}, return code {}", OUR_ALLOCM_ERR_OOM, ptr, ret); diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index 542695642..1ee90677b 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -1,4 +1,5 @@ #include +#include #include #define SNMALLOC_NAME_MANGLE(a) our_##a @@ -14,26 +15,20 @@ constexpr int SUCCESS = 0; void check_result(size_t size, size_t align, void* p, int err, bool null) { bool failed = false; - if (errno != err && err != SUCCESS) - { - // Note: successful calls are allowed to spuriously set errno - printf("Expected error: %d but got %d\n", err, errno); - abort(); - } - + EXPECT( + (errno == err) || (err == SUCCESS), + "Expected error: {} but got {}", + err, + errno); if (null) { - if (p != nullptr) - { - printf("Expected null, and got non-null return!\n"); - abort(); - } + EXPECT(p == nullptr, "Expected null but got {}", p); return; } if ((p == nullptr) && (size != 0)) { - printf("Unexpected null returned.\n"); + INFO("Unexpected null returned.\n"); failed = true; } const auto alloc_size = our_malloc_usable_size(p); @@ -57,8 +52,7 @@ void check_result(size_t size, size_t align, void* p, int err, bool null) const auto cheri_size = __builtin_cheri_length_get(p); if (cheri_size != alloc_size && (size != 0)) { - printf( - "Cheri size is %zu, but required to be %zu.\n", cheri_size, alloc_size); + INFO("Cheri size is {}, but required to be {}.", cheri_size, alloc_size); failed = true; } if (p != nullptr) @@ -82,16 +76,14 @@ void check_result(size_t size, size_t align, void* p, int err, bool null) #endif if (exact_size && (alloc_size != expected_size) && (size != 0)) { - printf( - "Usable size is %zu, but required to be %zu.\n", - alloc_size, - expected_size); + INFO( + "Usable size is {}, but required to be {}.", alloc_size, expected_size); failed = true; } if ((!exact_size) && (alloc_size < expected_size)) { - printf( - "Usable size is %zu, but required to be at least %zu.\n", + INFO( + "Usable size is {}, but required to be at least {}.", alloc_size, expected_size); failed = true; @@ -100,34 +92,27 @@ void check_result(size_t size, size_t align, void* p, int err, bool null) (static_cast(reinterpret_cast(p) % align) != 0) && (size != 0)) { - printf( - "Address is 0x%zx, but required to be aligned to 0x%zx.\n", - reinterpret_cast(p), - align); + INFO("Address is {}, but required to be aligned to {}.\n", p, align); failed = true; } if ( static_cast( reinterpret_cast(p) % natural_alignment(size)) != 0) { - printf( - "Address is 0x%zx, but should have natural alignment to 0x%zx.\n", - reinterpret_cast(p), + INFO( + "Address is {}, but should have natural alignment to {}.\n", + p, natural_alignment(size)); failed = true; } - if (failed) - { - printf("check_result failed! %p", p); - abort(); - } + EXPECT(!failed, "check_result failed! {}", p); our_free(p); } void test_calloc(size_t nmemb, size_t size, int err, bool null) { - printf("calloc(%zu, %zu) combined size %zu\n", nmemb, size, nmemb * size); + START_TEST("calloc({}, {}) combined size {}\n", nmemb, size, nmemb * size); errno = SUCCESS; void* p = our_calloc(nmemb, size); @@ -135,11 +120,7 @@ void test_calloc(size_t nmemb, size_t size, int err, bool null) { for (size_t i = 0; i < (size * nmemb); i++) { - if (((uint8_t*)p)[i] != 0) - { - printf("non-zero at @%zu\n", i); - abort(); - } + EXPECT(((uint8_t*)p)[i] == 0, "non-zero at {}", i); } } check_result(nmemb * size, 1, p, err, null); @@ -151,7 +132,7 @@ void test_realloc(void* p, size_t size, int err, bool null) if (p != nullptr) old_size = our_malloc_usable_size(p); - printf("realloc(%p(%zu), %zu)\n", p, old_size, size); + START_TEST("realloc({}({}), {})", p, old_size, size); errno = SUCCESS; auto new_p = our_realloc(p, size); // Realloc failure case, deallocate original block @@ -162,7 +143,7 @@ void test_realloc(void* p, size_t size, int err, bool null) void test_posix_memalign(size_t size, size_t align, int err, bool null) { - printf("posix_memalign(&p, %zu, %zu)\n", align, size); + START_TEST("posix_memalign(&p, {}, {})", align, size); void* p = nullptr; errno = our_posix_memalign(&p, align, size); check_result(size, align, p, err, null); @@ -170,7 +151,7 @@ void test_posix_memalign(size_t size, size_t align, int err, bool null) void test_memalign(size_t size, size_t align, int err, bool null) { - printf("memalign(%zu, %zu)\n", align, size); + START_TEST("memalign({}, {})", align, size); errno = SUCCESS; void* p = our_memalign(align, size); check_result(size, align, p, err, null); @@ -183,7 +164,7 @@ void test_reallocarray(void* p, size_t nmemb, size_t size, int err, bool null) if (p != nullptr) old_size = our_malloc_usable_size(p); - printf("reallocarray(%p(%zu), %zu)\n", p, old_size, tsize); + START_TEST("reallocarray({}({}), {})", p, old_size, tsize); errno = SUCCESS; auto new_p = our_reallocarray(p, nmemb, size); if (new_p == nullptr && tsize != 0) @@ -198,15 +179,11 @@ void test_reallocarr( if (size_old != (size_t)~0) p = our_malloc(size_old); + START_TEST("reallocarr({}({}), {})", p, nmemb, size); errno = SUCCESS; int r = our_reallocarr(&p, nmemb, size); - if (r != err) - { - printf("reallocarr failed! expected %d got %d\n", err, r); - abort(); - } + EXPECT(r == err, "reallocarr failed! expected {} got {}\n", err, r); - printf("reallocarr(%p(%zu), %zu)\n", p, nmemb, size); check_result(nmemb * size, 1, p, err, null); p = our_malloc(size); if (!p) @@ -221,11 +198,7 @@ void test_reallocarr( for (size_t i = 1; i < size; i++) { - if (static_cast(p)[i] != 1) - { - printf("data consistency failed! at %zu", i); - abort(); - } + EXPECT(static_cast(p)[i] == 1, "data consistency failed! at {}", i); } our_free(p); } @@ -239,16 +212,23 @@ int main(int argc, char** argv) // Smoke test the fatal error builder. Check that it can generate strings // including all of the kinds of things that it expects to be able to format. + // + // Note: We cannot use the check or assert macros here because they depend on + // `MessageBuilder` working. They are safe to use in any other test. void* fakeptr = reinterpret_cast(static_cast(0x42)); - FatalErrorBuilder<1024> b{ - "testing pointer {} size_t {} message, {} world, null is {}", + MessageBuilder<1024> b{ + "testing pointer {} size_t {} message, {} world, null is {}, -123456 is " + "{}, 1234567 is {}", fakeptr, size_t(42), "hello", - nullptr}; + nullptr, + -123456, + 1234567}; if ( strcmp( - "testing pointer 0x42 size_t 0x2a message, hello world, null is 0x0", + "testing pointer 0x42 size_t 0x2a message, hello world, null is 0x0, " + "-123456 is -123456, 1234567 is 1234567", b.get_message()) != 0) { printf("Incorrect rendering of fatal error message: %s\n", b.get_message()); @@ -265,7 +245,7 @@ int main(int argc, char** argv) for (smallsizeclass_t sc = 0; sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) { const size_t size = bits::one_at_bit(sc); - printf("malloc: %zu\n", size); + START_TEST("malloc: {}", size); errno = SUCCESS; check_result(size, 1, our_malloc(size), SUCCESS, false); errno = SUCCESS; @@ -320,7 +300,7 @@ int main(int argc, char** argv) for (smallsizeclass_t sc2 = 0; sc2 < (MAX_SMALL_SIZECLASS_BITS + 4); sc2++) { const size_t size2 = bits::one_at_bit(sc2); - printf("size1: %zu, size2:%zu\n", size, size2); + INFO("size1: {}, size2:{}\n", size, size2); test_realloc(our_malloc(size), size2, SUCCESS, false); test_realloc(our_malloc(size + 1), size2, SUCCESS, false); } @@ -373,32 +353,22 @@ int main(int argc, char** argv) test_reallocarr(size, 1, 0, SUCCESS, false); test_reallocarr(size, 2, size, SUCCESS, false); void* p = our_malloc(size); - if (p == nullptr) - { - printf("realloc alloc failed with %zu\n", size); - abort(); - } + EXPECT(p != nullptr, "realloc alloc failed with {}", size); int r = our_reallocarr(&p, 1, too_big_size); - if (r != ENOMEM) - { - printf("expected failure on allocation\n"); - abort(); - } + EXPECT(r == ENOMEM, "expected failure on allocation\n"); our_free(p); for (smallsizeclass_t sc2 = 0; sc2 < (MAX_SMALL_SIZECLASS_BITS + 4); sc2++) { const size_t size2 = bits::one_at_bit(sc2); - printf("size1: %zu, size2:%zu\n", size, size2); + START_TEST("size1: {}, size2:{}", size, size2); test_reallocarr(size, 1, size2, SUCCESS, false); } } - if (our_malloc_usable_size(nullptr) != 0) - { - printf("malloc_usable_size(nullptr) should be zero"); - abort(); - } + EXPECT( + our_malloc_usable_size(nullptr) == 0, + "malloc_usable_size(nullptr) should be zero"); snmalloc::debug_check_empty(); return 0; diff --git a/src/test/func/memcpy/func-memcpy.cc b/src/test/func/memcpy/func-memcpy.cc index 202b303d1..03ce7614c 100644 --- a/src/test/func/memcpy/func-memcpy.cc +++ b/src/test/func/memcpy/func-memcpy.cc @@ -27,6 +27,7 @@ int main() # define my_free(x) free(x) # endif # include "override/memcpy.cc" +# include "test/helpers.h" # include # include @@ -65,6 +66,7 @@ extern "C" void abort() */ void check_size(size_t size) { + START_TEST("checking {}-byte memcpy", size); auto* s = reinterpret_cast(my_malloc(size + 1)); auto* d = reinterpret_cast(my_malloc(size + 1)); d[size] = 0; @@ -94,9 +96,13 @@ void check_size(size_t size) static_cast(i), dst[i]); } - SNMALLOC_CHECK(dst[i] == (unsigned char)i); + EXPECT( + dst[i] == (unsigned char)i, + "dst[i] == {}, i == {}", + size_t(dst[i]), + i & 0xff); } - SNMALLOC_CHECK(d[size] == 0); + EXPECT(d[size] == 0, "d[size] == {}", d[size]); } my_free(s); my_free(d); @@ -104,6 +110,8 @@ void check_size(size_t size) void check_bounds(size_t size, size_t out_of_bounds) { + START_TEST( + "memcpy bounds, size {}, {} bytes out of bounds", size, out_of_bounds); auto* s = reinterpret_cast(my_malloc(size)); auto* d = reinterpret_cast(my_malloc(size)); for (size_t i = 0; i < size; ++i) @@ -125,7 +133,11 @@ void check_bounds(size_t size, size_t out_of_bounds) bounds_error = true; } can_longjmp = false; - SNMALLOC_CHECK(bounds_error == (out_of_bounds > 0)); + EXPECT( + bounds_error == (out_of_bounds > 0), + "bounds error: {}, out_of_bounds: {}", + bounds_error, + out_of_bounds); my_free(s); my_free(d); } diff --git a/src/test/helpers.h b/src/test/helpers.h new file mode 100644 index 000000000..a4ea84da9 --- /dev/null +++ b/src/test/helpers.h @@ -0,0 +1,39 @@ +#pragma once +#ifdef _MSC_VER +# define __PRETTY_FUNCTION__ __FUNCSIG__ +#endif + +namespace snmalloc +{ + /** + * The name of the function under test. This is set in the START_TEST macro + * and used for error reporting in EXPECT. + */ + const char* current_test = ""; + + /** + * Log that the test started. + */ +#define START_TEST(msg, ...) \ + do \ + { \ + current_test = __PRETTY_FUNCTION__; \ + MessageBuilder<1024> mb{"Starting test: " msg "\n", ##__VA_ARGS__}; \ + fputs(mb.get_message(), stderr); \ + } while (0) + + /** + * An assertion that fires even in debug builds. Uses the value set by + * START_TEST. + */ +#define EXPECT(x, msg, ...) \ + SNMALLOC_CHECK_MSG(x, " in test {} " msg "\n", current_test, ##__VA_ARGS__) + +#define INFO(msg, ...) \ + do \ + { \ + MessageBuilder<1024> mb{msg "\n", ##__VA_ARGS__}; \ + fputs(mb.get_message(), stderr); \ + } while (0) + +} From c27dd10c398a9295de85454af68330bbc7097f29 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Tue, 15 Feb 2022 20:58:39 +0000 Subject: [PATCH 207/302] shared lib linkage build fix for OpenBSD. disable purposely --no-undefined for this platform. --- CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b23d067ee..131c3e6df 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -233,7 +233,9 @@ else() endif() endif() -check_linker_flag(CXX "-Wl,--no-undefined" SNMALLOC_LINKER_SUPPORT_NO_ALLOW_SHLIB_UNDEF) +if (NOT ${CMAKE_SYSTEM_NAME} MATCHES "OpenBSD") + check_linker_flag(CXX "-Wl,--no-undefined" SNMALLOC_LINKER_SUPPORT_NO_ALLOW_SHLIB_UNDEF) +endif() function(add_warning_flags name) target_compile_options(${name} PRIVATE From c73846e41808fa73552f12331a02b6b146a960fa Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Wed, 2 Mar 2022 18:11:57 +0000 Subject: [PATCH 208/302] Improve the `memcpy` implementation. (#466) This now either outperforms, or performs as well within the region of measurement noise as, FreeBSD's libc memcpy, which is hand-written assembly. This uses a jump table for small copies with a sequence of power-of-two loads and stores for each, a vector-register copy with an overlapping copy for the last chunk for medium copies and, on x86, rep movsb for large copies. The checked version still incurs a noticeable overhead. --- src/override/memcpy.cc | 214 +++++++++++++++++++++++++++++++++++------ 1 file changed, 186 insertions(+), 28 deletions(-) diff --git a/src/override/memcpy.cc b/src/override/memcpy.cc index 9d96eba9e..617f7ec42 100644 --- a/src/override/memcpy.cc +++ b/src/override/memcpy.cc @@ -5,23 +5,6 @@ using namespace snmalloc; namespace { - /** - * The largest register size that we can use for loads and stores. These - * types are expected to work for overlapping copies: we can always load them - * into a register and store them. Note that this is at the C abstract - * machine level: the compiler may spill temporaries to the stack, just not - * to the source or destination object. - */ - static constexpr size_t LargestRegisterSize = -#ifdef __AVX__ - 32 -#elif defined(__SSE__) - 16 -#else - sizeof(uint64_t) -#endif - ; - /** * Copy a single element of a specified size. Uses a compiler builtin that * expands to a single load and store. @@ -51,7 +34,7 @@ namespace * Copy a block using the specified size. This copies as many complete * chunks of size `Size` as are possible from `len`. */ - template + template SNMALLOC_FAST_PATH_INLINE void block_copy(void* dst, const void* src, size_t len) { @@ -79,18 +62,188 @@ namespace * aligned to be copied as aligned chunks of `Size` bytes. */ template - SNMALLOC_FAST_PATH bool is_aligned_memcpy(void* dst, const void* src) + SNMALLOC_FAST_PATH_INLINE bool is_aligned_memcpy(void* dst, const void* src) { return (pointer_align_down(const_cast(src)) == src) && (pointer_align_down(dst) == dst); } /** - * Snmalloc checked memcpy. + * Copy a small size (`Size` bytes) as a sequence of power-of-two-sized loads + * and stores of decreasing size. `Word` is the largest size to attempt for a + * single copy. + */ + template + SNMALLOC_FAST_PATH_INLINE void small_copy(void* dst, const void* src) + { + static_assert(bits::is_pow2(Word), "Word size must be a power of two!"); + if constexpr (Size != 0) + { + if constexpr (Size >= Word) + { + copy_one(dst, src); + small_copy( + pointer_offset(dst, Word), pointer_offset(src, Word)); + } + else + { + small_copy(dst, src); + } + } + else + { + UNUSED(src); + UNUSED(dst); + } + } + + /** + * Generate small copies for all sizes up to `Size`, using `WordSize` as the + * largest size to copy in a single operation. + */ + template + SNMALLOC_FAST_PATH_INLINE void + small_copies(void* dst, const void* src, size_t len) + { + if (len == Size) + { + small_copy(dst, src); + } + if constexpr (Size > 0) + { + small_copies(dst, src, len); + } + } + + /** + * If the source and destination are the same displacement away from being + * aligned on a `BlockSize` boundary, do a small copy to ensure alignment and + * update `src`, `dst`, and `len` to reflect the remainder that needs + * copying. + * + * Note that this, like memcpy, requires that the source and destination do + * not overlap. It unconditionally copies `BlockSize` bytes, so a subsequent + * copy may not do the right thing. + */ + template + SNMALLOC_FAST_PATH_INLINE void + unaligned_start(void*& dst, const void*& src, size_t& len) + { + constexpr size_t block_mask = BlockSize - 1; + size_t src_addr = static_cast(reinterpret_cast(src)); + size_t dst_addr = static_cast(reinterpret_cast(dst)); + size_t src_offset = src_addr & block_mask; + if ((src_offset > 0) && (src_offset == (dst_addr & block_mask))) + { + size_t disp = BlockSize - src_offset; + small_copies(dst, src, disp); + src = pointer_offset(src, disp); + dst = pointer_offset(dst, disp); + len -= disp; + } + } + + /** + * Default architecture definition. Provides sane defaults. */ - template + struct GenericArch + { + /** + * The largest register size that we can use for loads and stores. These + * types are expected to work for overlapping copies: we can always load + * them into a register and store them. Note that this is at the C abstract + * machine level: the compiler may spill temporaries to the stack, just not + * to the source or destination object. + */ + SNMALLOC_UNUSED_FUNCTION + static constexpr size_t LargestRegisterSize = + std::max(sizeof(uint64_t), sizeof(void*)); + + /** + * Hook for architecture-specific optimisations. Does nothing in the + * default case. + */ + static SNMALLOC_FAST_PATH_INLINE bool copy(void*, const void*, size_t) + { + return false; + } + }; + +#if defined(__x86_64__) || defined(_M_X64) + /** + * x86-64 architecture. Prefers SSE registers for small and medium copies + * and uses `rep movsb` for large ones. + */ + struct X86_64Arch + { + /** + * The largest register size that we can use for loads and stores. These + * types are expected to work for overlapping copies: we can always load + * them into a register and store them. Note that this is at the C abstract + * machine level: the compiler may spill temporaries to the stack, just not + * to the source or destination object. + * + * We set this to 16 unconditionally for now because using AVX registers + * imposes stronger alignment requirements that seem to not be a net win. + */ + static constexpr size_t LargestRegisterSize = 16; + + /** + * Platform-specific copy hook. For large copies, use `rep movsb`. + */ + static SNMALLOC_FAST_PATH_INLINE bool + copy(void* dst, const void* src, size_t len) + { + // The Intel optimisation manual recommends doing this for sizes >256 + // bytes on modern systems and for all sizes on very modern systems. + // Testing shows that this is somewhat overly optimistic. + if (SNMALLOC_UNLIKELY(len >= 512)) + { + // Align to cache-line boundaries if possible. + unaligned_start<64, LargestRegisterSize>(dst, src, len); + // Bulk copy. This is aggressively optimised on modern x86 cores. +# ifdef __GNUC__ + asm volatile("rep movsb" + : "+S"(src), "+D"(dst), "+c"(len) + : + : "memory"); +# elif defined(_MSC_VER) + __movsb( + static_cast(dst), + static_cast(src), + len); +# else +# error No inline assembly or rep movsb intrinsic for this compiler. +# endif + return true; + } + return false; + } + }; +#endif + + using DefaultArch = +#ifdef __x86_64__ + X86_64Arch +#else + GenericArch +#endif + ; + + /** + * Snmalloc checked memcpy. The `Arch` parameter must provide: + * + * - A `size_t` value `LargestRegisterSize`, describing the largest size to + * use for single copies. + * - A `copy` function that takes (optionally, references to) the arguments + * of `memcpy` and returns `true` if it performs a copy, `false` + * otherwise. This can be used to special-case some or all sizes for a + * particular architecture. + */ + template void* memcpy(void* dst, const void* src, size_t len) { + auto orig_dst = dst; // 0 is a very common size for memcpy and we don't need to do external // pointer checks if we hit it. It's also the fastest case, to encourage // the compiler to favour the other cases. @@ -107,15 +260,20 @@ namespace check_bounds( src, len, "memcpy with source out of bounds of heap allocation"); } - // If this is a small size, do byte-by-byte copies. - if (len < LargestRegisterSize) + + // If this is a small size, use a jump table for small sizes. + if (len <= Arch::LargestRegisterSize) { - block_copy<1>(dst, src, len); - return dst; + small_copies(dst, src, len); + } + // If there's an architecture-specific hook, try using it. Otherwise do a + // simple bulk copy loop. + else if (!Arch::copy(dst, src, len)) + { + block_copy(dst, src, len); + copy_end(dst, src, len); } - block_copy(dst, src, len); - copy_end(dst, src, len); - return dst; + return orig_dst; } } // namespace From 8fc42d16988ebeb9f7e29cd0bb55f9943eb9c67b Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Thu, 3 Mar 2022 09:08:44 +0000 Subject: [PATCH 209/302] Add Pal::message. (#467) This provides a single place for reporting messages to the user. While here, be consistent about using stderr for things that should go to stderr. We were previously using a mix of stderr and stdout. --- src/pal/pal_freebsd_kernel.h | 9 +++++++++ src/pal/pal_noalloc.h | 8 ++++++++ src/pal/pal_posix.h | 27 ++++++++++++++++++++------- src/pal/pal_windows.h | 10 ++++++++-- src/test/helpers.h | 4 ++-- src/test/setup.h | 6 +++--- 6 files changed, 50 insertions(+), 14 deletions(-) diff --git a/src/pal/pal_freebsd_kernel.h b/src/pal/pal_freebsd_kernel.h index 72f84870c..2e25d915b 100644 --- a/src/pal/pal_freebsd_kernel.h +++ b/src/pal/pal_freebsd_kernel.h @@ -28,6 +28,15 @@ namespace snmalloc * PAL supports. */ static constexpr uint64_t pal_features = AlignedAllocation; + + /** + * Report a message to console, followed by a newline. + */ + static void message(const char* const str) noexcept + { + printf("%s\n", str); + } + [[noreturn]] void error(const char* const str) { panic("snmalloc error: %s", str); diff --git a/src/pal/pal_noalloc.h b/src/pal/pal_noalloc.h index 9092b65fe..d44d1495b 100644 --- a/src/pal/pal_noalloc.h +++ b/src/pal/pal_noalloc.h @@ -48,6 +48,14 @@ namespace snmalloc BasePAL::print_stack_trace(); } + /** + * Report a message to the user. + */ + static void message(const char* const str) noexcept + { + BasePAL::message(str); + } + /** * Report a fatal error an exit. */ diff --git a/src/pal/pal_posix.h b/src/pal/pal_posix.h index 4689f6928..d3ec0b788 100644 --- a/src/pal/pal_posix.h +++ b/src/pal/pal_posix.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #if __has_include() @@ -152,12 +153,23 @@ namespace snmalloc constexpr int SIZE = 1024; void* buffer[SIZE]; auto nptrs = backtrace(buffer, SIZE); - backtrace_symbols_fd(buffer, nptrs, STDOUT_FILENO); - UNUSED(write(STDOUT_FILENO, "\n", 1)); - UNUSED(fsync(STDOUT_FILENO)); + backtrace_symbols_fd(buffer, nptrs, STDERR_FILENO); + UNUSED(write(STDERR_FILENO, "\n", 1)); + UNUSED(fsync(STDERR_FILENO)); #endif } + /** + * Report a message to standard error, followed by a newline. + */ + static void message(const char* const str) noexcept + { + void* nl = const_cast("\n"); + struct iovec iov[] = {{const_cast(str), strlen(str)}, {nl, 1}}; + UNUSED(writev(STDERR_FILENO, iov, sizeof(iov) / sizeof(struct iovec))); + UNUSED(fsync(STDERR_FILENO)); + } + /** * Report a fatal error an exit. */ @@ -167,10 +179,11 @@ namespace snmalloc /// subsequent allocation will work. /// @attention: since the program is failing, we do not guarantee that /// previous bytes in stdout will be flushed - UNUSED(write(STDOUT_FILENO, "\n", 1)); - UNUSED(write(STDOUT_FILENO, str, strlen(str))); - UNUSED(write(STDOUT_FILENO, "\n", 1)); - UNUSED(fsync(STDOUT_FILENO)); + void* nl = const_cast("\n"); + struct iovec iov[] = { + {nl, 1}, {const_cast(str), strlen(str)}, {nl, 1}}; + UNUSED(writev(STDERR_FILENO, iov, sizeof(iov) / sizeof(struct iovec))); + UNUSED(fsync(STDERR_FILENO)); print_stack_trace(); abort(); } diff --git a/src/pal/pal_windows.h b/src/pal/pal_windows.h index 6f884203f..dc1a1f6df 100644 --- a/src/pal/pal_windows.h +++ b/src/pal/pal_windows.h @@ -116,10 +116,16 @@ namespace snmalloc low_memory_callbacks.register_notification(callback); } + static void message(const char* const str) + { + fputs(str, stderr); + fputc('\n', stderr); + fflush(stderr); + } + [[noreturn]] static void error(const char* const str) { - puts(str); - fflush(stdout); + message(str); abort(); } diff --git a/src/test/helpers.h b/src/test/helpers.h index a4ea84da9..7e11ba2ac 100644 --- a/src/test/helpers.h +++ b/src/test/helpers.h @@ -19,7 +19,7 @@ namespace snmalloc { \ current_test = __PRETTY_FUNCTION__; \ MessageBuilder<1024> mb{"Starting test: " msg "\n", ##__VA_ARGS__}; \ - fputs(mb.get_message(), stderr); \ + Pal::message(mb.get_message()); \ } while (0) /** @@ -33,7 +33,7 @@ namespace snmalloc do \ { \ MessageBuilder<1024> mb{msg "\n", ##__VA_ARGS__}; \ - fputs(mb.get_message(), stderr); \ + Pal::message(mb.get_message()); \ } while (0) } diff --git a/src/test/setup.h b/src/test/setup.h index 319902ea4..2b440a623 100644 --- a/src/test/setup.h +++ b/src/test/setup.h @@ -1,4 +1,5 @@ #if defined(SNMALLOC_CI_BUILD) +# include # if defined(WIN32) # include # include @@ -64,7 +65,7 @@ void print_stack_trace() void _cdecl error(int signal) { snmalloc::UNUSED(signal); - puts("*****ABORT******"); + snmalloc::Pal::message("*****ABORT******"); print_stack_trace(); @@ -75,7 +76,7 @@ LONG WINAPI VectoredHandler(struct _EXCEPTION_POINTERS* ExceptionInfo) { snmalloc::UNUSED(ExceptionInfo); - puts("*****UNHANDLED EXCEPTION******"); + snmalloc::Pal::message("*****UNHANDLED EXCEPTION******"); print_stack_trace(); @@ -96,7 +97,6 @@ void setup() SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX); } # else -# include # include void error_handle(int signal) { From 322e74fae3e3ecd66b224f28dd2d9822551346ef Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Thu, 3 Mar 2022 11:34:32 +0000 Subject: [PATCH 210/302] [FreeBSD] Exclude unused memory from core dumps. (#469) We currently include 256+GiB zeroes in core dumps. This is a QoI issue because it causes core dumps to take a long time (even if ZFS is able to compress large runs of zeros down to almost nothing, they still take a long time to load into debuggers). This patch removes anything marked as not-using or using-but-read-only (pagemap zero pages) as excluded from core dumps. Adding __builtin_trap() to the end of `func-malloc-fast` now gives a 21 MiB core dump rather than a 256 GiB one. --- src/pal/pal_bsd_aligned.h | 3 ++- src/pal/pal_freebsd.h | 57 +++++++++++++++++++++++++++++++++++++++ src/pal/pal_posix.h | 12 ++++++++- 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/src/pal/pal_bsd_aligned.h b/src/pal/pal_bsd_aligned.h index 7f1c1ea72..c770a529a 100644 --- a/src/pal/pal_bsd_aligned.h +++ b/src/pal/pal_bsd_aligned.h @@ -44,7 +44,8 @@ namespace snmalloc nullptr, size, prot, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_ALIGNED(log2align), + MAP_PRIVATE | MAP_ANONYMOUS | MAP_ALIGNED(log2align) | + OS::extra_mmap_flags(state_using), -1, 0); diff --git a/src/pal/pal_freebsd.h b/src/pal/pal_freebsd.h index 3cd2e4fae..98fc84313 100644 --- a/src/pal/pal_freebsd.h +++ b/src/pal/pal_freebsd.h @@ -43,6 +43,63 @@ namespace snmalloc (Aal::aal_name == RISCV ? 38 : Aal::address_bits); // TODO, if we ever backport to MIPS, this should yield 39 there. + /** + * Extra mmap flags. Exclude mappings from core files if they are + * read-only or pure reservations. + */ + static int extra_mmap_flags(bool state_using) + { + return state_using ? 0 : MAP_NOCORE; + } + + /** + * Notify platform that we will not be using these pages. + * + * We use the `MADV_FREE` and `NADV_NOCORE` flags to `madvise`. The first + * allows the system to discard the page and replace it with a CoW mapping + * of the zero page. The second prevents this mapping from appearing in + * core files. + */ + static void notify_not_using(void* p, size_t size) noexcept + { + SNMALLOC_ASSERT(is_aligned_block(p, size)); + + if constexpr (DEBUG) + memset(p, 0x5a, size); + + madvise(p, size, MADV_FREE | MADV_NOCORE); + + if constexpr (PalEnforceAccess) + { + mprotect(p, size, PROT_NONE); + } + } + + /** + * Notify platform that we will be using these pages for reading. + * + * This is used only for pages full of zeroes and so we exclude them from + * core dumps. + */ + static void notify_using_readonly(void* p, size_t size) noexcept + { + PALBSD_Aligned::notify_using_readonly(p, size); + madvise(p, size, MADV_NOCORE); + } + + /** + * Notify platform that we will be using these pages. + * + * We may have previously marked this memory as not being included in core + * files, so mark it for inclusion again. + */ + template + static void notify_using(void* p, size_t size) noexcept + { + PALBSD_Aligned::notify_using(p, size); + madvise(p, size, MADV_CORE); + } + # if defined(__CHERI_PURE_CAPABILITY__) static_assert( aal_supports, diff --git a/src/pal/pal_posix.h b/src/pal/pal_posix.h index d3ec0b788..242125f68 100644 --- a/src/pal/pal_posix.h +++ b/src/pal/pal_posix.h @@ -299,6 +299,15 @@ namespace snmalloc bzero(p, size); } + /** + * Extension point to allow subclasses to provide extra mmap flags. The + * argument indicates whether the memory should be in use or not. + */ + static int extra_mmap_flags(bool) + { + return 0; + } + /** * Reserve memory. * @@ -319,7 +328,8 @@ namespace snmalloc nullptr, size, prot, - MAP_PRIVATE | MAP_ANONYMOUS | DefaultMMAPFlags::flags, + MAP_PRIVATE | MAP_ANONYMOUS | DefaultMMAPFlags::flags | + OS::extra_mmap_flags(false), AnonFD::fd, 0); From 1a75316ba27881e79e97e659314907c6004198df Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Thu, 3 Mar 2022 13:58:37 +0000 Subject: [PATCH 211/302] `madvise` arguments are not flags. (#470) You can't or together multiple `madvise` flags in a single call. --- src/pal/pal_freebsd.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pal/pal_freebsd.h b/src/pal/pal_freebsd.h index 98fc84313..66ef92565 100644 --- a/src/pal/pal_freebsd.h +++ b/src/pal/pal_freebsd.h @@ -67,7 +67,8 @@ namespace snmalloc if constexpr (DEBUG) memset(p, 0x5a, size); - madvise(p, size, MADV_FREE | MADV_NOCORE); + madvise(p, size, MADV_NOCORE); + madvise(p, size, MADV_FREE); if constexpr (PalEnforceAccess) { From cf9b6290c7c9104deaab477f7a5f074d7ee30e63 Mon Sep 17 00:00:00 2001 From: David CARLIER Date: Fri, 4 Mar 2022 11:29:11 +0000 Subject: [PATCH 212/302] =?UTF-8?q?Follow-up=20on=20#469=20for=20Linux=20t?= =?UTF-8?q?his=20time,=20overriding=20read=20only=20pages=20cod=E2=80=A6?= =?UTF-8?q?=20(#471)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Follow-up on #469 for Linux this time, overriding read only pages code path too. --- src/pal/pal_linux.h | 48 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/src/pal/pal_linux.h b/src/pal/pal_linux.h index 139a6afa2..aa28deda2 100644 --- a/src/pal/pal_linux.h +++ b/src/pal/pal_linux.h @@ -34,6 +34,14 @@ namespace snmalloc */ static constexpr int default_mmap_flags = MAP_NORESERVE; + static void* reserve(size_t size) noexcept + { + void* p = PALPOSIX::reserve(size); + if (p) + madvise(p, size, MADV_DONTDUMP); + return p; + } + /** * OS specific function for zeroing memory. * @@ -68,20 +76,40 @@ namespace snmalloc { SNMALLOC_ASSERT(is_aligned_block(p, size)); + // Fill memory so that when we switch the pages back on we don't make + // assumptions on the content. + if constexpr (DEBUG) + memset(p, 0x5a, size); + + madvise(p, size, MADV_DONTDUMP); + madvise(p, size, MADV_FREE); + if constexpr (PalEnforceAccess) { - // Fill memory so that when we switch the pages back on we don't make - // assumptions on the content. - if constexpr (DEBUG) - memset(p, 0x5a, size); - - madvise(p, size, MADV_FREE); mprotect(p, size, PROT_NONE); } - else - { - madvise(p, size, MADV_FREE); - } + } + + /** + * Notify platform that we will be using these pages for reading. + * + * This is used only for pages full of zeroes and so we exclude them from + * core dumps. + */ + static void notify_using_readonly(void* p, size_t size) noexcept + { + PALPOSIX::notify_using_readonly(p, size); + madvise(p, size, MADV_DONTDUMP); + } + + /** + * Notify platform that we will be using these pages. + */ + template + static void notify_using(void* p, size_t size) noexcept + { + PALPOSIX::notify_using(p, size); + madvise(p, size, MADV_DODUMP); } static uint64_t get_entropy64() From 18ccfdecac305b6b91f84b638addbdcbf008c1ec Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Fri, 4 Mar 2022 13:33:11 +0000 Subject: [PATCH 213/302] Refactor memcpy to allow different versions. (#472) Expose a memcpy.h that contains all of the bits of memcpy and clean up the bounds checks header so that versions with both read and write checks can coexist. --- src/mem/bounds_checks.h | 6 +- src/mem/memcpy.h | 279 +++++++++++++++++++++++++++ src/override/memcpy.cc | 281 +--------------------------- src/test/func/memcpy/func-memcpy.cc | 5 +- src/test/perf/memcpy/memcpy.cc | 8 +- 5 files changed, 294 insertions(+), 285 deletions(-) create mode 100644 src/mem/memcpy.h diff --git a/src/mem/bounds_checks.h b/src/mem/bounds_checks.h index bbd1d7bd0..2843530bc 100644 --- a/src/mem/bounds_checks.h +++ b/src/mem/bounds_checks.h @@ -77,11 +77,13 @@ namespace snmalloc * The template parameter indicates whether this is a read. If so, this * function is a no-op when `CheckReads` is false. */ - template + template< + CheckDirection Direction = CheckDirection::Write, + bool CheckBoth = CheckReads> SNMALLOC_FAST_PATH_INLINE void check_bounds(const void* ptr, size_t len, const char* msg = "") { - if constexpr ((Direction == CheckDirection::Write) || CheckReads) + if constexpr ((Direction == CheckDirection::Write) || CheckBoth) { auto& alloc = ThreadAlloc::get(); void* p = const_cast(ptr); diff --git a/src/mem/memcpy.h b/src/mem/memcpy.h new file mode 100644 index 000000000..40eea5e5d --- /dev/null +++ b/src/mem/memcpy.h @@ -0,0 +1,279 @@ +#pragma once +#include "../mem/bounds_checks.h" + +namespace snmalloc +{ + /** + * Copy a single element of a specified size. Uses a compiler builtin that + * expands to a single load and store. + */ + template + SNMALLOC_FAST_PATH_INLINE void copy_one(void* dst, const void* src) + { +#if __has_builtin(__builtin_memcpy_inline) + __builtin_memcpy_inline(dst, src, Size); +#else + // Define a structure of size `Size` that has alignment 1 and a default + // copy-assignment operator. We can then copy the data as this type. The + // compiler knows the exact width and so will generate the correct wide + // instruction for us (clang 10 and gcc 12 both generate movups for the + // 16-byte version of this when targeting SSE. + struct Block + { + char data[Size]; + }; + auto* d = static_cast(dst); + auto* s = static_cast(src); + *d = *s; +#endif + } + + /** + * Copy a block using the specified size. This copies as many complete + * chunks of size `Size` as are possible from `len`. + */ + template + SNMALLOC_FAST_PATH_INLINE void + block_copy(void* dst, const void* src, size_t len) + { + for (size_t i = 0; (i + Size) <= len; i += Size) + { + copy_one(pointer_offset(dst, i), pointer_offset(src, i)); + } + } + + /** + * Perform an overlapping copy of the end. This will copy one (potentially + * unaligned) `T` from the end of the source to the end of the destination. + * This may overlap other bits of the copy. + */ + template + SNMALLOC_FAST_PATH_INLINE void + copy_end(void* dst, const void* src, size_t len) + { + copy_one( + pointer_offset(dst, len - Size), pointer_offset(src, len - Size)); + } + + /** + * Predicate indicating whether the source and destination are sufficiently + * aligned to be copied as aligned chunks of `Size` bytes. + */ + template + SNMALLOC_FAST_PATH_INLINE bool is_aligned_memcpy(void* dst, const void* src) + { + return (pointer_align_down(const_cast(src)) == src) && + (pointer_align_down(dst) == dst); + } + + /** + * Copy a small size (`Size` bytes) as a sequence of power-of-two-sized loads + * and stores of decreasing size. `Word` is the largest size to attempt for a + * single copy. + */ + template + SNMALLOC_FAST_PATH_INLINE void small_copy(void* dst, const void* src) + { + static_assert(bits::is_pow2(Word), "Word size must be a power of two!"); + if constexpr (Size != 0) + { + if constexpr (Size >= Word) + { + copy_one(dst, src); + small_copy( + pointer_offset(dst, Word), pointer_offset(src, Word)); + } + else + { + small_copy(dst, src); + } + } + else + { + UNUSED(src); + UNUSED(dst); + } + } + + /** + * Generate small copies for all sizes up to `Size`, using `WordSize` as the + * largest size to copy in a single operation. + */ + template + SNMALLOC_FAST_PATH_INLINE void + small_copies(void* dst, const void* src, size_t len) + { + if (len == Size) + { + small_copy(dst, src); + } + if constexpr (Size > 0) + { + small_copies(dst, src, len); + } + } + + /** + * If the source and destination are the same displacement away from being + * aligned on a `BlockSize` boundary, do a small copy to ensure alignment and + * update `src`, `dst`, and `len` to reflect the remainder that needs + * copying. + * + * Note that this, like memcpy, requires that the source and destination do + * not overlap. It unconditionally copies `BlockSize` bytes, so a subsequent + * copy may not do the right thing. + */ + template + SNMALLOC_FAST_PATH_INLINE void + unaligned_start(void*& dst, const void*& src, size_t& len) + { + constexpr size_t block_mask = BlockSize - 1; + size_t src_addr = static_cast(reinterpret_cast(src)); + size_t dst_addr = static_cast(reinterpret_cast(dst)); + size_t src_offset = src_addr & block_mask; + if ((src_offset > 0) && (src_offset == (dst_addr & block_mask))) + { + size_t disp = BlockSize - src_offset; + small_copies(dst, src, disp); + src = pointer_offset(src, disp); + dst = pointer_offset(dst, disp); + len -= disp; + } + } + + /** + * Default architecture definition. Provides sane defaults. + */ + struct GenericArch + { + /** + * The largest register size that we can use for loads and stores. These + * types are expected to work for overlapping copies: we can always load + * them into a register and store them. Note that this is at the C abstract + * machine level: the compiler may spill temporaries to the stack, just not + * to the source or destination object. + */ + SNMALLOC_UNUSED_FUNCTION + static constexpr size_t LargestRegisterSize = + std::max(sizeof(uint64_t), sizeof(void*)); + + /** + * Hook for architecture-specific optimisations. Does nothing in the + * default case. + */ + static SNMALLOC_FAST_PATH_INLINE bool copy(void*, const void*, size_t) + { + return false; + } + }; + +#if defined(__x86_64__) || defined(_M_X64) + /** + * x86-64 architecture. Prefers SSE registers for small and medium copies + * and uses `rep movsb` for large ones. + */ + struct X86_64Arch + { + /** + * The largest register size that we can use for loads and stores. These + * types are expected to work for overlapping copies: we can always load + * them into a register and store them. Note that this is at the C abstract + * machine level: the compiler may spill temporaries to the stack, just not + * to the source or destination object. + * + * We set this to 16 unconditionally for now because using AVX registers + * imposes stronger alignment requirements that seem to not be a net win. + */ + static constexpr size_t LargestRegisterSize = 16; + + /** + * Platform-specific copy hook. For large copies, use `rep movsb`. + */ + static SNMALLOC_FAST_PATH_INLINE bool + copy(void* dst, const void* src, size_t len) + { + // The Intel optimisation manual recommends doing this for sizes >256 + // bytes on modern systems and for all sizes on very modern systems. + // Testing shows that this is somewhat overly optimistic. + if (SNMALLOC_UNLIKELY(len >= 512)) + { + // Align to cache-line boundaries if possible. + unaligned_start<64, LargestRegisterSize>(dst, src, len); + // Bulk copy. This is aggressively optimised on modern x86 cores. +# ifdef __GNUC__ + asm volatile("rep movsb" + : "+S"(src), "+D"(dst), "+c"(len) + : + : "memory"); +# elif defined(_MSC_VER) + __movsb( + static_cast(dst), + static_cast(src), + len); +# else +# error No inline assembly or rep movsb intrinsic for this compiler. +# endif + return true; + } + return false; + } + }; +#endif + + using DefaultArch = +#ifdef __x86_64__ + X86_64Arch +#else + GenericArch +#endif + ; + + /** + * Snmalloc checked memcpy. The `Arch` parameter must provide: + * + * - A `size_t` value `LargestRegisterSize`, describing the largest size to + * use for single copies. + * - A `copy` function that takes (optionally, references to) the arguments + * of `memcpy` and returns `true` if it performs a copy, `false` + * otherwise. This can be used to special-case some or all sizes for a + * particular architecture. + */ + template< + bool Checked, + bool ReadsChecked = CheckReads, + typename Arch = DefaultArch> + SNMALLOC_FAST_PATH_INLINE void* memcpy(void* dst, const void* src, size_t len) + { + auto orig_dst = dst; + // 0 is a very common size for memcpy and we don't need to do external + // pointer checks if we hit it. It's also the fastest case, to encourage + // the compiler to favour the other cases. + if (SNMALLOC_UNLIKELY(len == 0)) + { + return dst; + } + + if constexpr (Checked) + { + // Check the bounds of the arguments. + check_bounds( + dst, len, "memcpy with destination out of bounds of heap allocation"); + check_bounds( + src, len, "memcpy with source out of bounds of heap allocation"); + } + + // If this is a small size, use a jump table for small sizes. + if (len <= Arch::LargestRegisterSize) + { + small_copies(dst, src, len); + } + // If there's an architecture-specific hook, try using it. Otherwise do a + // simple bulk copy loop. + else if (!Arch::copy(dst, src, len)) + { + block_copy(dst, src, len); + copy_end(dst, src, len); + } + return orig_dst; + } +} // namespace diff --git a/src/override/memcpy.cc b/src/override/memcpy.cc index 617f7ec42..5de742e1d 100644 --- a/src/override/memcpy.cc +++ b/src/override/memcpy.cc @@ -1,281 +1,6 @@ -#include "../mem/bounds_checks.h" -#include "override.h" - -using namespace snmalloc; - -namespace -{ - /** - * Copy a single element of a specified size. Uses a compiler builtin that - * expands to a single load and store. - */ - template - SNMALLOC_FAST_PATH_INLINE void copy_one(void* dst, const void* src) - { -#if __has_builtin(__builtin_memcpy_inline) - __builtin_memcpy_inline(dst, src, Size); -#else - // Define a structure of size `Size` that has alignment 1 and a default - // copy-assignment operator. We can then copy the data as this type. The - // compiler knows the exact width and so will generate the correct wide - // instruction for us (clang 10 and gcc 12 both generate movups for the - // 16-byte version of this when targeting SSE. - struct Block - { - char data[Size]; - }; - auto* d = static_cast(dst); - auto* s = static_cast(src); - *d = *s; -#endif - } - - /** - * Copy a block using the specified size. This copies as many complete - * chunks of size `Size` as are possible from `len`. - */ - template - SNMALLOC_FAST_PATH_INLINE void - block_copy(void* dst, const void* src, size_t len) - { - for (size_t i = 0; (i + Size) <= len; i += Size) - { - copy_one(pointer_offset(dst, i), pointer_offset(src, i)); - } - } - - /** - * Perform an overlapping copy of the end. This will copy one (potentially - * unaligned) `T` from the end of the source to the end of the destination. - * This may overlap other bits of the copy. - */ - template - SNMALLOC_FAST_PATH_INLINE void - copy_end(void* dst, const void* src, size_t len) - { - copy_one( - pointer_offset(dst, len - Size), pointer_offset(src, len - Size)); - } - - /** - * Predicate indicating whether the source and destination are sufficiently - * aligned to be copied as aligned chunks of `Size` bytes. - */ - template - SNMALLOC_FAST_PATH_INLINE bool is_aligned_memcpy(void* dst, const void* src) - { - return (pointer_align_down(const_cast(src)) == src) && - (pointer_align_down(dst) == dst); - } - - /** - * Copy a small size (`Size` bytes) as a sequence of power-of-two-sized loads - * and stores of decreasing size. `Word` is the largest size to attempt for a - * single copy. - */ - template - SNMALLOC_FAST_PATH_INLINE void small_copy(void* dst, const void* src) - { - static_assert(bits::is_pow2(Word), "Word size must be a power of two!"); - if constexpr (Size != 0) - { - if constexpr (Size >= Word) - { - copy_one(dst, src); - small_copy( - pointer_offset(dst, Word), pointer_offset(src, Word)); - } - else - { - small_copy(dst, src); - } - } - else - { - UNUSED(src); - UNUSED(dst); - } - } +#include "mem/memcpy.h" - /** - * Generate small copies for all sizes up to `Size`, using `WordSize` as the - * largest size to copy in a single operation. - */ - template - SNMALLOC_FAST_PATH_INLINE void - small_copies(void* dst, const void* src, size_t len) - { - if (len == Size) - { - small_copy(dst, src); - } - if constexpr (Size > 0) - { - small_copies(dst, src, len); - } - } - - /** - * If the source and destination are the same displacement away from being - * aligned on a `BlockSize` boundary, do a small copy to ensure alignment and - * update `src`, `dst`, and `len` to reflect the remainder that needs - * copying. - * - * Note that this, like memcpy, requires that the source and destination do - * not overlap. It unconditionally copies `BlockSize` bytes, so a subsequent - * copy may not do the right thing. - */ - template - SNMALLOC_FAST_PATH_INLINE void - unaligned_start(void*& dst, const void*& src, size_t& len) - { - constexpr size_t block_mask = BlockSize - 1; - size_t src_addr = static_cast(reinterpret_cast(src)); - size_t dst_addr = static_cast(reinterpret_cast(dst)); - size_t src_offset = src_addr & block_mask; - if ((src_offset > 0) && (src_offset == (dst_addr & block_mask))) - { - size_t disp = BlockSize - src_offset; - small_copies(dst, src, disp); - src = pointer_offset(src, disp); - dst = pointer_offset(dst, disp); - len -= disp; - } - } - - /** - * Default architecture definition. Provides sane defaults. - */ - struct GenericArch - { - /** - * The largest register size that we can use for loads and stores. These - * types are expected to work for overlapping copies: we can always load - * them into a register and store them. Note that this is at the C abstract - * machine level: the compiler may spill temporaries to the stack, just not - * to the source or destination object. - */ - SNMALLOC_UNUSED_FUNCTION - static constexpr size_t LargestRegisterSize = - std::max(sizeof(uint64_t), sizeof(void*)); - - /** - * Hook for architecture-specific optimisations. Does nothing in the - * default case. - */ - static SNMALLOC_FAST_PATH_INLINE bool copy(void*, const void*, size_t) - { - return false; - } - }; - -#if defined(__x86_64__) || defined(_M_X64) - /** - * x86-64 architecture. Prefers SSE registers for small and medium copies - * and uses `rep movsb` for large ones. - */ - struct X86_64Arch - { - /** - * The largest register size that we can use for loads and stores. These - * types are expected to work for overlapping copies: we can always load - * them into a register and store them. Note that this is at the C abstract - * machine level: the compiler may spill temporaries to the stack, just not - * to the source or destination object. - * - * We set this to 16 unconditionally for now because using AVX registers - * imposes stronger alignment requirements that seem to not be a net win. - */ - static constexpr size_t LargestRegisterSize = 16; - - /** - * Platform-specific copy hook. For large copies, use `rep movsb`. - */ - static SNMALLOC_FAST_PATH_INLINE bool - copy(void* dst, const void* src, size_t len) - { - // The Intel optimisation manual recommends doing this for sizes >256 - // bytes on modern systems and for all sizes on very modern systems. - // Testing shows that this is somewhat overly optimistic. - if (SNMALLOC_UNLIKELY(len >= 512)) - { - // Align to cache-line boundaries if possible. - unaligned_start<64, LargestRegisterSize>(dst, src, len); - // Bulk copy. This is aggressively optimised on modern x86 cores. -# ifdef __GNUC__ - asm volatile("rep movsb" - : "+S"(src), "+D"(dst), "+c"(len) - : - : "memory"); -# elif defined(_MSC_VER) - __movsb( - static_cast(dst), - static_cast(src), - len); -# else -# error No inline assembly or rep movsb intrinsic for this compiler. -# endif - return true; - } - return false; - } - }; -#endif - - using DefaultArch = -#ifdef __x86_64__ - X86_64Arch -#else - GenericArch -#endif - ; - - /** - * Snmalloc checked memcpy. The `Arch` parameter must provide: - * - * - A `size_t` value `LargestRegisterSize`, describing the largest size to - * use for single copies. - * - A `copy` function that takes (optionally, references to) the arguments - * of `memcpy` and returns `true` if it performs a copy, `false` - * otherwise. This can be used to special-case some or all sizes for a - * particular architecture. - */ - template - void* memcpy(void* dst, const void* src, size_t len) - { - auto orig_dst = dst; - // 0 is a very common size for memcpy and we don't need to do external - // pointer checks if we hit it. It's also the fastest case, to encourage - // the compiler to favour the other cases. - if (SNMALLOC_UNLIKELY(len == 0)) - { - return dst; - } - - if constexpr (checked) - { - // Check the bounds of the arguments. - check_bounds( - dst, len, "memcpy with destination out of bounds of heap allocation"); - check_bounds( - src, len, "memcpy with source out of bounds of heap allocation"); - } - - // If this is a small size, use a jump table for small sizes. - if (len <= Arch::LargestRegisterSize) - { - small_copies(dst, src, len); - } - // If there's an architecture-specific hook, try using it. Otherwise do a - // simple bulk copy loop. - else if (!Arch::copy(dst, src, len)) - { - block_copy(dst, src, len); - copy_end(dst, src, len); - } - return orig_dst; - } -} // namespace +#include "override.h" extern "C" { @@ -285,6 +10,6 @@ extern "C" SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(memcpy)(void* dst, const void* src, size_t len) { - return memcpy(dst, src, len); + return snmalloc::memcpy(dst, src, len); } } diff --git a/src/test/func/memcpy/func-memcpy.cc b/src/test/func/memcpy/func-memcpy.cc index 03ce7614c..e99569e37 100644 --- a/src/test/func/memcpy/func-memcpy.cc +++ b/src/test/func/memcpy/func-memcpy.cc @@ -35,6 +35,8 @@ int main() # include # include +using namespace snmalloc; + /** * Jump buffer used to jump out of `abort()` for recoverable errors. */ @@ -84,7 +86,8 @@ void check_size(size_t size) { dst[i] = 0; } - my_memcpy(dst, src, sz); + void* ret = my_memcpy(dst, src, sz); + EXPECT(ret == dst, "Return value should be {}, was {}", dst, ret); for (size_t i = 0; i < sz; ++i) { if (dst[i] != static_cast(i)) diff --git a/src/test/perf/memcpy/memcpy.cc b/src/test/perf/memcpy/memcpy.cc index bfa6cb4be..b42c47443 100644 --- a/src/test/perf/memcpy/memcpy.cc +++ b/src/test/perf/memcpy/memcpy.cc @@ -1,11 +1,11 @@ +#include "mem/memcpy.h" + #include #include - -#define SNMALLOC_NAME_MANGLE(a) our_##a -#include "override/memcpy.cc" - #include +using namespace snmalloc; + struct Shape { void* object; From dec7d9c015c481dfa2d46d3a91023ce311bd2699 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Fri, 4 Mar 2022 13:43:25 +0000 Subject: [PATCH 214/302] NFC memcpy: move small_copies test to Arch::copy While small_copies seems suitable in general and, in particular, on X86, it appears to be slower than a byte-wise loop on PowerPC. --- src/mem/memcpy.h | 48 ++++++++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/src/mem/memcpy.h b/src/mem/memcpy.h index 40eea5e5d..b28e5e117 100644 --- a/src/mem/memcpy.h +++ b/src/mem/memcpy.h @@ -161,9 +161,20 @@ namespace snmalloc * Hook for architecture-specific optimisations. Does nothing in the * default case. */ - static SNMALLOC_FAST_PATH_INLINE bool copy(void*, const void*, size_t) + static SNMALLOC_FAST_PATH_INLINE void + copy(void* dst, const void* src, size_t len) { - return false; + // If this is a small size, use a jump table for small sizes. + if (len <= LargestRegisterSize) + { + small_copies(dst, src, len); + } + // Otherwise do a simple bulk copy loop. + else + { + block_copy(dst, src, len); + copy_end(dst, src, len); + } } }; @@ -189,13 +200,20 @@ namespace snmalloc /** * Platform-specific copy hook. For large copies, use `rep movsb`. */ - static SNMALLOC_FAST_PATH_INLINE bool + static SNMALLOC_FAST_PATH_INLINE void copy(void* dst, const void* src, size_t len) { + // If this is a small size, use a jump table for small sizes, like on the + // generic architecture case above. + if (len <= LargestRegisterSize) + { + small_copies(dst, src, len); + } + // The Intel optimisation manual recommends doing this for sizes >256 // bytes on modern systems and for all sizes on very modern systems. // Testing shows that this is somewhat overly optimistic. - if (SNMALLOC_UNLIKELY(len >= 512)) + else if (SNMALLOC_UNLIKELY(len >= 512)) { // Align to cache-line boundaries if possible. unaligned_start<64, LargestRegisterSize>(dst, src, len); @@ -213,9 +231,14 @@ namespace snmalloc # else # error No inline assembly or rep movsb intrinsic for this compiler. # endif - return true; } - return false; + + // Otherwise do a simple bulk copy loop. + else + { + block_copy(dst, src, len); + copy_end(dst, src, len); + } } }; #endif @@ -262,18 +285,7 @@ namespace snmalloc src, len, "memcpy with source out of bounds of heap allocation"); } - // If this is a small size, use a jump table for small sizes. - if (len <= Arch::LargestRegisterSize) - { - small_copies(dst, src, len); - } - // If there's an architecture-specific hook, try using it. Otherwise do a - // simple bulk copy loop. - else if (!Arch::copy(dst, src, len)) - { - block_copy(dst, src, len); - copy_end(dst, src, len); - } + Arch::copy(dst, src, len); return orig_dst; } } // namespace From 611d4dc6175dafbadfbecbab27e855df84feb2ff Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Fri, 4 Mar 2022 10:28:35 +0000 Subject: [PATCH 215/302] PowerPC64 memcpy arch tuning --- src/mem/memcpy.h | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/mem/memcpy.h b/src/mem/memcpy.h index b28e5e117..30eb7fbf6 100644 --- a/src/mem/memcpy.h +++ b/src/mem/memcpy.h @@ -243,9 +243,49 @@ namespace snmalloc }; #endif +#if defined(__powerpc64__) + struct PPC64Arch + { + /** + * Modern POWER machines have vector registers + */ + static constexpr size_t LargestRegisterSize = 16; + + /** + * For large copies (128 bytes or above), use a copy loop that moves up to + * 128 bytes at once with pre-loop alignment up to 64 bytes. + */ + static SNMALLOC_FAST_PATH_INLINE void + copy(void* dst, const void* src, size_t len) + { + if (len < LargestRegisterSize) + { + block_copy<1>(dst, src, len); + } + else if (SNMALLOC_UNLIKELY(len >= 128)) + { + // Eight vector operations per loop + static constexpr size_t block_size = 128; + + // Cache-line align first + unaligned_start<64, LargestRegisterSize>(dst, src, len); + block_copy(dst, src, len); + copy_end(dst, src, len); + } + else + { + block_copy(dst, src, len); + copy_end(dst, src, len); + } + } + }; +#endif + using DefaultArch = #ifdef __x86_64__ X86_64Arch +#elif defined(__powerpc64__) + PPC64Arch #else GenericArch #endif From a602643fd21203482f59545adc841d1769bfc4bd Mon Sep 17 00:00:00 2001 From: David CARLIER Date: Mon, 7 Mar 2022 11:29:59 +0000 Subject: [PATCH 216/302] g++ 12 unit test build fix. (#476) test_random_allocation, g++ sees the removal from the list of the test case as UAF. --- src/test/func/memory/memory.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index a650a7c8d..6fa16c609 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -134,8 +134,8 @@ void test_random_allocation() auto& cell = objects[index % count]; if (cell != nullptr) { - alloc.dealloc(cell); allocated.erase(cell); + alloc.dealloc(cell); cell = nullptr; alloc_count--; } From 528700045373af944aad9baaa0bcc824c8e19a43 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 11 Mar 2022 18:16:06 +0000 Subject: [PATCH 217/302] Buddy (#468) # Small changes before rewrite * Additional bit in remote allocator to prevent type confusion with the backend. * Move Chunk allocator to backend. * Improvements to RedBlack tree * Expose message from Pal # Complete backend rewrite This provides two key changes: * We use buddy allocators to allow memory to reconsolidated * The backend is factored into a series of small operations that allocate and deallocate memory. The backend now uses "Ranges", there are two ranges that don't require a parent range: * EmptyRange - Never returns any memory * PalRange - Returns memory from the platform. All other ranges require a parent range to supply memory to them. Some ranges support both allocation and deallocation, and some just deallocation. For instance, CommitRange supports both, and maps requests to the parent range, but will Commit and Decommit the memory. As the ranges perform only a single task, they are generally small and easy to follow. The two exceptions to this are the two BuddyRanges (Large and Small). Large is for CHUNK_SIZE and above blocks, while Small is for below CHUNK_SIZE blocks. Both are implemented with a buddy allocator, but the SmallBuddyRange uses in place meta-data, while the LargeBuddyRange uses the pagemap for its meta-data. This means the LargeBuddyRange can keep the majority of memory it is managing decommitted. The Backend glues together the various ranges to support the appropriate way to manage memory on the platform. --- src/backend/address_space.h | 201 ------------- src/backend/address_space_core.h | 314 -------------------- src/backend/backend.h | 400 ++++++++++---------------- src/backend/backend_concept.h | 4 +- src/backend/buddy.h | 122 ++++++++ src/{mem => backend}/chunkallocator.h | 9 + src/backend/commitrange.h | 44 +++ src/backend/commonconfig.h | 2 + src/backend/empty_range.h | 29 ++ src/backend/fixedglobalconfig.h | 9 +- src/backend/globalconfig.h | 10 +- src/backend/globalrange.h | 54 ++++ src/backend/largebuddyrange.h | 289 +++++++++++++++++++ src/backend/pagemap.h | 6 +- src/backend/pagemapregisterrange.h | 43 +++ src/backend/palrange.h | 59 ++++ src/backend/range_helpers.h | 37 +++ src/backend/smallbuddyrange.h | 226 +++++++++++++++ src/backend/statsrange.h | 68 +++++ src/backend/subrange.h | 55 ++++ src/ds/address.h | 6 +- src/ds/redblacktree.h | 168 ++++------- src/mem/corealloc.h | 44 +-- src/mem/localalloc.h | 18 +- src/mem/metaslab.h | 71 +++-- src/mem/pool.h | 2 +- src/mem/remoteallocator.h | 15 +- src/mem/remotecache.h | 15 +- src/override/malloc-extensions.cc | 6 +- src/override/rust.cc | 6 +- src/pal/pal.h | 6 + src/pal/pal_windows.h | 3 +- src/snmalloc_core.h | 1 - src/test/func/malloc/malloc.cc | 5 +- src/test/func/redblack/redblack.cc | 15 + 35 files changed, 1400 insertions(+), 962 deletions(-) delete mode 100644 src/backend/address_space.h delete mode 100644 src/backend/address_space_core.h create mode 100644 src/backend/buddy.h rename src/{mem => backend}/chunkallocator.h (97%) create mode 100644 src/backend/commitrange.h create mode 100644 src/backend/empty_range.h create mode 100644 src/backend/globalrange.h create mode 100644 src/backend/largebuddyrange.h create mode 100644 src/backend/pagemapregisterrange.h create mode 100644 src/backend/palrange.h create mode 100644 src/backend/range_helpers.h create mode 100644 src/backend/smallbuddyrange.h create mode 100644 src/backend/statsrange.h create mode 100644 src/backend/subrange.h diff --git a/src/backend/address_space.h b/src/backend/address_space.h deleted file mode 100644 index 15c1ae3eb..000000000 --- a/src/backend/address_space.h +++ /dev/null @@ -1,201 +0,0 @@ -#pragma once -#include "../ds/address.h" -#include "../ds/flaglock.h" -#include "../pal/pal.h" -#include "address_space_core.h" - -#include -#ifdef SNMALLOC_TRACING -# include -#endif - -namespace snmalloc -{ - /** - * Implements a power of two allocator, where all blocks are aligned to the - * same power of two as their size. This is what snmalloc uses to get - * alignment of very large sizeclasses. - * - * It cannot unreserve memory, so this does not require the - * usual complexity of a buddy allocator. - */ - template< - SNMALLOC_CONCEPT(ConceptPAL) PAL, - SNMALLOC_CONCEPT(ConceptBackendMetaRange) Pagemap> - class AddressSpaceManager - { - AddressSpaceManagerCore core; - - /** - * This is infrequently used code, a spin lock simplifies the code - * considerably, and should never be on the fast path. - */ - FlagWord spin_lock{}; - - public: - /** - * Returns a pointer to a block of memory of the supplied size. - * The block will be committed, if specified by the template parameter. - * The returned block is guaranteed to be aligened to the size. - * - * Only request 2^n sizes, and not less than a pointer. - * - * On StrictProvenance architectures, any underlying allocations made as - * part of satisfying the request will be registered with the provided - * arena_map for use in subsequent amplification. - */ - template - capptr::Chunk reserve(size_t size) - { -#ifdef SNMALLOC_TRACING - std::cout << "ASM reserve request:" << size << std::endl; -#endif - SNMALLOC_ASSERT(bits::is_pow2(size)); - SNMALLOC_ASSERT(size >= sizeof(void*)); - - /* - * For sufficiently large allocations with platforms that support aligned - * allocations, try asking the platform directly. - */ - if constexpr (pal_supports) - { - if (size >= PAL::minimum_alloc_size) - { - auto base = - capptr::Chunk(PAL::template reserve_aligned(size)); - Pagemap::register_range(address_cast(base), size); - return base; - } - } - - capptr::Chunk res; - { - FlagLock lock(spin_lock); - res = core.template reserve(size); - if (res == nullptr) - { - // Allocation failed ask OS for more memory - capptr::Chunk block = nullptr; - size_t block_size = 0; - if constexpr (pal_supports) - { - /* - * We will have handled the case where size >= - * minimum_alloc_size above, so we are left to handle only small - * things here. - */ - block_size = PAL::minimum_alloc_size; - - void* block_raw = PAL::template reserve_aligned(block_size); - - // It's a bit of a lie to convert without applying bounds, but the - // platform will have bounded block for us and it's better that - // the rest of our internals expect CBChunk bounds. - block = capptr::Chunk(block_raw); - } - else if constexpr (!pal_supports) - { - // Need at least 2 times the space to guarantee alignment. - bool overflow; - size_t needed_size = bits::umul(size, 2, overflow); - if (overflow) - { - return nullptr; - } - // Magic number (27) for over-allocating a block of memory - // These should be further refined based on experiments. - constexpr size_t min_size = bits::one_at_bit(27); - for (size_t size_request = bits::max(needed_size, min_size); - size_request >= needed_size; - size_request = size_request / 2) - { - block = capptr::Chunk(PAL::reserve(size_request)); - if (block != nullptr) - { - block_size = size_request; - break; - } - } - - // Ensure block is pointer aligned. - if ( - pointer_align_up(block, sizeof(void*)) != block || - bits::align_up(block_size, sizeof(void*)) > block_size) - { - auto diff = - pointer_diff(block, pointer_align_up(block, sizeof(void*))); - block_size = block_size - diff; - block_size = bits::align_down(block_size, sizeof(void*)); - } - } - if (block == nullptr) - { - return nullptr; - } - - Pagemap::register_range(address_cast(block), block_size); - - core.template add_range(block, block_size); - - // still holding lock so guaranteed to succeed. - res = core.template reserve(size); - } - } - - // Don't need lock while committing pages. - if constexpr (committed) - core.template commit_block(res, size); - - return res; - } - - /** - * Aligns block to next power of 2 above size, and unused space at the end - * of the block is retained by the address space manager. - * - * This is useful for allowing the space required for alignment to be - * used, by smaller objects. - */ - template - capptr::Chunk reserve_with_left_over(size_t size) - { - SNMALLOC_ASSERT(size >= sizeof(void*)); - - size = bits::align_up(size, sizeof(void*)); - - size_t rsize = bits::next_pow2(size); - - auto res = reserve(rsize); - - if (res != nullptr) - { - if (rsize > size) - { - FlagLock lock(spin_lock); - core.template add_range(pointer_offset(res, size), rsize - size); - } - - if constexpr (committed) - core.template commit_block(res, size); - } - return res; - } - - /** - * Default constructor. An address-space manager constructed in this way - * does not own any memory at the start and will request any that it needs - * from the PAL. - */ - AddressSpaceManager() = default; - - /** - * Add a range of memory to the address space. - * Divides blocks into power of two sizes with natural alignment - */ - void add_range(capptr::Chunk base, size_t length) - { - FlagLock lock(spin_lock); - core.template add_range(base, length); - } - }; -} // namespace snmalloc diff --git a/src/backend/address_space_core.h b/src/backend/address_space_core.h deleted file mode 100644 index 533b17356..000000000 --- a/src/backend/address_space_core.h +++ /dev/null @@ -1,314 +0,0 @@ -#pragma once -#include "../ds/address.h" -#include "../ds/flaglock.h" -#include "../mem/allocconfig.h" -#include "../mem/metaslab.h" -#include "../pal/pal.h" -#include "backend_concept.h" - -#include -#ifdef SNMALLOC_TRACING -# include -#endif - -namespace snmalloc -{ - /** - * Implements a power of two allocator, where all blocks are aligned to the - * same power of two as their size. This is what snmalloc uses to get - * alignment of very large sizeclasses. - * - * It cannot unreserve memory, so this does not require the - * usual complexity of a buddy allocator. - * - * TODO: This manages pieces of memory smaller than (1U << MIN_CHUNK_BITS) to - * source Metaslab and LocalCache objects. On CHERI, where ASLR and guard - * pages are not needed, it may be worth switching to a design where we - * bootstrap allocators with at least two embedded Metaslab-s that can be used - * to construct slabs for LocalCache and, of course, additional Metaslab - * objects. That would let us stop splitting memory below that threshold - * here, and may reduce address space fragmentation or address space committed - * to Metaslab objects in perpetuity; it could also make {set,get}_next less - * scary. - */ - template - class AddressSpaceManagerCore - { - struct FreeChunk - { - capptr::Chunk next; - }; - - /** - * Stores the blocks of address space - * - * The array indexes based on power of two size. - * - * The entries for each size form a linked list. For sizes below - * MIN_CHUNK_SIZE they are linked through the first location in the - * block of memory. For sizes of, and above, MIN_CHUNK_SIZE they are - * linked using the pagemap. We only use the smaller than MIN_CHUNK_SIZE - * allocations for meta-data, so we can be sure that the next pointers - * never occur in a blocks that are ultimately used for object allocations. - * - * bits::BITS is used for simplicity, we do not use below the pointer size, - * and large entries will be unlikely to be supported by the platform. - */ - std::array, bits::BITS> ranges = {}; - - /** - * Checks a block satisfies its invariant. - */ - inline void check_block(capptr::Chunk base, size_t align_bits) - { - SNMALLOC_ASSERT( - address_cast(base) == - bits::align_up(address_cast(base), bits::one_at_bit(align_bits))); - // All blocks need to be bigger than a pointer. - SNMALLOC_ASSERT(bits::one_at_bit(align_bits) >= sizeof(void*)); - UNUSED(base, align_bits); - } - - /** - * Set next pointer for a power of two address range. - * - * This abstracts the use of either - * - the pagemap; or - * - the first pointer word of the block - * to store the next pointer for the list of unused address space of a - * particular size. - */ - void set_next( - size_t align_bits, - capptr::Chunk base, - capptr::Chunk next) - { - if (align_bits >= MIN_CHUNK_BITS) - { - // The pagemap stores `MetaEntry`s; abuse the metaslab field to be the - // next block in the stack of blocks. - // - // The pagemap entries here have nullptr as their remote, and so other - // accesses to the pagemap (by external_pointer, for example) will not - // attempt to follow this "Metaslab" pointer. - // - // dealloc() can reject attempts to free such MetaEntry-s due to the - // zero sizeclass. - MetaEntry t(reinterpret_cast(next.unsafe_ptr()), nullptr); - Pagemap::set_metaentry(address_cast(base), 1, t); - return; - } - - base->next = next; - } - - /** - * Get next pointer for a power of two address range. - * - * This abstracts the use of either - * - the pagemap; or - * - the first pointer word of the block - * to store the next pointer for the list of unused address space of a - * particular size. - */ - capptr::Chunk - get_next(size_t align_bits, capptr::Chunk base) - { - if (align_bits >= MIN_CHUNK_BITS) - { - const MetaEntry& t = - Pagemap::template get_metaentry(address_cast(base)); - return capptr::Chunk( - reinterpret_cast(t.get_metaslab_no_remote())); - } - - return base->next; - } - - /** - * Adds a block to `ranges`. - */ - template - void add_block(size_t align_bits, capptr::Chunk base) - { - check_block(base, align_bits); - SNMALLOC_ASSERT(align_bits < 64); - - set_next(align_bits, base, ranges[align_bits]); - ranges[align_bits] = base.template as_static(); - } - - /** - * Find a block of the correct size. May split larger blocks - * to satisfy this request. - */ - template - capptr::Chunk remove_block(size_t align_bits) - { - capptr::Chunk first = ranges[align_bits]; - if (first == nullptr) - { - if (align_bits == (bits::BITS - 1)) - { - // Out of memory - errno = ENOMEM; - return nullptr; - } - - // Look for larger block and split up recursively - capptr::Chunk bigger = remove_block(align_bits + 1); - - if (SNMALLOC_UNLIKELY(bigger == nullptr)) - return nullptr; - - // This block is going to be broken up into sub CHUNK_SIZE blocks - // so we need to commit it to enable the next pointers to be used - // inside the block. - if ((align_bits + 1) == MIN_CHUNK_BITS) - { - commit_block(bigger, MIN_CHUNK_SIZE); - } - - size_t half_bigger_size = bits::one_at_bit(align_bits); - auto left_over = pointer_offset(bigger, half_bigger_size); - - add_block( - align_bits, - Aal::capptr_bound( - left_over, half_bigger_size)); - check_block(left_over.as_static(), align_bits); - check_block(bigger.as_static(), align_bits); - return Aal::capptr_bound( - bigger, half_bigger_size); - } - - check_block(first, align_bits); - ranges[align_bits] = get_next(align_bits, first); - return first.as_void(); - } - - public: - /** - * Add a range of memory to the address space. - * Divides blocks into power of two sizes with natural alignment - */ - template - void add_range(capptr::Chunk base, size_t length) - { - // For start and end that are not chunk sized, we need to - // commit the pages to track the allocations. - auto base_chunk = pointer_align_up(base, MIN_CHUNK_SIZE); - auto end = pointer_offset(base, length); - auto end_chunk = pointer_align_down(end, MIN_CHUNK_SIZE); - auto start_length = pointer_diff(base, base_chunk); - auto end_length = pointer_diff(end_chunk, end); - if (start_length != 0) - commit_block(base, start_length); - if (end_length != 0) - commit_block(end_chunk, end_length); - - // Find the minimum set of maximally aligned blocks in this range. - // Each block's alignment and size are equal. - while (length >= sizeof(void*)) - { - size_t base_align_bits = bits::ctz(address_cast(base)); - size_t length_align_bits = (bits::BITS - 1) - bits::clz(length); - size_t align_bits = bits::min(base_align_bits, length_align_bits); - size_t align = bits::one_at_bit(align_bits); - - /* - * Now that we have found a maximally-aligned block, we can set bounds - * and be certain that we won't hit representation imprecision. - */ - auto b = - Aal::capptr_bound(base, align); - - check_block(b, align_bits); - add_block(align_bits, b); - - base = pointer_offset(base, align); - length -= align; - } - } - - /** - * Commit a block of memory - */ - template - void commit_block(capptr::Chunk base, size_t size) - { - // Rounding required for sub-page allocations. - auto page_start = pointer_align_down(base); - auto page_end = - pointer_align_up(pointer_offset(base, size)); - size_t using_size = pointer_diff(page_start, page_end); - PAL::template notify_using(page_start.unsafe_ptr(), using_size); - } - - /** - * Returns a pointer to a block of memory of the supplied size. - * The block will be committed, if specified by the template parameter. - * The returned block is guaranteed to be aligened to the size. - * - * Only request 2^n sizes, and not less than a pointer. - * - * On StrictProvenance architectures, any underlying allocations made as - * part of satisfying the request will be registered with the provided - * arena_map for use in subsequent amplification. - */ - template - capptr::Chunk reserve(size_t size) - { -#ifdef SNMALLOC_TRACING - std::cout << "ASM Core reserve request:" << size << std::endl; -#endif - - SNMALLOC_ASSERT(bits::is_pow2(size)); - SNMALLOC_ASSERT(size >= sizeof(void*)); - - return remove_block(bits::next_pow2_bits(size)); - } - - /** - * Aligns block to next power of 2 above size, and unused space at the end - * of the block is retained by the address space manager. - * - * This is useful for allowing the space required for alignment to be - * used by smaller objects. - */ - template - capptr::Chunk reserve_with_left_over(size_t size) - { - SNMALLOC_ASSERT(size >= sizeof(void*)); - - size = bits::align_up(size, sizeof(void*)); - - size_t rsize = bits::next_pow2(size); - - auto res = reserve(rsize); - - if (res != nullptr) - { - if (rsize > size) - { - /* - * Set bounds on the allocation requested but leave the residual with - * wider bounds for the moment; we'll fix that above in add_range. - */ - size_t residual_size = rsize - size; - auto residual = pointer_offset(res, size); - res = Aal::capptr_bound(res, size); - add_range(residual, residual_size); - } - } - return res; - } - - /** - * Default constructor. An address-space manager constructed in this way - * does not own any memory at the start and will request any that it needs - * from the PAL. - */ - AddressSpaceManagerCore() = default; - }; -} // namespace snmalloc diff --git a/src/backend/backend.h b/src/backend/backend.h index e7cff28da..c84e00c02 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -2,9 +2,19 @@ #include "../mem/allocconfig.h" #include "../mem/metaslab.h" #include "../pal/pal.h" -#include "address_space.h" +#include "chunkallocator.h" +#include "commitrange.h" #include "commonconfig.h" +#include "empty_range.h" +#include "globalrange.h" +#include "largebuddyrange.h" #include "pagemap.h" +#include "pagemapregisterrange.h" +#include "palrange.h" +#include "range_helpers.h" +#include "smallbuddyrange.h" +#include "statsrange.h" +#include "subrange.h" #if defined(SNMALLOC_CHECK_CLIENT) && !defined(OPEN_ENCLAVE) /** @@ -21,213 +31,6 @@ namespace snmalloc { - /** - * This helper class implements the core functionality to allocate from an - * address space and pagemap. Any backend implementation can use this class to - * help with basic address space managment. - */ - template< - SNMALLOC_CONCEPT(ConceptPAL) PAL, - typename LocalState, - SNMALLOC_CONCEPT(ConceptBackendMetaRange) Pagemap> - class AddressSpaceAllocatorCommon - { - // Size of local address space requests. Currently aimed at 2MiB large - // pages but should make this configurable (i.e. for OE, so we don't need as - // much space). -#ifdef OPEN_ENCLAVE - // Don't prefetch address space on OE, as it is limited. - // This could cause perf issues during warm-up phases. - constexpr static size_t LOCAL_CACHE_BLOCK = 0; -#else - constexpr static size_t LOCAL_CACHE_BLOCK = bits::one_at_bit(21); -#endif - -#ifdef SNMALLOC_META_PROTECTED - // When protecting the meta-data, we use a smaller block for the meta-data - // that is randomised inside a larger block. This needs to be at least a - // page so that we can use guard pages. - constexpr static size_t LOCAL_CACHE_META_BLOCK = - bits::max(MIN_CHUNK_SIZE * 2, OS_PAGE_SIZE); - static_assert( - LOCAL_CACHE_META_BLOCK <= LOCAL_CACHE_BLOCK, - "LOCAL_CACHE_META_BLOCK must be smaller than LOCAL_CACHE_BLOCK"); -#endif - - public: - /** - * Provide a block of meta-data with size and align. - * - * Backend allocator may use guard pages and separate area of - * address space to protect this from corruption. - */ - static capptr::Chunk alloc_meta_data( - AddressSpaceManager& global, - LocalState* local_state, - size_t size) - { - return reserve(global, local_state, size); - } - - /** - * Returns a chunk of memory with alignment and size of `size`, and a - * metaslab block. - * - * It additionally set the meta-data for this chunk of memory to - * be - * (remote, sizeclass, metaslab) - * where metaslab, is the second element of the pair return. - */ - static std::pair, Metaslab*> alloc_chunk( - AddressSpaceManager& global, - LocalState* local_state, - size_t size, - RemoteAllocator* remote, - sizeclass_t sizeclass) - { - SNMALLOC_ASSERT(bits::is_pow2(size)); - SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE); - - auto meta = reinterpret_cast( - reserve(global, local_state, sizeof(Metaslab)).unsafe_ptr()); - - if (meta == nullptr) - return {nullptr, nullptr}; - - capptr::Chunk p = reserve(global, local_state, size); - -#ifdef SNMALLOC_TRACING - std::cout << "Alloc chunk: " << p.unsafe_ptr() << " (" << size << ")" - << std::endl; -#endif - if (p == nullptr) - { - // TODO: This is leaking `meta`. Currently there is no facility for - // meta-data reuse, so will leave until we develop more expressive - // meta-data management. -#ifdef SNMALLOC_TRACING - std::cout << "Out of memory" << std::endl; -#endif - return {p, nullptr}; - } - - meta->meta_common.chunk = p; - - MetaEntry t(meta, remote, sizeclass); - Pagemap::set_metaentry(address_cast(p), size, t); - return {p, meta}; - } - - private: - /** - * Internal method for acquiring state from the local and global address - * space managers. - */ - template - static capptr::Chunk reserve( - AddressSpaceManager& global, - LocalState* local_state, - size_t size) - { -#ifdef SNMALLOC_META_PROTECTED - constexpr auto MAX_CACHED_SIZE = - is_meta ? LOCAL_CACHE_META_BLOCK : LOCAL_CACHE_BLOCK; -#else - constexpr auto MAX_CACHED_SIZE = LOCAL_CACHE_BLOCK; -#endif - - capptr::Chunk p; - if ((local_state != nullptr) && (size <= MAX_CACHED_SIZE)) - { -#ifdef SNMALLOC_META_PROTECTED - auto& local = is_meta ? local_state->local_meta_address_space : - local_state->local_address_space; -#else - auto& local = local_state->local_address_space; -#endif - - p = local.template reserve_with_left_over(size); - if (p != nullptr) - { - return p; - } - - auto refill_size = LOCAL_CACHE_BLOCK; - auto refill = global.template reserve(refill_size); - if (refill == nullptr) - return nullptr; - -#ifdef SNMALLOC_META_PROTECTED - if (is_meta) - { - refill = sub_range(refill, LOCAL_CACHE_BLOCK, LOCAL_CACHE_META_BLOCK); - refill_size = LOCAL_CACHE_META_BLOCK; - } -#endif - PAL::template notify_using(refill.unsafe_ptr(), refill_size); - local.template add_range(refill, refill_size); - - // This should succeed - return local.template reserve_with_left_over(size); - } - -#ifdef SNMALLOC_META_PROTECTED - // During start up we need meta-data before we have a local allocator - // This code protects that meta-data with randomisation, and guard pages. - if (local_state == nullptr && is_meta) - { - size_t rsize = bits::max(OS_PAGE_SIZE, bits::next_pow2(size)); - size_t size_request = rsize * 64; - - p = global.template reserve(size_request); - if (p == nullptr) - return nullptr; - - p = sub_range(p, size_request, rsize); - - PAL::template notify_using(p.unsafe_ptr(), rsize); - return p; - } - - // This path does not apply any guard pages to very large - // meta data requests. There are currently no meta data-requests - // this large. This assert checks for this assumption breaking. - SNMALLOC_ASSERT(!is_meta); -#endif - - p = global.template reserve_with_left_over(size); - return p; - } - -#ifdef SNMALLOC_META_PROTECTED - /** - * Returns a sub-range of [return, return+sub_size] that is contained in - * the range [base, base+full_size]. The first and last slot are not used - * so that the edges can be used for guard pages. - */ - static capptr::Chunk - sub_range(capptr::Chunk base, size_t full_size, size_t sub_size) - { - SNMALLOC_ASSERT(bits::is_pow2(full_size)); - SNMALLOC_ASSERT(bits::is_pow2(sub_size)); - SNMALLOC_ASSERT(full_size % sub_size == 0); - SNMALLOC_ASSERT(full_size / sub_size >= 4); - - size_t offset_mask = full_size - sub_size; - - // Don't use first or last block in the larger reservation - // Loop required to get uniform distribution. - size_t offset; - do - { - offset = get_entropy64() & offset_mask; - } while ((offset == 0) || (offset == offset_mask)); - - return pointer_offset(base, offset); - } -#endif - }; - /** * This class implements the standard backend for handling allocations. * It abstracts page table management and address space management. @@ -278,7 +81,7 @@ namespace snmalloc * Set the metadata associated with a chunk. */ SNMALLOC_FAST_PATH - static void set_metaentry(address_t p, size_t size, MetaEntry t) + static void set_metaentry(address_t p, size_t size, const MetaEntry& t) { for (address_t a = p; a < p + size; a += MIN_CHUNK_SIZE) { @@ -313,38 +116,70 @@ namespace snmalloc } }; - /** - * Local state for the backend allocator. - * - * This class contains thread local structures to make the implementation - * of the backend allocator more efficient. - */ - class LocalState - { - template< - SNMALLOC_CONCEPT(ConceptPAL) PAL2, - typename LocalState, - SNMALLOC_CONCEPT(ConceptBackendMetaRange) Pagemap2> - friend class AddressSpaceAllocatorCommon; +#if defined(_WIN32) || defined(__CHERI_PURE_CAPABILITY__) + static constexpr bool CONSOLIDATE_PAL_ALLOCS = false; +#else + static constexpr bool CONSOLIDATE_PAL_ALLOCS = true; +#endif + +#if defined(OPEN_ENCLAVE) + // Single global buddy allocator is used on open enclave due to + // the limited address space. + using GlobalR = GlobalRange>>>; + using ObjectRange = GlobalR; + using GlobalMetaRange = ObjectRange; +#else + // Set up source of memory + using P = PalRange; + using Base = std:: + conditional_t>; + // Global range of memory + using StatsR = StatsRange>; + using GlobalR = GlobalRange; + +# ifdef SNMALLOC_META_PROTECTED + // Source for object allocations + using ObjectRange = + LargeBuddyRange, 21, 21, Pagemap>; + // Set up protected range for metadata + using SubR = CommitRange, DefaultPal>; + using MetaRange = + SmallBuddyRange>; + using GlobalMetaRange = GlobalRange; +# else + // Source for object allocations and metadata + // No separation between the two + using ObjectRange = SmallBuddyRange< + LargeBuddyRange, 21, 21, Pagemap>>; + using GlobalMetaRange = GlobalRange; +# endif +#endif - AddressSpaceManagerCore local_address_space; + struct LocalState + { + typename ObjectRange::State object_range; #ifdef SNMALLOC_META_PROTECTED - /** - * Secondary local address space, so we can apply some randomisation - * and guard pages to protect the meta-data. - */ - AddressSpaceManagerCore local_meta_address_space; + typename MetaRange::State meta_range; + + typename MetaRange::State& get_meta_range() + { + return meta_range; + } +#else + typename ObjectRange::State& get_meta_range() + { + return object_range; + } #endif }; - SNMALLOC_REQUIRE_CONSTINIT - static inline AddressSpaceManager address_space; - - private: - using AddressSpaceAllocator = - AddressSpaceAllocatorCommon; - public: template static std::enable_if_t init() @@ -361,7 +196,17 @@ namespace snmalloc auto [heap_base, heap_length] = Pagemap::concretePagemap.init(base, length); - address_space.add_range(capptr::Chunk(heap_base), heap_length); + + Pagemap::register_range(address_cast(heap_base), heap_length); + + // Push memory into the global range. + range_to_pow_2_blocks( + capptr::Chunk(heap_base), + heap_length, + [&](capptr::Chunk p, size_t sz, bool) { + typename GlobalR::State g; + g->dealloc_range(p, sz); + }); } /** @@ -378,8 +223,21 @@ namespace snmalloc static capptr::Chunk alloc_meta_data(LocalState* local_state, size_t size) { - return AddressSpaceAllocator::alloc_meta_data( - address_space, local_state, size); + capptr::Chunk p; + if (local_state != nullptr) + { + p = local_state->get_meta_range()->alloc_range_with_leftover(size); + } + else + { + typename GlobalMetaRange::State global_state; + p = global_state->alloc_range(bits::next_pow2(size)); + } + + if (p == nullptr) + errno = ENOMEM; + + return p; } /** @@ -392,13 +250,73 @@ namespace snmalloc * where metaslab, is the second element of the pair return. */ static std::pair, Metaslab*> alloc_chunk( - LocalState* local_state, + LocalState& local_state, size_t size, RemoteAllocator* remote, sizeclass_t sizeclass) { - return AddressSpaceAllocator::alloc_chunk( - address_space, local_state, size, remote, sizeclass); + SNMALLOC_ASSERT(bits::is_pow2(size)); + SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE); + + auto meta_cap = + local_state.get_meta_range()->alloc_range(sizeof(Metaslab)); + + auto meta = meta_cap.template as_reinterpret().unsafe_ptr(); + + if (meta == nullptr) + { + errno = ENOMEM; + return {nullptr, nullptr}; + } + + auto p = local_state.object_range->alloc_range(size); + +#ifdef SNMALLOC_TRACING + std::cout << "Alloc chunk: " << p.unsafe_ptr() << " (" << size << ")" + << std::endl; +#endif + if (p == nullptr) + { + local_state.get_meta_range()->dealloc_range(meta_cap, sizeof(Metaslab)); + errno = ENOMEM; +#ifdef SNMALLOC_TRACING + std::cout << "Out of memory" << std::endl; +#endif + return {p, nullptr}; + } + + meta->meta_common.chunk = p; + + MetaEntry t(meta, remote, sizeclass); + Pagemap::set_metaentry(address_cast(p), size, t); + + p = Aal::capptr_bound(p, size); + return {p, meta}; + } + + static void dealloc_chunk( + LocalState& local_state, ChunkRecord* chunk_record, size_t size) + { + auto chunk = chunk_record->meta_common.chunk; + + local_state.get_meta_range()->dealloc_range( + capptr::Chunk(chunk_record), sizeof(Metaslab)); + + // TODO, should we set the sizeclass to something specific here? + + local_state.object_range->dealloc_range(chunk, size); + } + + static size_t get_current_usage() + { + typename StatsR::State stats_state; + return stats_state->get_current_usage(); + } + + static size_t get_peak_usage() + { + typename StatsR::State stats_state; + return stats_state->get_peak_usage(); } }; } // namespace snmalloc diff --git a/src/backend/backend_concept.h b/src/backend/backend_concept.h index 9aac9ffb2..895e19af2 100644 --- a/src/backend/backend_concept.h +++ b/src/backend/backend_concept.h @@ -4,7 +4,7 @@ # include # include "../ds/concept.h" # include "../pal/pal_concept.h" - +# include "../ds/address.h" namespace snmalloc { class MetaEntry; @@ -20,7 +20,7 @@ namespace snmalloc requires( address_t addr, size_t sz, - MetaEntry t) + const MetaEntry& t) { { Meta::set_metaentry(addr, sz, t) } -> ConceptSame; diff --git a/src/backend/buddy.h b/src/backend/buddy.h new file mode 100644 index 000000000..7e94c845a --- /dev/null +++ b/src/backend/buddy.h @@ -0,0 +1,122 @@ +#pragma once + +#include "../ds/address.h" +#include "../ds/bits.h" +#include "../ds/redblacktree.h" + +namespace snmalloc +{ + /** + * Class representing a buddy allocator + * + * Underlying node `Rep` representation is passed in. + * + * The allocator can handle blocks between inclusive MIN_SIZE_BITS and + * exclusive MAX_SIZE_BITS. + */ + template + class Buddy + { + std::array, MAX_SIZE_BITS - MIN_SIZE_BITS> trees; + + size_t to_index(size_t size) + { + auto log = snmalloc::bits::next_pow2_bits(size); + SNMALLOC_ASSERT(log >= MIN_SIZE_BITS); + SNMALLOC_ASSERT(log < MAX_SIZE_BITS); + + return log - MIN_SIZE_BITS; + } + + void validate_block(typename Rep::Contents addr, size_t size) + { + SNMALLOC_ASSERT(bits::is_pow2(size)); + SNMALLOC_ASSERT(addr == Rep::align_down(addr, size)); + UNUSED(addr, size); + } + + public: + constexpr Buddy() = default; + /** + * Add a block to the buddy allocator. + * + * Blocks needs to be power of two size and aligned to the same power of + * two. + * + * Returns null, if the block is successfully added. Otherwise, returns the + * consolidated block that is MAX_SIZE_BITS big, and hence too large for + * this allocator. + */ + typename Rep::Contents add_block(typename Rep::Contents addr, size_t size) + { + auto idx = to_index(size); + + validate_block(addr, size); + + auto buddy = Rep::buddy(addr, size); + + auto path = trees[idx].get_root_path(); + bool contains_buddy = trees[idx].find(path, buddy); + + if (contains_buddy) + { + // Only check if we can consolidate after we know the buddy is in + // the buddy allocator. This is required to prevent possible segfaults + // from looking at the buddies meta-data, which we only know exists + // once we have found it in the red-black tree. + if (Rep::can_consolidate(addr, size)) + { + trees[idx].remove_path(path); + + // Add to next level cache + size *= 2; + addr = Rep::align_down(addr, size); + if (size == bits::one_at_bit(MAX_SIZE_BITS)) + // Too big for this buddy allocator. + return addr; + return add_block(addr, size); + } + + // Re-traverse as the path was to the buddy, + // but the representation says we cannot combine. + // We must find the correct place for this element. + // Something clever could be done here, but it's not worth it. + // path = trees[idx].get_root_path(); + trees[idx].find(path, addr); + } + trees[idx].insert_path(path, addr); + return Rep::null; + } + + /** + * Removes a block of size from the buddy allocator. + * + * Return Rep::null if this cannot be satisfied. + */ + typename Rep::Contents remove_block(size_t size) + { + auto idx = to_index(size); + + auto addr = trees[idx].remove_min(); + if (addr != Rep::null) + { + validate_block(addr, size); + return addr; + } + + if (size * 2 == bits::one_at_bit(MAX_SIZE_BITS)) + // Too big for this buddy allocator + return Rep::null; + + auto bigger = remove_block(size * 2); + if (bigger == Rep::null) + return Rep::null; + + auto second = Rep::offset(bigger, size); + + // Split large block + add_block(second, size); + return bigger; + } + }; +} // namespace snmalloc \ No newline at end of file diff --git a/src/mem/chunkallocator.h b/src/backend/chunkallocator.h similarity index 97% rename from src/mem/chunkallocator.h rename to src/backend/chunkallocator.h index bd2103426..09e4fff24 100644 --- a/src/mem/chunkallocator.h +++ b/src/backend/chunkallocator.h @@ -1,5 +1,11 @@ #pragma once +/*** + * WARNING: This file is not currently in use. The functionality has not + * be transistioned to the new backend. It does not seem to be required + * but further testing is required before we delete it. + */ + #include "../backend/backend_concept.h" #include "../ds/mpmcstack.h" #include "../ds/spmcstack.h" @@ -316,7 +322,10 @@ namespace snmalloc SharedStateHandle::template alloc_meta_data(local_state, size); if (p == nullptr) + { + errno = ENOMEM; return nullptr; + } return new (p.unsafe_ptr()) U(std::forward(args)...); } diff --git a/src/backend/commitrange.h b/src/backend/commitrange.h new file mode 100644 index 000000000..a0032d6a7 --- /dev/null +++ b/src/backend/commitrange.h @@ -0,0 +1,44 @@ +#pragma once + +#include "../ds/ptrwrap.h" + +namespace snmalloc +{ + template + class CommitRange + { + typename ParentRange::State parent{}; + + public: + class State + { + CommitRange commit_range{}; + + public: + constexpr State() = default; + + CommitRange* operator->() + { + return &commit_range; + } + }; + + static constexpr bool Aligned = ParentRange::Aligned; + + constexpr CommitRange() = default; + + capptr::Chunk alloc_range(size_t size) + { + auto range = parent->alloc_range(size); + if (range != nullptr) + PAL::template notify_using(range.unsafe_ptr(), size); + return range; + } + + void dealloc_range(capptr::Chunk base, size_t size) + { + PAL::notify_not_using(base.unsafe_ptr(), size); + parent->dealloc_range(base, size); + } + }; +} // namespace snmalloc \ No newline at end of file diff --git a/src/backend/commonconfig.h b/src/backend/commonconfig.h index 74f949d0b..66314dbf1 100644 --- a/src/backend/commonconfig.h +++ b/src/backend/commonconfig.h @@ -1,6 +1,8 @@ #pragma once +#include "../backend/backend_concept.h" #include "../ds/defines.h" +#include "../mem/remoteallocator.h" namespace snmalloc { diff --git a/src/backend/empty_range.h b/src/backend/empty_range.h new file mode 100644 index 000000000..5feabd90e --- /dev/null +++ b/src/backend/empty_range.h @@ -0,0 +1,29 @@ +#include "../ds/ptrwrap.h" + +namespace snmalloc +{ + class EmptyRange + { + public: + class State + { + public: + EmptyRange* operator->() + { + static EmptyRange range{}; + return ⦥ + } + + constexpr State() = default; + }; + + static constexpr bool Aligned = true; + + constexpr EmptyRange() = default; + + capptr::Chunk alloc_range(size_t) + { + return nullptr; + } + }; +} // namespace snmalloc \ No newline at end of file diff --git a/src/backend/fixedglobalconfig.h b/src/backend/fixedglobalconfig.h index a0fe819da..9bb33ae5f 100644 --- a/src/backend/fixedglobalconfig.h +++ b/src/backend/fixedglobalconfig.h @@ -1,7 +1,7 @@ #pragma once #include "../backend/backend.h" -#include "../mem/chunkallocator.h" +#include "../backend/chunkallocator.h" #include "../mem/corealloc.h" #include "../mem/pool.h" #include "commonconfig.h" @@ -19,17 +19,10 @@ namespace snmalloc private: using Backend = BackendAllocator; - inline static ChunkAllocatorState chunk_allocator_state; inline static GlobalPoolState alloc_pool; public: - static ChunkAllocatorState& - get_chunk_allocator_state(typename Backend::LocalState* = nullptr) - { - return chunk_allocator_state; - } - static GlobalPoolState& pool() { return alloc_pool; diff --git a/src/backend/globalconfig.h b/src/backend/globalconfig.h index b1243a511..3b4685694 100644 --- a/src/backend/globalconfig.h +++ b/src/backend/globalconfig.h @@ -1,7 +1,7 @@ #pragma once #include "../backend/backend.h" -#include "../mem/chunkallocator.h" +#include "../backend/chunkallocator.h" #include "../mem/corealloc.h" #include "../mem/pool.h" #include "commonconfig.h" @@ -36,8 +36,6 @@ namespace snmalloc private: using Backend = BackendAllocator; - SNMALLOC_REQUIRE_CONSTINIT - inline static ChunkAllocatorState chunk_allocator_state; SNMALLOC_REQUIRE_CONSTINIT inline static GlobalPoolState alloc_pool; @@ -49,12 +47,6 @@ namespace snmalloc inline static FlagWord initialisation_lock{}; public: - static ChunkAllocatorState& - get_chunk_allocator_state(Backend::LocalState* = nullptr) - { - return chunk_allocator_state; - } - static GlobalPoolState& pool() { return alloc_pool; diff --git a/src/backend/globalrange.h b/src/backend/globalrange.h new file mode 100644 index 000000000..7b4bb0bbe --- /dev/null +++ b/src/backend/globalrange.h @@ -0,0 +1,54 @@ +#pragma once + +#include "../ds/defines.h" +#include "../ds/helpers.h" +#include "../ds/ptrwrap.h" + +namespace snmalloc +{ + /** + * Makes the supplied ParentRange into a global variable, + * and protects access with a lock. + */ + template + class GlobalRange + { + typename ParentRange::State parent{}; + + /** + * This is infrequently used code, a spin lock simplifies the code + * considerably, and should never be on the fast path. + */ + FlagWord spin_lock{}; + + public: + class State + { + SNMALLOC_REQUIRE_CONSTINIT static inline GlobalRange global_range{}; + + public: + constexpr GlobalRange* operator->() + { + return &global_range; + } + + constexpr State() = default; + }; + + static constexpr bool Aligned = ParentRange::Aligned; + + constexpr GlobalRange() = default; + + capptr::Chunk alloc_range(size_t size) + { + FlagLock lock(spin_lock); + return parent->alloc_range(size); + } + + void dealloc_range(capptr::Chunk base, size_t size) + { + FlagLock lock(spin_lock); + parent->dealloc_range(base, size); + } + }; +} // namespace snmalloc \ No newline at end of file diff --git a/src/backend/largebuddyrange.h b/src/backend/largebuddyrange.h new file mode 100644 index 000000000..1d15dfe1f --- /dev/null +++ b/src/backend/largebuddyrange.h @@ -0,0 +1,289 @@ +#pragma once + +#include "../ds/address.h" +#include "../ds/bits.h" +#include "../mem/allocconfig.h" +#include "../mem/metaslab.h" +#include "../pal/pal.h" +#include "buddy.h" +#include "range_helpers.h" + +#include + +namespace snmalloc +{ + /** + * Class for using the pagemap entries for the buddy allocator. + */ + template + class BuddyChunkRep + { + public: + using Holder = uintptr_t; + using Contents = uintptr_t; + + static constexpr address_t RED_BIT = 2; + + static constexpr Contents null = 0; + + static void set(Holder* ptr, Contents r) + { + SNMALLOC_ASSERT((r & (MIN_CHUNK_SIZE - 1)) == 0); + // Preserve lower bits. + *ptr = r | address_cast(*ptr & (MIN_CHUNK_SIZE - 1)) | BACKEND_MARKER; + } + + static Contents get(const Holder* ptr) + { + return *ptr & ~(MIN_CHUNK_SIZE - 1); + } + + static Holder& ref(bool direction, Contents k) + { + MetaEntry& entry = + Pagemap::template get_metaentry_mut(address_cast(k)); + if (direction) + return *reinterpret_cast(&entry.meta); + + return *reinterpret_cast(&entry.remote_and_sizeclass); + } + + static bool is_red(Contents k) + { + return (ref(true, k) & RED_BIT) == RED_BIT; + } + + static void set_red(Contents k, bool new_is_red) + { + if (new_is_red != is_red(k)) + ref(true, k) ^= RED_BIT; + } + + static Contents offset(Contents k, size_t size) + { + return k + size; + } + + static Contents buddy(Contents k, size_t size) + { + return k ^ size; + } + + static Contents align_down(Contents k, size_t size) + { + return k & ~(size - 1); + } + + static bool compare(Contents k1, Contents k2) + { + return k1 > k2; + } + + static bool equal(Contents k1, Contents k2) + { + return k1 == k2; + } + + static uintptr_t printable(Contents k) + { + return k; + } + + static bool can_consolidate(Contents k, size_t size) + { + // Need to know both entries exist in the pagemap. + // This must only be called if that has already been + // ascertained. + // The buddy could be in a part of the pagemap that has + // not been registered and thus could segfault on access. + auto larger = bits::max(k, buddy(k, size)); + MetaEntry& entry = + Pagemap::template get_metaentry_mut(address_cast(larger)); + return !entry.is_boundary(); + } + }; + + template< + typename ParentRange, + size_t REFILL_SIZE_BITS, + size_t MAX_SIZE_BITS, + SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap, + bool Consolidate = true> + class LargeBuddyRange + { + typename ParentRange::State parent{}; + + static constexpr size_t REFILL_SIZE = bits::one_at_bit(REFILL_SIZE_BITS); + + /** + * + */ + Buddy, MIN_CHUNK_BITS, MAX_SIZE_BITS> buddy_large; + + /** + * The parent might not support deallocation if this buddy allocator covers + * the whole range. Uses template insanity to make this work. + */ + template + std::enable_if_t + parent_dealloc_range(capptr::Chunk base, size_t size) + { + static_assert( + MAX_SIZE_BITS != (bits::BITS - 1), "Don't set SFINAE parameter"); + parent->dealloc_range(base, size); + } + + void dealloc_overflow(capptr::Chunk overflow) + { + if constexpr (MAX_SIZE_BITS != (bits::BITS - 1)) + { + if (overflow != nullptr) + { + parent->dealloc_range(overflow, bits::one_at_bit(MAX_SIZE_BITS)); + } + } + else + { + if (overflow != nullptr) + abort(); + } + } + + /** + * Add a range of memory to the address space. + * Divides blocks into power of two sizes with natural alignment + */ + void add_range(capptr::Chunk base, size_t length) + { + range_to_pow_2_blocks( + base, + length, + [this](capptr::Chunk base, size_t align, bool first) { + if constexpr (!Consolidate) + { + // Tag first entry so we don't consolidate it. + if (first) + { + Pagemap::get_metaentry_mut(address_cast(base)).set_boundary(); + } + } + else + { + UNUSED(first); + } + + auto overflow = capptr::Chunk(reinterpret_cast( + buddy_large.add_block(base.unsafe_uintptr(), align))); + + dealloc_overflow(overflow); + }); + } + + capptr::Chunk refill(size_t size) + { + if (ParentRange::Aligned) + { + // TODO have to add consolidation blocker for these cases. + if (size >= REFILL_SIZE) + { + return parent->alloc_range(size); + } + + auto refill_range = parent->alloc_range(REFILL_SIZE); + if (refill_range != nullptr) + add_range(pointer_offset(refill_range, size), REFILL_SIZE - size); + return refill_range; + } + + // Need to overallocate to get the alignment right. + bool overflow = false; + size_t needed_size = bits::umul(size, 2, overflow); + if (overflow) + { + return nullptr; + } + + auto refill_size = bits::max(needed_size, REFILL_SIZE); + while (needed_size <= refill_size) + { + auto refill = parent->alloc_range(refill_size); + + if (refill != nullptr) + { + add_range(refill, refill_size); + + SNMALLOC_ASSERT(refill_size < bits::one_at_bit(MAX_SIZE_BITS)); + static_assert( + (REFILL_SIZE < bits::one_at_bit(MAX_SIZE_BITS)) || + ParentRange::Aligned, + "Required to prevent overflow."); + + return alloc_range(size); + } + + refill_size >>= 1; + } + + return nullptr; + } + + public: + class State + { + LargeBuddyRange buddy_range; + + public: + LargeBuddyRange* operator->() + { + return &buddy_range; + } + + constexpr State() = default; + }; + + static constexpr bool Aligned = true; + + constexpr LargeBuddyRange() = default; + + capptr::Chunk alloc_range(size_t size) + { + SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE); + SNMALLOC_ASSERT(bits::is_pow2(size)); + + if (size >= (bits::one_at_bit(MAX_SIZE_BITS) - 1)) + { + if (ParentRange::Aligned) + return parent->alloc_range(size); + + return nullptr; + } + + auto result = capptr::Chunk( + reinterpret_cast(buddy_large.remove_block(size))); + + if (result != nullptr) + return result; + + return refill(size); + } + + void dealloc_range(capptr::Chunk base, size_t size) + { + SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE); + SNMALLOC_ASSERT(bits::is_pow2(size)); + + if constexpr (MAX_SIZE_BITS != (bits::BITS - 1)) + { + if (size >= (bits::one_at_bit(MAX_SIZE_BITS) - 1)) + { + parent_dealloc_range(base, size); + return; + } + } + + auto overflow = capptr::Chunk(reinterpret_cast( + buddy_large.add_block(base.unsafe_uintptr(), size))); + dealloc_overflow(overflow); + } + }; +} // namespace snmalloc \ No newline at end of file diff --git a/src/backend/pagemap.h b/src/backend/pagemap.h index 18463b9b5..85bb11323 100644 --- a/src/backend/pagemap.h +++ b/src/backend/pagemap.h @@ -115,7 +115,7 @@ namespace snmalloc static_assert( has_bounds_ == has_bounds, "Don't set SFINAE template parameter!"); #ifdef SNMALLOC_TRACING - std::cout << "Pagemap.init " << b << " (" << s << ")" << std::endl; + message<1024>("Pagemap.init {} ({})", b, s); #endif SNMALLOC_ASSERT(s != 0); // TODO take account of pagemap size in the calculation of how big it @@ -311,10 +311,10 @@ namespace snmalloc return base + (entry_index << GRANULARITY_BITS); } - void set(address_t p, T t) + void set(address_t p, const T& t) { #ifdef SNMALLOC_TRACING - std::cout << "Pagemap.Set " << (void*)(uintptr_t)p << std::endl; + message<1024>("Pagemap.Set {}", p); #endif if constexpr (has_bounds) { diff --git a/src/backend/pagemapregisterrange.h b/src/backend/pagemapregisterrange.h new file mode 100644 index 000000000..de55b1373 --- /dev/null +++ b/src/backend/pagemapregisterrange.h @@ -0,0 +1,43 @@ +#pragma once + +#include "../ds/address.h" +#include "../pal/pal.h" + +namespace snmalloc +{ + template< + SNMALLOC_CONCEPT(ConceptBackendMetaRange) Pagemap, + typename ParentRange> + class PagemapRegisterRange + { + typename ParentRange::State state{}; + + public: + class State + { + PagemapRegisterRange range; + + public: + PagemapRegisterRange* operator->() + { + return ⦥ + } + + constexpr State() = default; + }; + + constexpr PagemapRegisterRange() = default; + + static constexpr bool Aligned = ParentRange::Aligned; + + capptr::Chunk alloc_range(size_t size) + { + auto base = state->alloc_range(size); + + if (base != nullptr) + Pagemap::register_range(address_cast(base), size); + + return base; + } + }; +} // namespace snmalloc \ No newline at end of file diff --git a/src/backend/palrange.h b/src/backend/palrange.h new file mode 100644 index 000000000..82d36ce3a --- /dev/null +++ b/src/backend/palrange.h @@ -0,0 +1,59 @@ +#pragma once +#include "../ds/address.h" +#include "../pal/pal.h" + +namespace snmalloc +{ + template + class PalRange + { + public: + class State + { + public: + PalRange* operator->() + { + // There is no state required for the PalRange + // using a global just to satisfy the typing. + static PalRange range{}; + return ⦥ + } + + constexpr State() = default; + }; + + static constexpr bool Aligned = pal_supports; + + constexpr PalRange() = default; + + capptr::Chunk alloc_range(size_t size) + { + if (bits::next_pow2_bits(size) >= bits::BITS - 1) + { + return nullptr; + } + + if constexpr (pal_supports) + { + SNMALLOC_ASSERT(size >= PAL::minimum_alloc_size); + auto result = + capptr::Chunk(PAL::template reserve_aligned(size)); + +#ifdef SNMALLOC_TRACING + message<1024>("Pal range alloc: {} ({})", result.unsafe_ptr(), size); +#endif + return result; + } + else + { + auto result = capptr::Chunk(PAL::reserve(size)); + +#ifdef SNMALLOC_TRACING + message<1024>("Pal range alloc: {} ({})", result.unsafe_ptr(), size); +#endif + + return result; + } + } + }; +} // namespace snmalloc \ No newline at end of file diff --git a/src/backend/range_helpers.h b/src/backend/range_helpers.h new file mode 100644 index 000000000..1dd6ea4d4 --- /dev/null +++ b/src/backend/range_helpers.h @@ -0,0 +1,37 @@ +#pragma once + +#include "../ds/ptrwrap.h" + +namespace snmalloc +{ + template + void range_to_pow_2_blocks(capptr::Chunk base, size_t length, F f) + { + auto end = pointer_offset(base, length); + base = pointer_align_up(base, bits::one_at_bit(MIN_BITS)); + end = pointer_align_down(end, bits::one_at_bit(MIN_BITS)); + length = pointer_diff(base, end); + + bool first = true; + + // Find the minimum set of maximally aligned blocks in this range. + // Each block's alignment and size are equal. + while (length >= sizeof(void*)) + { + size_t base_align_bits = bits::ctz(address_cast(base)); + size_t length_align_bits = (bits::BITS - 1) - bits::clz(length); + size_t align_bits = bits::min(base_align_bits, length_align_bits); + size_t align = bits::one_at_bit(align_bits); + + /* + * Now that we have found a maximally-aligned block, we can set bounds + * and be certain that we won't hit representation imprecision. + */ + f(base, align, first); + first = false; + + base = pointer_offset(base, align); + length -= align; + } + } +} // namespace snmalloc \ No newline at end of file diff --git a/src/backend/smallbuddyrange.h b/src/backend/smallbuddyrange.h new file mode 100644 index 000000000..b813b674d --- /dev/null +++ b/src/backend/smallbuddyrange.h @@ -0,0 +1,226 @@ +#pragma once + +#include "../ds/address.h" +#include "../pal/pal.h" +#include "range_helpers.h" + +namespace snmalloc +{ + /** + * struct for representing the redblack nodes + * directly inside the meta data. + */ + struct FreeChunk + { + capptr::Chunk left; + capptr::Chunk right; + }; + + /** + * Class for using the allocations own space to store in the RBTree. + */ + class BuddyInplaceRep + { + public: + using Holder = capptr::Chunk; + using Contents = capptr::Chunk; + + static constexpr Contents null = nullptr; + + static constexpr address_t MASK = 1; + static void set(Holder* ptr, Contents r) + { + SNMALLOC_ASSERT((address_cast(r) & MASK) == 0); + if (r == nullptr) + *ptr = capptr::Chunk( + reinterpret_cast((*ptr).unsafe_uintptr() & MASK)); + else + // Preserve lower bit. + *ptr = pointer_offset(r, (address_cast(*ptr) & MASK)) + .template as_static(); + } + + static Contents get(Holder* ptr) + { + return pointer_align_down<2, FreeChunk>((*ptr).as_void()); + } + + static Holder& ref(bool direction, Contents r) + { + if (direction) + return r->left; + + return r->right; + } + + static bool is_red(Contents k) + { + if (k == nullptr) + return false; + return (address_cast(ref(false, k)) & MASK) == MASK; + } + + static void set_red(Contents k, bool new_is_red) + { + if (new_is_red != is_red(k)) + { + auto& r = ref(false, k); + auto old_addr = pointer_align_down<2, FreeChunk>(r.as_void()); + + if (new_is_red) + { + if (old_addr == nullptr) + r = capptr::Chunk(reinterpret_cast(MASK)); + else + r = pointer_offset(old_addr, MASK).template as_static(); + } + else + { + r = old_addr; + } + } + } + + static Contents offset(Contents k, size_t size) + { + return pointer_offset(k, size).template as_static(); + } + + static Contents buddy(Contents k, size_t size) + { + // This is just doing xor size, but with what API + // exists on capptr. + auto base = pointer_align_down(k.as_void(), size * 2); + auto offset = (address_cast(k) & size) ^ size; + return pointer_offset(base, offset).template as_static(); + } + + static Contents align_down(Contents k, size_t size) + { + return pointer_align_down(k.as_void(), size); + } + + static bool compare(Contents k1, Contents k2) + { + return address_cast(k1) > address_cast(k2); + } + + static bool equal(Contents k1, Contents k2) + { + return address_cast(k1) == address_cast(k2); + } + + static address_t printable(Contents k) + { + return address_cast(k); + } + + static bool can_consolidate(Contents k, size_t size) + { + UNUSED(k, size); + return true; + } + }; + + template + class SmallBuddyRange + { + typename ParentRange::State parent{}; + + static constexpr size_t MIN_BITS = + bits::next_pow2_bits_const(sizeof(FreeChunk)); + + Buddy buddy_small; + + /** + * Add a range of memory to the address space. + * Divides blocks into power of two sizes with natural alignment + */ + void add_range(capptr::Chunk base, size_t length) + { + range_to_pow_2_blocks( + base, length, [this](capptr::Chunk base, size_t align, bool) { + capptr::Chunk overflow = + buddy_small.add_block(base.as_reinterpret(), align) + .template as_reinterpret(); + if (overflow != nullptr) + parent->dealloc_range(overflow, bits::one_at_bit(MIN_CHUNK_BITS)); + }); + } + + capptr::Chunk refill(size_t size) + { + auto refill = parent->alloc_range(MIN_CHUNK_SIZE); + + if (refill != nullptr) + add_range(pointer_offset(refill, size), MIN_CHUNK_SIZE - size); + + return refill; + } + + public: + class State + { + SmallBuddyRange buddy_range; + + public: + SmallBuddyRange* operator->() + { + return &buddy_range; + } + + constexpr State() = default; + }; + + static constexpr bool Aligned = true; + static_assert(ParentRange::Aligned, "ParentRange must be aligned"); + + constexpr SmallBuddyRange() = default; + + capptr::Chunk alloc_range(size_t size) + { + if (size >= MIN_CHUNK_SIZE) + { + return parent->alloc_range(size); + } + + auto result = buddy_small.remove_block(size); + if (result != nullptr) + { + result->left = nullptr; + result->right = nullptr; + return result.template as_reinterpret(); + } + return refill(size); + } + + capptr::Chunk alloc_range_with_leftover(size_t size) + { + SNMALLOC_ASSERT(size <= MIN_CHUNK_SIZE); + + auto rsize = bits::next_pow2(size); + + auto result = alloc_range(rsize); + + if (result == nullptr) + return nullptr; + + auto remnant = pointer_offset(result, size); + + add_range(remnant, rsize - size); + + return result.template as_reinterpret(); + } + + void dealloc_range(capptr::Chunk base, size_t size) + { + if (size >= MIN_CHUNK_SIZE) + { + parent->dealloc_range(base, size); + return; + } + + add_range(base, size); + } + }; +} // namespace snmalloc \ No newline at end of file diff --git a/src/backend/statsrange.h b/src/backend/statsrange.h new file mode 100644 index 000000000..0a61fefcc --- /dev/null +++ b/src/backend/statsrange.h @@ -0,0 +1,68 @@ +#pragma once + +#include + +namespace snmalloc +{ + /** + * Used to measure memory usage. + */ + template + class StatsRange + { + typename ParentRange::State parent{}; + + static inline std::atomic current_usage{}; + static inline std::atomic peak_usage{}; + + public: + class State + { + StatsRange stats_range{}; + + public: + constexpr StatsRange* operator->() + { + return &stats_range; + } + + constexpr State() = default; + }; + + static constexpr bool Aligned = ParentRange::Aligned; + + constexpr StatsRange() = default; + + capptr::Chunk alloc_range(size_t size) + { + auto result = parent->alloc_range(size); + if (result != nullptr) + { + auto prev = current_usage.fetch_add(size); + auto curr = peak_usage.load(); + while (curr < prev + size) + { + if (peak_usage.compare_exchange_weak(curr, prev + size)) + break; + } + } + return result; + } + + void dealloc_range(capptr::Chunk base, size_t size) + { + current_usage -= size; + parent->dealloc_range(base, size); + } + + size_t get_current_usage() + { + return current_usage.load(); + } + + size_t get_peak_usage() + { + return peak_usage.load(); + } + }; +} // namespace snmalloc \ No newline at end of file diff --git a/src/backend/subrange.h b/src/backend/subrange.h new file mode 100644 index 000000000..c83bdab6b --- /dev/null +++ b/src/backend/subrange.h @@ -0,0 +1,55 @@ +#pragma once +#include "../mem/entropy.h" + +namespace snmalloc +{ + /** + * Creates an area inside a large allocation that is larger by + * 2^RATIO_BITS. Will not return a the block at the start or + * the end of the large allocation. + */ + template + class SubRange + { + typename ParentRange::State parent{}; + + public: + class State + { + SubRange sub_range{}; + + public: + constexpr State() = default; + + SubRange* operator->() + { + return &sub_range; + } + }; + + constexpr SubRange() = default; + + static constexpr bool Aligned = ParentRange::Aligned; + + capptr::Chunk alloc_range(size_t sub_size) + { + SNMALLOC_ASSERT(bits::is_pow2(sub_size)); + + auto full_size = sub_size << RATIO_BITS; + auto overblock = parent->alloc_range(full_size); + if (overblock == nullptr) + return nullptr; + + size_t offset_mask = full_size - sub_size; + // Don't use first or last block in the larger reservation + // Loop required to get uniform distribution. + size_t offset; + do + { + offset = get_entropy64() & offset_mask; + } while ((offset == 0) || (offset == offset_mask)); + + return pointer_offset(overblock, offset); + } + }; +} // namespace snmalloc \ No newline at end of file diff --git a/src/ds/address.h b/src/ds/address.h index 13c6ed848..0c3cdc72d 100644 --- a/src/ds/address.h +++ b/src/ds/address.h @@ -72,13 +72,17 @@ namespace snmalloc * as per above, and uses the wrapper types in its own definition, e.g., of * capptr_bound. */ - template inline SNMALLOC_FAST_PATH address_t address_cast(CapPtr a) { return address_cast(a.unsafe_ptr()); } + inline SNMALLOC_FAST_PATH address_t address_cast(uintptr_t a) + { + return static_cast(a); + } + /** * Test if a pointer is aligned to a given size, which must be a power of * two. diff --git a/src/ds/redblacktree.h b/src/ds/redblacktree.h index 80325b3b2..10eb55f58 100644 --- a/src/ds/redblacktree.h +++ b/src/ds/redblacktree.h @@ -1,44 +1,15 @@ #pragma once +#include "../pal/pal.h" #include "concept.h" #include "defines.h" #include #include #include +#include namespace snmalloc { - template - class debug_out - { - public: - template - static void msg(A a, Args... args) - { - if constexpr (TRACE) - { -#ifdef SNMALLOC_TRACING - std::cout << a; -#else - UNUSED(a); -#endif - - msg(args...); - } - } - - template - static void msg() - { - if constexpr (TRACE && new_line) - { -#ifdef SNMALLOC_TRACING - std::cout << std::endl; -#endif - } - } - }; - #ifdef __cpp_concepts template concept RBRepTypes = requires() @@ -118,20 +89,20 @@ namespace snmalloc return Rep::get(ptr); } - K operator=(K t) + ChildRef& operator=(const K& t) { // Use representations assigment, so we update the correct bits // color and other things way also be stored in the Holder. Rep::set(ptr, t); - return t; + return *this; } - bool operator==(ChildRef& t) + bool operator==(const ChildRef t) const { return ptr == t.ptr; } - bool operator!=(ChildRef& t) + bool operator!=(const ChildRef t) const { return ptr != t.ptr; } @@ -140,6 +111,11 @@ namespace snmalloc { return ptr; } + + bool is_null() + { + return Rep::get(ptr) == Rep::null; + } }; // Root field of the tree @@ -171,60 +147,47 @@ namespace snmalloc UNUSED(curr, lower, upper); return 0; } - if (curr == Rep::null) - return 1; - - if ( - ((lower != Rep::null) && curr < lower) || - ((upper != Rep::null) && curr > upper)) + else { - if constexpr (TRACE) + if (curr == Rep::null) + return 1; + + if ( + ((lower != Rep::null) && Rep::compare(lower, curr)) || + ((upper != Rep::null) && Rep::compare(curr, upper))) { - debug_out::msg( - "Invariant failed: ", - curr, - " is out of bounds ", - lower, - ", ", - upper); - print(); + report_fatal_error( + "Invariant failed: {} is out of bounds {}..{}", + Rep::printable(curr), + Rep::printable(lower), + Rep::printable(upper)); } - snmalloc::error("Invariant failed"); - } - if ( - Rep::is_red(curr) && - (Rep::is_red(get_dir(true, curr)) || Rep::is_red(get_dir(false, curr)))) - { - if constexpr (TRACE) + if ( + Rep::is_red(curr) && + (Rep::is_red(get_dir(true, curr)) || + Rep::is_red(get_dir(false, curr)))) { - debug_out::msg( - "Red invariant failed: ", curr, " is red and has red children"); - print(); + report_fatal_error( + "Invariant failed: {} is red and has red child", + Rep::printable(curr)); } - snmalloc::error("Invariant failed"); - } - int left_inv = invariant(get_dir(true, curr), lower, curr); - int right_inv = invariant(get_dir(false, curr), curr, upper); + int left_inv = invariant(get_dir(true, curr), lower, curr); + int right_inv = invariant(get_dir(false, curr), curr, upper); - if (left_inv != right_inv) - { - if constexpr (TRACE) + if (left_inv != right_inv) { - debug_out::msg( - "Balance failed: ", - curr, - " has different black depths on left and right"); - print(); + report_fatal_error( + "Invariant failed: {} has different black depths", + Rep::printable(curr)); } - snmalloc::error("Invariant failed"); - } - if (Rep::is_red(curr)) - return left_inv; - else + if (Rep::is_red(curr)) + return left_inv; + return left_inv + 1; + } } struct RBStep @@ -293,7 +256,7 @@ namespace snmalloc bool move(bool direction) { auto next = get_dir(direction, curr()); - if (next == Rep::null) + if (next.is_null()) return false; path[length] = {next, direction}; length++; @@ -308,7 +271,7 @@ namespace snmalloc auto next = get_dir(direction, curr()); path[length] = {next, direction}; length++; - return next != Rep::null; + return !(next.is_null()); } // Remove top element from the path. @@ -354,16 +317,12 @@ namespace snmalloc { for (size_t i = 0; i < length; i++) { - debug_out::msg( - "->", + message<1024>( + " -> {} @ {} ({})", K(path[i].node), - "@", path[i].node.addr(), - " (", - path[i].dir, - ") "); + path[i].dir); } - debug_out::msg(); } } }; @@ -378,8 +337,8 @@ namespace snmalloc { if constexpr (TRACE) { - debug_out::msg("-------"); - debug_out::msg(msg); + message<100>("-------"); + message<1024>(msg); path.print(); print(base); } @@ -390,7 +349,7 @@ namespace snmalloc } public: - RBTree() {} + constexpr RBTree() = default; void print() { @@ -401,9 +360,9 @@ namespace snmalloc { if constexpr (TRACE) { - if (curr == Rep::null) + if (curr.is_null()) { - debug_out::msg(indent, "\\_", "null"); + message<1024>("{}\\_null", indent); return; } @@ -415,17 +374,14 @@ namespace snmalloc auto reset = "\e[0m"; #endif - debug_out::msg( + message<1024>( + "{}\\_{}{}{}@{} ({})", indent, - "\\_", colour, curr, reset, - "@", curr.addr(), - " (", - depth, - ")"); + depth); if ((get_dir(true, curr) != 0) || (get_dir(false, curr) != 0)) { auto s_indent = std::string(indent); @@ -439,14 +395,14 @@ namespace snmalloc { bool dir; - if (path.curr() == Rep::null) + if (path.curr().is_null()) return false; do { - if (path.curr() == value) + if (Rep::equal(path.curr(), value)) return true; - dir = path.curr() > value; + dir = Rep::compare(path.curr(), value); } while (path.move_inc_null(dir)); return false; @@ -455,7 +411,7 @@ namespace snmalloc bool remove_path(RBPath& path) { ChildRef splice = path.curr(); - SNMALLOC_ASSERT(splice != Rep::null); + SNMALLOC_ASSERT(!(splice.is_null())); debug_log("Removing", path); @@ -494,6 +450,8 @@ namespace snmalloc debug_log("Splice done", path); + // TODO: Clear node contents? + // Red leaf removal requires no rebalancing. if (leaf_red) return true; @@ -618,7 +576,7 @@ namespace snmalloc // Insert an element at the given path. void insert_path(RBPath path, K value) { - SNMALLOC_ASSERT(path.curr() == Rep::null); + SNMALLOC_ASSERT(path.curr().is_null()); path.curr() = value; get_dir(true, path.curr()) = Rep::null; get_dir(false, path.curr()) = Rep::null; @@ -706,7 +664,7 @@ namespace snmalloc K remove_min() { - if (get_root() == Rep::null) + if (get_root().is_null()) return Rep::null; auto path = get_root_path(); @@ -722,7 +680,7 @@ namespace snmalloc bool remove_elem(K value) { - if (get_root() == Rep::null) + if (get_root().is_null()) return false; auto path = get_root_path(); @@ -749,4 +707,4 @@ namespace snmalloc return RBPath(root); } }; -} +} // namespace snmalloc diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index f621e0c3d..48d326325 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -1,8 +1,8 @@ #pragma once +#include "../backend/chunkallocator.h" #include "../ds/defines.h" #include "allocconfig.h" -#include "chunkallocator.h" #include "localcache.h" #include "metaslab.h" #include "pool.h" @@ -58,11 +58,6 @@ namespace snmalloc */ MetaslabCache alloc_classes[NUM_SMALL_SIZECLASSES]; - /** - * Local cache for the Chunk allocator. - */ - ChunkAllocatorLocalState chunk_local_state; - /** * Local entropy source and current version of keys for * this thread @@ -392,11 +387,11 @@ namespace snmalloc // TODO delay the clear to the next user of the slab, or teardown so // don't touch the cache lines at this point in snmalloc_check_client. auto chunk_record = clear_slab(meta, sizeclass); - ChunkAllocator::dealloc( + + SharedStateHandle::dealloc_chunk( get_backend_local_state(), - chunk_local_state, chunk_record, - sizeclass_to_slab_sizeclass(sizeclass)); + sizeclass_to_slab_size(sizeclass)); return true; }); @@ -419,23 +414,17 @@ namespace snmalloc // Handle large deallocation here. size_t entry_sizeclass = entry.get_sizeclass().as_large(); size_t size = bits::one_at_bit(entry_sizeclass); - size_t slab_sizeclass = - metaentry_chunk_sizeclass_to_slab_sizeclass(entry_sizeclass); #ifdef SNMALLOC_TRACING - std::cout << "Large deallocation: " << size - << " chunk sizeclass: " << slab_sizeclass << std::endl; + std::cout << "Large deallocation: " << size << std::endl; #else UNUSED(size); #endif auto slab_record = reinterpret_cast(meta); - ChunkAllocator::dealloc( - get_backend_local_state(), - chunk_local_state, - slab_record, - slab_sizeclass); + SharedStateHandle::dealloc_chunk( + get_backend_local_state(), slab_record, size); return; } @@ -594,9 +583,6 @@ namespace snmalloc message_queue().invariant(); } - ChunkAllocator::register_local_state( - get_backend_local_state(), chunk_local_state); - if constexpr (DEBUG) { for (smallsizeclass_t i = 0; i < NUM_SMALL_SIZECLASSES; i++) @@ -686,7 +672,7 @@ namespace snmalloc SNMALLOC_FAST_PATH void dealloc_local_object(CapPtr p) { - auto entry = + const MetaEntry& entry = SharedStateHandle::Pagemap::get_metaentry(snmalloc::address_cast(p)); if (SNMALLOC_LIKELY(dealloc_local_object_fast(entry, p, entropy))) return; @@ -791,21 +777,17 @@ namespace snmalloc // No existing free list get a new slab. size_t slab_size = sizeclass_to_slab_size(sizeclass); - size_t slab_sizeclass = sizeclass_to_slab_sizeclass(sizeclass); #ifdef SNMALLOC_TRACING std::cout << "rsize " << rsize << std::endl; std::cout << "slab size " << slab_size << std::endl; #endif - auto [slab, meta] = - snmalloc::ChunkAllocator::alloc_chunk( - get_backend_local_state(), - chunk_local_state, - sizeclass_t::from_small_class(sizeclass), - slab_sizeclass, - slab_size, - public_state()); + auto [slab, meta] = SharedStateHandle::alloc_chunk( + get_backend_local_state(), + slab_size, + public_state(), + sizeclass_t::from_small_class(sizeclass)); if (slab == nullptr) { diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 6fb378ebd..d9bf18493 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -180,13 +180,11 @@ namespace snmalloc return check_init([&](CoreAlloc* core_alloc) { // Grab slab of correct size // Set remote as large allocator remote. - auto [chunk, meta] = ChunkAllocator::alloc_chunk( + auto [chunk, meta] = SharedStateHandle::alloc_chunk( core_alloc->get_backend_local_state(), - core_alloc->chunk_local_state, - size_to_sizeclass_full(size), - large_size_to_chunk_sizeclass(size), large_size_to_chunk_size(size), - core_alloc->public_state()); + core_alloc->public_state(), + size_to_sizeclass_full(size)); // set up meta data so sizeclass is correct, and hence alloc size, and // external pointer. #ifdef SNMALLOC_TRACING @@ -201,7 +199,7 @@ namespace snmalloc if (zero_mem == YesZero && chunk.unsafe_ptr() != nullptr) { SharedStateHandle::Pal::template zero( - chunk.unsafe_ptr(), size); + chunk.unsafe_ptr(), bits::next_pow2(size)); } return capptr_chunk_is_alloc(capptr_to_user_address_control(chunk)); @@ -268,7 +266,7 @@ namespace snmalloc std::cout << "Remote dealloc post" << p.unsafe_ptr() << " size " << alloc_size(p.unsafe_ptr()) << std::endl; #endif - MetaEntry entry = + const MetaEntry& entry = SharedStateHandle::Pagemap::get_metaentry(address_cast(p)); local_cache.remote_dealloc_cache.template dealloc( entry.get_remote()->trunc_id(), p, key_global); @@ -716,7 +714,7 @@ namespace snmalloc // To handle this case we require the uninitialised pagemap contain an // entry for the first chunk of memory, that states it represents a // large object, so we can pull the check for null off the fast path. - MetaEntry entry = + const MetaEntry& entry = SharedStateHandle::Pagemap::get_metaentry(address_cast(p_raw)); return sizeclass_full_to_size(entry.get_sizeclass()); @@ -761,7 +759,7 @@ namespace snmalloc size_t remaining_bytes(const void* p) { #ifndef SNMALLOC_PASS_THROUGH - MetaEntry entry = + const MetaEntry& entry = SharedStateHandle::Pagemap::template get_metaentry( address_cast(p)); @@ -790,7 +788,7 @@ namespace snmalloc size_t index_in_object(const void* p) { #ifndef SNMALLOC_PASS_THROUGH - MetaEntry entry = + const MetaEntry& entry = SharedStateHandle::Pagemap::template get_metaentry( address_cast(p)); diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index bad2d3c27..a046e0903 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -227,7 +227,26 @@ namespace snmalloc */ class MetaEntry { - Metaslab* meta{nullptr}; // may also be ChunkRecord* + template + friend class BuddyChunkRep; + + /** + * The pointer to the metaslab, the bottom bit is used to indicate if this + * is the first chunk in a PAL allocation, that cannot be combined with + * the preceeding chunk. + */ + uintptr_t meta{0}; + + /** + * Bit used to indicate this should not be considered part of the previous + * PAL allocation. + * + * Some platforms cannot treat different PalAllocs as a single allocation. + * This is true on CHERI as the combined permission might not be + * representable. It is also true on Windows as you cannot Commit across + * multiple continuous VirtualAllocs. + */ + static constexpr address_t BOUNDARY_BIT = 1; /** * A bit-packed pointer to the owning allocator (if any), and the sizeclass @@ -253,7 +272,8 @@ namespace snmalloc */ SNMALLOC_FAST_PATH MetaEntry(Metaslab* meta, uintptr_t remote_and_sizeclass) - : meta(meta), remote_and_sizeclass(remote_and_sizeclass) + : meta(reinterpret_cast(meta)), + remote_and_sizeclass(remote_and_sizeclass) {} SNMALLOC_FAST_PATH @@ -261,23 +281,13 @@ namespace snmalloc Metaslab* meta, RemoteAllocator* remote, sizeclass_t sizeclass = sizeclass_t()) - : meta(meta) + : meta(reinterpret_cast(meta)) { /* remote might be nullptr; cast to uintptr_t before offsetting */ remote_and_sizeclass = pointer_offset(reinterpret_cast(remote), sizeclass.raw()); } - /** - * Return the Metaslab field as a void*, guarded by an assert that there is - * no remote that owns this chunk. - */ - [[nodiscard]] SNMALLOC_FAST_PATH void* get_metaslab_no_remote() const - { - SNMALLOC_ASSERT(get_remote() == nullptr); - return static_cast(meta); - } - /** * Return the Metaslab metadata associated with this chunk, guarded by an * assert that this chunk is being used as a slab (i.e., has an associated @@ -286,7 +296,7 @@ namespace snmalloc [[nodiscard]] SNMALLOC_FAST_PATH Metaslab* get_metaslab() const { SNMALLOC_ASSERT(get_remote() != nullptr); - return meta; + return reinterpret_cast(meta & ~BOUNDARY_BIT); } /** @@ -295,7 +305,7 @@ namespace snmalloc * only safe use for this is to pass it to the two-argument constructor of * this class. */ - [[nodiscard]] SNMALLOC_FAST_PATH uintptr_t get_remote_and_sizeclass() + [[nodiscard]] SNMALLOC_FAST_PATH uintptr_t get_remote_and_sizeclass() const { return remote_and_sizeclass; } @@ -303,7 +313,8 @@ namespace snmalloc [[nodiscard]] SNMALLOC_FAST_PATH RemoteAllocator* get_remote() const { return reinterpret_cast( - pointer_align_down(remote_and_sizeclass)); + pointer_align_down( + remote_and_sizeclass)); } [[nodiscard]] SNMALLOC_FAST_PATH sizeclass_t get_sizeclass() const @@ -312,7 +323,32 @@ namespace snmalloc // https://github.com/CTSRD-CHERI/llvm-project/issues/588 return sizeclass_t::from_raw( static_cast(remote_and_sizeclass) & - (alignof(RemoteAllocator) - 1)); + (REMOTE_WITH_BACKEND_MARKER_ALIGN - 1)); + } + + MetaEntry(const MetaEntry&) = delete; + + MetaEntry& operator=(const MetaEntry& other) + { + // Don't overwrite the boundary bit with the other's + meta = (other.meta & ~BOUNDARY_BIT) | address_cast(meta & BOUNDARY_BIT); + remote_and_sizeclass = other.remote_and_sizeclass; + return *this; + } + + void set_boundary() + { + meta |= BOUNDARY_BIT; + } + + [[nodiscard]] bool is_boundary() const + { + return meta & BOUNDARY_BIT; + } + + bool clear_boundary_bit() + { + return meta &= ~BOUNDARY_BIT; } }; @@ -328,5 +364,4 @@ namespace snmalloc uint16_t unused = 0; uint16_t length = 0; }; - } // namespace snmalloc diff --git a/src/mem/pool.h b/src/mem/pool.h index ffb9572b6..c7924c63a 100644 --- a/src/mem/pool.h +++ b/src/mem/pool.h @@ -1,9 +1,9 @@ #pragma once +#include "../backend/chunkallocator.h" #include "../ds/flaglock.h" #include "../ds/mpmcstack.h" #include "../pal/pal_concept.h" -#include "chunkallocator.h" #include "pooled.h" namespace snmalloc diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index 3c060e6b0..f8b309eab 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -2,7 +2,6 @@ #include "../mem/allocconfig.h" #include "../mem/freelist.h" -#include "../mem/metaslab.h" #include "../mem/sizeclasstable.h" #include @@ -11,9 +10,19 @@ namespace snmalloc { // Remotes need to be aligned enough that the bottom bits have enough room for - // all the size classes, both large and small. + // all the size classes, both large and small. An additional bit is required + // to separate backend uses. static constexpr size_t REMOTE_MIN_ALIGN = - bits::max(CACHELINE_SIZE, SIZECLASS_REP_SIZE); + bits::max(CACHELINE_SIZE, SIZECLASS_REP_SIZE) << 1; + + // This bit is set on the RemoteAllocator* to indicate it is + // actually being used by the backend for some other use. + static constexpr size_t BACKEND_MARKER = REMOTE_MIN_ALIGN >> 1; + + // The bit above the sizeclass is always zero unless this is used + // by the backend to represent another datastructure such as the buddy + // allocator entries. + constexpr size_t REMOTE_WITH_BACKEND_MARKER_ALIGN = BACKEND_MARKER; /** * Global key for all remote lists. diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index cafe15835..5e7559971 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -101,18 +101,25 @@ namespace snmalloc if (!list[i].empty()) { auto [first, last] = list[i].extract_segment(key); - MetaEntry entry = + const MetaEntry& entry = SharedStateHandle::Pagemap::get_metaentry(address_cast(first)); + auto remote = entry.get_remote(); + // If the allocator is not correctly aligned, then the bit that is + // set implies this is used by the backend, and we should not be + // deallocating memory here. + snmalloc_check_client( + (address_cast(remote) & BACKEND_MARKER) == 0, + "Delayed detection of attempt to free internal structure."); if constexpr (SharedStateHandle::Options.QueueHeadsAreTame) { auto domesticate_nop = [](freelist::QueuePtr p) { return freelist::HeadPtr(p.unsafe_ptr()); }; - entry.get_remote()->enqueue(first, last, key, domesticate_nop); + remote->enqueue(first, last, key, domesticate_nop); } else { - entry.get_remote()->enqueue(first, last, key, domesticate); + remote->enqueue(first, last, key, domesticate); } sent_something = true; } @@ -134,7 +141,7 @@ namespace snmalloc // Use the next N bits to spread out remote deallocs in our own // slot. auto r = resend.take(key, domesticate); - MetaEntry entry = + const MetaEntry& entry = SharedStateHandle::Pagemap::get_metaentry(address_cast(r)); auto i = entry.get_remote()->trunc_id(); size_t slot = get_slot(i, post_round); diff --git a/src/override/malloc-extensions.cc b/src/override/malloc-extensions.cc index a76040167..1e0810b45 100644 --- a/src/override/malloc-extensions.cc +++ b/src/override/malloc-extensions.cc @@ -6,8 +6,8 @@ using namespace snmalloc; void get_malloc_info_v1(malloc_info_v1* stats) { - auto unused_chunks = Globals::get_chunk_allocator_state().unused_memory(); - auto peak = Globals::get_chunk_allocator_state().peak_memory_usage(); - stats->current_memory_usage = peak - unused_chunks; + auto curr = Globals::get_current_usage(); + auto peak = Globals::get_peak_usage(); + stats->current_memory_usage = curr; stats->peak_memory_usage = peak; } diff --git a/src/override/rust.cc b/src/override/rust.cc index ebce39b15..889232320 100644 --- a/src/override/rust.cc +++ b/src/override/rust.cc @@ -48,8 +48,6 @@ extern "C" SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(rust_realloc)( extern "C" SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(rust_statistics)( size_t* current_memory_usage, size_t* peak_memory_usage) { - auto unused_chunks = Globals::get_chunk_allocator_state().unused_memory(); - auto peak = Globals::get_chunk_allocator_state().peak_memory_usage(); - *current_memory_usage = peak - unused_chunks; - *peak_memory_usage = peak; + *current_memory_usage = Globals::get_current_usage(); + *peak_memory_usage = Globals::get_peak_usage(); } \ No newline at end of file diff --git a/src/pal/pal.h b/src/pal/pal.h index c4dfd0ca0..b89391479 100644 --- a/src/pal/pal.h +++ b/src/pal/pal.h @@ -168,4 +168,10 @@ namespace snmalloc Pal::error(msg.get_message()); } + template + inline void message(Args... args) + { + MessageBuilder msg{std::forward(args)...}; + Pal::message(msg.get_message()); + } } // namespace snmalloc diff --git a/src/pal/pal_windows.h b/src/pal/pal_windows.h index dc1a1f6df..6d64a25c8 100644 --- a/src/pal/pal_windows.h +++ b/src/pal/pal_windows.h @@ -150,7 +150,8 @@ namespace snmalloc void* r = VirtualAlloc(p, size, MEM_COMMIT, PAGE_READWRITE); if (r == nullptr) - error("out of memory"); + report_fatal_error( + "out of memory: {} ({}) could not be committed", p, size); } /// OS specific function for zeroing memory diff --git a/src/snmalloc_core.h b/src/snmalloc_core.h index d8b14aa03..7ae0a3314 100644 --- a/src/snmalloc_core.h +++ b/src/snmalloc_core.h @@ -1,6 +1,5 @@ #pragma once -#include "backend/address_space.h" #include "backend/commonconfig.h" #include "backend/pagemap.h" #include "mem/globalalloc.h" diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index 1ee90677b..08172b3df 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -135,10 +135,11 @@ void test_realloc(void* p, size_t size, int err, bool null) START_TEST("realloc({}({}), {})", p, old_size, size); errno = SUCCESS; auto new_p = our_realloc(p, size); - // Realloc failure case, deallocate original block + check_result(size, 1, new_p, err, null); + // Realloc failure case, deallocate original block as not + // handled by check_result. if (new_p == nullptr && size != 0) our_free(p); - check_result(size, 1, new_p, err, null); } void test_posix_memalign(size_t size, size_t align, int err, bool null) diff --git a/src/test/func/redblack/redblack.cc b/src/test/func/redblack/redblack.cc index 2c96541e8..590133130 100644 --- a/src/test/func/redblack/redblack.cc +++ b/src/test/func/redblack/redblack.cc @@ -73,6 +73,21 @@ class Rep if (new_is_red != is_red(k)) array[k].left.value ^= 1; } + + static bool compare(key k1, key k2) + { + return k1 > k2; + } + + static bool equal(key k1, key k2) + { + return k1 == k2; + } + + static size_t printable(key k) + { + return k; + } }; template From 62cce1a20e650a908395891eca6b371353c2f85b Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Thu, 17 Mar 2022 17:53:15 +0000 Subject: [PATCH 218/302] backend/largebuddyrange: remove unnecessary casts Since Holder is just an alias for uintptr_t and the fields in the MetaEntry are uintptr_t-s, just return the lvalue-s directly rather than jumping through *reinterpret_cast(& ...). --- src/backend/largebuddyrange.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/largebuddyrange.h b/src/backend/largebuddyrange.h index 1d15dfe1f..5f1824840 100644 --- a/src/backend/largebuddyrange.h +++ b/src/backend/largebuddyrange.h @@ -43,9 +43,9 @@ namespace snmalloc MetaEntry& entry = Pagemap::template get_metaentry_mut(address_cast(k)); if (direction) - return *reinterpret_cast(&entry.meta); + return entry.meta; - return *reinterpret_cast(&entry.remote_and_sizeclass); + return entry.remote_and_sizeclass; } static bool is_red(Contents k) From 4ad99d7392df7b5816db92889da847ea6b39728a Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Thu, 17 Mar 2022 17:55:47 +0000 Subject: [PATCH 219/302] Downgrade some casts Do a quick sweep through the codebase to eliminate some reinterpret_cast<>s where less fire power would do just fine. --- src/backend/pagemap.h | 7 +++---- src/test/func/malloc/malloc.cc | 8 ++------ src/test/func/memcpy/func-memcpy.cc | 8 ++++---- src/test/perf/memcpy/memcpy.cc | 4 ++-- 4 files changed, 11 insertions(+), 16 deletions(-) diff --git a/src/backend/pagemap.h b/src/backend/pagemap.h index 85bb11323..6bb259d6e 100644 --- a/src/backend/pagemap.h +++ b/src/backend/pagemap.h @@ -130,7 +130,7 @@ namespace snmalloc // Put pagemap at start of range. // TODO CHERI capability bound here! - body = reinterpret_cast(b); + body = static_cast(b); body_opt = body; // Advance by size of pagemap. // Note that base needs to be aligned to GRANULARITY for the rest of the @@ -176,8 +176,7 @@ namespace snmalloc // Begin pagemap at random offset within the additionally allocated space. static_assert(bits::is_pow2(sizeof(T)), "Next line assumes this."); size_t offset = get_entropy64() & (additional_size - sizeof(T)); - auto new_body = - reinterpret_cast(pointer_offset(new_body_untyped, offset)); + auto new_body = pointer_offset(new_body_untyped, offset); if constexpr (pal_supports) { @@ -191,7 +190,7 @@ namespace snmalloc start_page, pointer_diff(start_page, end_page)); } #else - auto new_body = reinterpret_cast(new_body_untyped); + auto new_body = static_cast(new_body_untyped); #endif // Ensure bottom page is committed // ASSUME: new memory is zeroed. diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index 08172b3df..b5bd9ff35 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -88,16 +88,12 @@ void check_result(size_t size, size_t align, void* p, int err, bool null) expected_size); failed = true; } - if ( - (static_cast(reinterpret_cast(p) % align) != 0) && - (size != 0)) + if (((address_cast(p) % align) != 0) && (size != 0)) { INFO("Address is {}, but required to be aligned to {}.\n", p, align); failed = true; } - if ( - static_cast( - reinterpret_cast(p) % natural_alignment(size)) != 0) + if ((address_cast(p) % natural_alignment(size)) != 0) { INFO( "Address is {}, but should have natural alignment to {}.\n", diff --git a/src/test/func/memcpy/func-memcpy.cc b/src/test/func/memcpy/func-memcpy.cc index e99569e37..ad8ea17d3 100644 --- a/src/test/func/memcpy/func-memcpy.cc +++ b/src/test/func/memcpy/func-memcpy.cc @@ -69,8 +69,8 @@ extern "C" void abort() void check_size(size_t size) { START_TEST("checking {}-byte memcpy", size); - auto* s = reinterpret_cast(my_malloc(size + 1)); - auto* d = reinterpret_cast(my_malloc(size + 1)); + auto* s = static_cast(my_malloc(size + 1)); + auto* d = static_cast(my_malloc(size + 1)); d[size] = 0; s[size] = 255; for (size_t start = 0; start < size; start++) @@ -115,8 +115,8 @@ void check_bounds(size_t size, size_t out_of_bounds) { START_TEST( "memcpy bounds, size {}, {} bytes out of bounds", size, out_of_bounds); - auto* s = reinterpret_cast(my_malloc(size)); - auto* d = reinterpret_cast(my_malloc(size)); + auto* s = static_cast(my_malloc(size)); + auto* d = static_cast(my_malloc(size)); for (size_t i = 0; i < size; ++i) { s[i] = static_cast(i); diff --git a/src/test/perf/memcpy/memcpy.cc b/src/test/perf/memcpy/memcpy.cc index b42c47443..62f5da35f 100644 --- a/src/test/perf/memcpy/memcpy.cc +++ b/src/test/perf/memcpy/memcpy.cc @@ -30,7 +30,7 @@ void shape(size_t size) // size / alignment) * alignment; Shape s; s.object = ThreadAlloc::get().alloc(rsize); - s.dst = reinterpret_cast(s.object) + offset; + s.dst = static_cast(s.object) + offset; // Bring into cache the destination of the copy. memset(s.dst, 0xFF, size); allocs.push_back(s); @@ -51,7 +51,7 @@ void test_memcpy(size_t size, void* src, Memcpy mc) { for (auto& s : allocs) { - auto* dst = reinterpret_cast(s.dst); + auto* dst = static_cast(s.dst); mc(dst, src, size); } } From 26324e8bfc28eda735a3335f83f3f1d8f21cc4a0 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Thu, 17 Mar 2022 17:34:26 +0000 Subject: [PATCH 220/302] Add and plumb unsafe_{to,from}_uintptr casts These encapsulate the wildly powerful reinterpret_cast<> operator where one side is a uintptr_t and the other is a native pointer. In both cases we require the pointer type to be explicitly given. --- src/ds/address.h | 20 +++++++------- src/ds/ptrwrap.h | 26 ++++++++++++++++++- src/mem/freelist.h | 4 +-- src/mem/metaslab.h | 12 ++++----- .../func/external_pagemap/external_pagemap.cc | 2 +- src/test/func/malloc/malloc.cc | 2 +- 6 files changed, 45 insertions(+), 21 deletions(-) diff --git a/src/ds/address.h b/src/ds/address.h index 0c3cdc72d..a6a161a7e 100644 --- a/src/ds/address.h +++ b/src/ds/address.h @@ -28,8 +28,8 @@ namespace snmalloc inline U* pointer_offset(T* base, size_t diff) { SNMALLOC_ASSERT(base != nullptr); /* Avoid UB */ - return reinterpret_cast( - reinterpret_cast(base) + static_cast(diff)); + return unsafe_from_uintptr( + unsafe_to_uintptr(base) + static_cast(diff)); } template @@ -129,8 +129,8 @@ namespace snmalloc template inline T* pointer_align_down(void* p) { - return reinterpret_cast( - pointer_align_down(reinterpret_cast(p))); + return unsafe_from_uintptr( + pointer_align_down(unsafe_to_uintptr(p))); } template< @@ -164,8 +164,8 @@ namespace snmalloc #if __has_builtin(__builtin_align_up) return static_cast(__builtin_align_up(p, alignment)); #else - return reinterpret_cast( - bits::align_up(reinterpret_cast(p), alignment)); + return unsafe_from_uintptr( + bits::align_up(unsafe_to_uintptr(p), alignment)); #endif } } @@ -197,8 +197,8 @@ namespace snmalloc #if __has_builtin(__builtin_align_down) return static_cast(__builtin_align_down(p, alignment)); #else - return reinterpret_cast( - bits::align_down(reinterpret_cast(p), alignment)); + return unsafe_from_uintptr( + bits::align_down(unsafe_to_uintptr(p), alignment)); #endif } @@ -221,8 +221,8 @@ namespace snmalloc #if __has_builtin(__builtin_align_up) return static_cast(__builtin_align_up(p, alignment)); #else - return reinterpret_cast( - bits::align_up(reinterpret_cast(p), alignment)); + return unsafe_from_uintptr( + bits::align_up(unsafe_to_uintptr(p), alignment)); #endif } diff --git a/src/ds/ptrwrap.h b/src/ds/ptrwrap.h index 8b91a01bb..b94c90f4b 100644 --- a/src/ds/ptrwrap.h +++ b/src/ds/ptrwrap.h @@ -7,6 +7,30 @@ namespace snmalloc { + /* + * reinterpret_cast<> is a powerful primitive that, excitingly, does not + * require the programmer to annotate the expected *source* type. We + * therefore wrap its use to interconvert between uintptr_t and pointer types. + */ + + /** + * Convert a pointer to a uintptr_t. Template argument inference is + * prohibited. + */ + template + SNMALLOC_FAST_PATH_INLINE uintptr_t + unsafe_to_uintptr(std::enable_if_t* p) + { + return reinterpret_cast(p); + } + + /** Convert a uintptr_t to a T*. */ + template + SNMALLOC_FAST_PATH_INLINE T* unsafe_from_uintptr(uintptr_t p) + { + return reinterpret_cast(p); + } + /** * To assist in providing a uniform interface regardless of pointer wrapper, * we also export intrinsic pointer and atomic pointer aliases, as the postfix @@ -321,7 +345,7 @@ namespace snmalloc [[nodiscard]] SNMALLOC_FAST_PATH uintptr_t unsafe_uintptr() const { - return reinterpret_cast(this->unsafe_capptr); + return unsafe_to_uintptr(this->unsafe_capptr); } }; diff --git a/src/mem/freelist.h b/src/mem/freelist.h index bac781cc7..9850b6126 100644 --- a/src/mem/freelist.h +++ b/src/mem/freelist.h @@ -239,8 +239,8 @@ namespace snmalloc if constexpr (CHECK_CLIENT && !aal_supports) { - return reinterpret_cast*>( - reinterpret_cast(next) ^ key.key_next); + return unsafe_from_uintptr>( + unsafe_to_uintptr>(next) ^ key.key_next); } else { diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index a046e0903..947b22de9 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -272,7 +272,7 @@ namespace snmalloc */ SNMALLOC_FAST_PATH MetaEntry(Metaslab* meta, uintptr_t remote_and_sizeclass) - : meta(reinterpret_cast(meta)), + : meta(unsafe_to_uintptr(meta)), remote_and_sizeclass(remote_and_sizeclass) {} @@ -281,11 +281,11 @@ namespace snmalloc Metaslab* meta, RemoteAllocator* remote, sizeclass_t sizeclass = sizeclass_t()) - : meta(reinterpret_cast(meta)) + : meta(unsafe_to_uintptr(meta)) { /* remote might be nullptr; cast to uintptr_t before offsetting */ - remote_and_sizeclass = - pointer_offset(reinterpret_cast(remote), sizeclass.raw()); + remote_and_sizeclass = pointer_offset( + unsafe_to_uintptr(remote), sizeclass.raw()); } /** @@ -296,7 +296,7 @@ namespace snmalloc [[nodiscard]] SNMALLOC_FAST_PATH Metaslab* get_metaslab() const { SNMALLOC_ASSERT(get_remote() != nullptr); - return reinterpret_cast(meta & ~BOUNDARY_BIT); + return unsafe_from_uintptr(meta & ~BOUNDARY_BIT); } /** @@ -312,7 +312,7 @@ namespace snmalloc [[nodiscard]] SNMALLOC_FAST_PATH RemoteAllocator* get_remote() const { - return reinterpret_cast( + return unsafe_from_uintptr( pointer_align_down( remote_and_sizeclass)); } diff --git a/src/test/func/external_pagemap/external_pagemap.cc b/src/test/func/external_pagemap/external_pagemap.cc index 6fee03f24..3d139a4b3 100644 --- a/src/test/func/external_pagemap/external_pagemap.cc +++ b/src/test/func/external_pagemap/external_pagemap.cc @@ -20,7 +20,7 @@ int main() auto& global = GlobalChunkmap::pagemap(); SNMALLOC_CHECK(&p == &global); // Get a valid heap address - uintptr_t addr = reinterpret_cast(malloc(42)); + uintptr_t addr = unsafe_to_uintptr(malloc(42)); // Make this very strongly aligned addr &= ~0xfffffULL; void* page = p.page_for_address(addr); diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index b5bd9ff35..4d7ae282f 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -212,7 +212,7 @@ int main(int argc, char** argv) // // Note: We cannot use the check or assert macros here because they depend on // `MessageBuilder` working. They are safe to use in any other test. - void* fakeptr = reinterpret_cast(static_cast(0x42)); + void* fakeptr = unsafe_from_uintptr(static_cast(0x42)); MessageBuilder<1024> b{ "testing pointer {} size_t {} message, {} world, null is {}, -123456 is " "{}, 1234567 is {}", From a022a75b911f4b8714b127893d16e4a35772e421 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Sun, 20 Mar 2022 19:28:03 +0000 Subject: [PATCH 221/302] Fix Debug symbols for Windows CI. (#483) --- CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 131c3e6df..339f4f69a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -239,10 +239,12 @@ endif() function(add_warning_flags name) target_compile_options(${name} PRIVATE - $<$:/Zi /W4 /WX /wd4127 /wd4324 /wd4201 $<${ci_or_debug}:/DEBUG>> + $<$:/Zi /W4 /WX /wd4127 /wd4324 /wd4201> $<$,$>>:-fno-exceptions -fno-rtti -Wall -Wextra -Werror -Wundef> $<$:-Wsign-conversion -Wconversion>) - target_link_options(${name} PRIVATE $<$:-Wl,--no-undefined>) + target_link_options(${name} PRIVATE + $<$:-Wl,--no-undefined> + $<$:$<${ci_or_debug}:/DEBUG>>) endfunction() From a3c8abc3c33d495ba50c17c315d6728ba58e745d Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 18 Mar 2022 14:50:58 +0000 Subject: [PATCH 222/302] Switch atomic_flag to atomic_bool The constructor for atomic_flag is challenging to use in a constexpr. It requires std::atomic_flag flag = ATOMIC_FLAG_INIT; which is not constexpr on some compilers in C++20. Switching to atomic_bool solves this problem. --- src/ds/flaglock.h | 17 ++++++----------- src/pal/pal_ds.h | 9 +++++---- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/ds/flaglock.h b/src/ds/flaglock.h index 6155d2b45..c25f1eed6 100644 --- a/src/ds/flaglock.h +++ b/src/ds/flaglock.h @@ -17,7 +17,7 @@ namespace snmalloc * @brief flag * The underlying atomic field. */ - std::atomic_flag flag = ATOMIC_FLAG_INIT; + std::atomic_bool flag{false}; constexpr DebugFlagWord() = default; @@ -84,7 +84,7 @@ namespace snmalloc */ struct ReleaseFlagWord { - std::atomic_flag flag = ATOMIC_FLAG_INIT; + std::atomic_bool flag{false}; constexpr ReleaseFlagWord() = default; @@ -112,23 +112,18 @@ namespace snmalloc public: FlagLock(FlagWord& lock) : lock(lock) { - while (lock.flag.test_and_set(std::memory_order_acquire)) + while (lock.flag.exchange(true, std::memory_order_acquire)) { // assert_not_owned_by_current_thread is only called when the first // acquiring is failed; which means the lock is already held somewhere // else. lock.assert_not_owned_by_current_thread(); -#ifdef __cpp_lib_atomic_flag_test - // acquire ordering because we need other thread's release to be - // visible. This loop is better for spin-waiting because it won't issue + // This loop is better for spin-waiting because it won't issue // expensive write operation (xchg for example). - while (lock.flag.test(std::memory_order_acquire)) + while (lock.flag.load(std::memory_order_relaxed)) { Aal::pause(); } -#else - Aal::pause(); -#endif } lock.set_owner(); } @@ -136,7 +131,7 @@ namespace snmalloc ~FlagLock() { lock.clear_owner(); - lock.flag.clear(std::memory_order_release); + lock.flag.store(false, std::memory_order_release); } }; } // namespace snmalloc diff --git a/src/pal/pal_ds.h b/src/pal/pal_ds.h index 543fb8a1c..d6197d1e4 100644 --- a/src/pal/pal_ds.h +++ b/src/pal/pal_ds.h @@ -142,10 +142,10 @@ namespace snmalloc void check(uint64_t time_ms) { - static std::atomic_flag lock = ATOMIC_FLAG_INIT; + static std::atomic_bool lock{false}; - // Depulicate calls into here, and make single threaded. - if (lock.test_and_set()) + // Deduplicate calls into here, and make single threaded. + if (lock.exchange(true, std::memory_order_acquire)) return; timers.apply_all([time_ms](PalTimerObject* curr) { @@ -156,7 +156,8 @@ namespace snmalloc curr->pal_notify(curr); } }); - lock.clear(); + + lock.store(false, std::memory_order_release); } }; } // namespace snmalloc From 6ad7f65d19f36eea905b14ae0753114287ff2c09 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 18 Mar 2022 14:55:31 +0000 Subject: [PATCH 223/302] Remove unused code. --- src/ds/dllist.h | 209 -------------------------------------------- src/ds/invalidptr.h | 53 ----------- src/mem/metaslab.h | 1 - 3 files changed, 263 deletions(-) delete mode 100644 src/ds/dllist.h delete mode 100644 src/ds/invalidptr.h diff --git a/src/ds/dllist.h b/src/ds/dllist.h deleted file mode 100644 index 95bb8e568..000000000 --- a/src/ds/dllist.h +++ /dev/null @@ -1,209 +0,0 @@ -#pragma once - -#include "address.h" -#include "helpers.h" -#include "invalidptr.h" -#include "ptrwrap.h" - -#include -#include - -namespace snmalloc -{ - template< - class T, - template typename Ptr = Pointer, - class Terminator = std::nullptr_t, - void on_clear(Ptr) = ignore> - class DLList final - { - private: - static_assert( - std::is_same>::value, - "T->prev must be a Ptr"); - static_assert( - std::is_same>::value, - "T->next must be a Ptr"); - - Ptr head = Terminator(); - Ptr tail = Terminator(); - - public: - ~DLList() - { - clear(); - } - - DLList() = default; - - DLList(DLList&& o) noexcept - { - head = o.head; - tail = o.tail; - - o.head = nullptr; - o.tail = nullptr; - } - - DLList& operator=(DLList&& o) noexcept - { - head = o.head; - tail = o.tail; - - o.head = nullptr; - o.tail = nullptr; - return *this; - } - - SNMALLOC_FAST_PATH bool is_empty() - { - return head == Terminator(); - } - - SNMALLOC_FAST_PATH Ptr get_head() - { - return head; - } - - Ptr get_tail() - { - return tail; - } - - SNMALLOC_FAST_PATH Ptr pop() - { - Ptr item = head; - - if (item != Terminator()) - remove(item); - - return item; - } - - Ptr pop_tail() - { - Ptr item = tail; - - if (item != Terminator()) - remove(item); - - return item; - } - - void insert(Ptr item) - { - debug_check_not_contains(item); - - item->next = head; - item->prev = Terminator(); - - if (head != Terminator()) - head->prev = item; - else - tail = item; - - head = item; - - if constexpr (DEBUG) - debug_check(); - } - - void insert_back(Ptr item) - { - debug_check_not_contains(item); - - item->prev = tail; - item->next = Terminator(); - - if (tail != Terminator()) - tail->next = item; - else - head = item; - - tail = item; - - debug_check(); - } - - SNMALLOC_FAST_PATH void remove(Ptr item) - { - debug_check_contains(item); - - if (item->next != Terminator()) - item->next->prev = item->prev; - else - tail = item->prev; - - if (item->prev != Terminator()) - item->prev->next = item->next; - else - head = item->next; - - debug_check(); - } - - void clear() - { - while (head != nullptr) - { - auto c = head; - remove(c); - on_clear(c); - } - } - - void debug_check_contains(Ptr item) - { - if constexpr (DEBUG) - { - debug_check(); - Ptr curr = head; - - while (curr != item) - { - SNMALLOC_ASSERT(curr != Terminator()); - curr = curr->next; - } - } - else - { - UNUSED(item); - } - } - - void debug_check_not_contains(Ptr item) - { - if constexpr (DEBUG) - { - debug_check(); - Ptr curr = head; - - while (curr != Terminator()) - { - SNMALLOC_ASSERT(curr != item); - curr = curr->next; - } - } - else - { - UNUSED(item); - } - } - - void debug_check() - { - if constexpr (DEBUG) - { - Ptr item = head; - Ptr prev = Terminator(); - - while (item != Terminator()) - { - SNMALLOC_ASSERT(item->prev == prev); - prev = item; - item = item->next; - } - } - } - }; -} // namespace snmalloc diff --git a/src/ds/invalidptr.h b/src/ds/invalidptr.h deleted file mode 100644 index 1a75f0b29..000000000 --- a/src/ds/invalidptr.h +++ /dev/null @@ -1,53 +0,0 @@ -#pragma once - -#include "address.h" - -namespace snmalloc -{ - /** - * Invalid pointer class. This is similar to `std::nullptr_t`, but allows - * other values. - */ - template - struct InvalidPointer - { - /** - * Equality comparison. Two invalid pointer values with the same sentinel - * are always the same, invalid pointer values with different sentinels are - * always different. - */ - template - constexpr bool operator==(const InvalidPointer&) - { - return Sentinel == OtherSentinel; - } - /** - * Equality comparison. Two invalid pointer values with the same sentinel - * are always the same, invalid pointer values with different sentinels are - * always different. - */ - template - constexpr bool operator!=(const InvalidPointer&) - { - return Sentinel != OtherSentinel; - } - /** - * Implicit conversion, creates a pointer with the value of the sentinel. - * On CHERI and other provenance-tracking systems, this is a - * provenance-free integer and so will trap if dereferenced, on other - * systems the sentinel should be a value in unmapped memory. - */ - template - operator T*() const - { - return reinterpret_cast(Sentinel); - } - /** - * Implicit conversion to an address, returns the sentinel value. - */ - operator address_t() const - { - return Sentinel; - } - }; -} // namespace snmalloc diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 947b22de9..e386f16de 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -1,6 +1,5 @@ #pragma once -#include "../ds/dllist.h" #include "../ds/helpers.h" #include "../ds/seqset.h" #include "../mem/remoteallocator.h" From 9d97a388061012fde49d300f1994b1ab785b54ee Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Sun, 13 Mar 2022 15:43:55 +0000 Subject: [PATCH 224/302] Move MetaEntry and MetaCommon to backend These are almost entirely backend concerns, so move their definitions over there. Use C++ friend classes to ensure that MetaCommon structures are opaque to frontend code (at least, at compile time, and neglecting the rest of C++). (These structures contain high-authority pointers and so should be as closely guarded as we can make them.) The bits that leak out are - the encoding of RemoteAllocator* and sizeclass_t into the uinptr_t within a MetaEntry. This, however, is almost entirely a frontend concern, so detach the method definitions from the class and leave those in mem/metaslab.h for the moment. - the size of metadata structures pointed to by the MetaEntry meta field. Rather than use sizeof(Metaslab) (and assert that sizeof(ChunkRecord) is smaller), instead, define PAGEMAP_METADATA_STRUCT_SIZE once and assert that all records fit. Additionally, add an assertion that Metaslab is exactly this size, not for semantic reasons, but because we expect it to be true. The bits that leak in are - the need to zero memory corresponding to a chunk. Rather than having an escape hatch that reveals the MetaCommon.chunk, move the zeroing call into a small wrapper method within the MetaCommon class itself. - the need to get the address of a chunk. We want to assert that we've got the right chunk on occasion (well, at least once so far) and so add a class method to expose the address_t view of the chunk pointer without exposing the pointer itself. --- src/backend/backend.h | 9 +- src/backend/chunkallocator.h | 16 +-- src/backend/largebuddyrange.h | 1 - src/backend/metatypes.h | 196 ++++++++++++++++++++++++++++++++++ src/mem/corealloc.h | 8 +- src/mem/metaslab.h | 188 ++++++++------------------------ 6 files changed, 253 insertions(+), 165 deletions(-) create mode 100644 src/backend/metatypes.h diff --git a/src/backend/backend.h b/src/backend/backend.h index c84e00c02..528abb5d2 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -1,6 +1,5 @@ #pragma once #include "../mem/allocconfig.h" -#include "../mem/metaslab.h" #include "../pal/pal.h" #include "chunkallocator.h" #include "commitrange.h" @@ -8,6 +7,7 @@ #include "empty_range.h" #include "globalrange.h" #include "largebuddyrange.h" +#include "metatypes.h" #include "pagemap.h" #include "pagemapregisterrange.h" #include "palrange.h" @@ -259,7 +259,7 @@ namespace snmalloc SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE); auto meta_cap = - local_state.get_meta_range()->alloc_range(sizeof(Metaslab)); + local_state.get_meta_range()->alloc_range(PAGEMAP_METADATA_STRUCT_SIZE); auto meta = meta_cap.template as_reinterpret().unsafe_ptr(); @@ -277,7 +277,8 @@ namespace snmalloc #endif if (p == nullptr) { - local_state.get_meta_range()->dealloc_range(meta_cap, sizeof(Metaslab)); + local_state.get_meta_range()->dealloc_range( + meta_cap, PAGEMAP_METADATA_STRUCT_SIZE); errno = ENOMEM; #ifdef SNMALLOC_TRACING std::cout << "Out of memory" << std::endl; @@ -300,7 +301,7 @@ namespace snmalloc auto chunk = chunk_record->meta_common.chunk; local_state.get_meta_range()->dealloc_range( - capptr::Chunk(chunk_record), sizeof(Metaslab)); + capptr::Chunk(chunk_record), PAGEMAP_METADATA_STRUCT_SIZE); // TODO, should we set the sizeclass to something specific here? diff --git a/src/backend/chunkallocator.h b/src/backend/chunkallocator.h index 09e4fff24..335f4f5a5 100644 --- a/src/backend/chunkallocator.h +++ b/src/backend/chunkallocator.h @@ -7,6 +7,7 @@ */ #include "../backend/backend_concept.h" +#include "../backend/metatypes.h" #include "../ds/mpmcstack.h" #include "../ds/spmcstack.h" #include "../mem/metaslab.h" @@ -29,22 +30,15 @@ namespace snmalloc MetaCommon meta_common; std::atomic next; }; - static_assert(std::is_standard_layout_v); - static_assert( - offsetof(ChunkRecord, meta_common) == 0, - "ChunkRecord and Metaslab must share a common prefix"); +#if defined(USE_METADATA_CONCEPT) + static_assert(ConceptMetadataStruct); +#endif /** * How many slab sizes that can be provided. */ constexpr size_t NUM_SLAB_SIZES = Pal::address_bits - MIN_CHUNK_BITS; - /** - * Used to ensure the per slab meta data is large enough for both use cases. - */ - static_assert( - sizeof(Metaslab) >= sizeof(ChunkRecord), "We conflate these two types."); - /** * Number of free stacks per chunk size that each allocator will use. * For performance ideally a power of 2. We will return to the central @@ -256,7 +250,7 @@ namespace snmalloc #endif state.add_peak_memory_usage(slab_size); - state.add_peak_memory_usage(sizeof(Metaslab)); + state.add_peak_memory_usage(PAGEMAP_METADATA_STRUCT_SIZE); // TODO handle bounded versus lazy pagemaps in stats state.add_peak_memory_usage( (slab_size / MIN_CHUNK_SIZE) * sizeof(MetaEntry)); diff --git a/src/backend/largebuddyrange.h b/src/backend/largebuddyrange.h index 5f1824840..889f7e046 100644 --- a/src/backend/largebuddyrange.h +++ b/src/backend/largebuddyrange.h @@ -3,7 +3,6 @@ #include "../ds/address.h" #include "../ds/bits.h" #include "../mem/allocconfig.h" -#include "../mem/metaslab.h" #include "../pal/pal.h" #include "buddy.h" #include "range_helpers.h" diff --git a/src/backend/metatypes.h b/src/backend/metatypes.h new file mode 100644 index 000000000..1f089e0e3 --- /dev/null +++ b/src/backend/metatypes.h @@ -0,0 +1,196 @@ +#pragma once +#include "../ds/concept.h" +#include "../mem/allocconfig.h" /* TODO: CACHELINE_SIZE */ +#include "../pal/pal_concept.h" + +namespace snmalloc +{ + /** + * A guaranteed type-stable sub-structure of all metadata referenced by the + * Pagemap. Use-specific structures (Metaslab, ChunkRecord) are expected to + * have this at offset zero so that, even in the face of concurrent mutation + * and reuse of the memory backing that metadata, the types of these fields + * remain fixed. + * + * This class's data is fully private but is friends with the relevant backend + * types and, thus, is "opaque" to the frontend. + */ + class MetaCommon + { + friend class ChunkAllocator; + + template + friend class BackendAllocator; + + capptr::Chunk chunk; + + public: + /** + * Expose the address of, though not the authority to, our corresponding + * chunk. + */ + [[nodiscard]] SNMALLOC_FAST_PATH address_t chunk_address() + { + return address_cast(this->chunk); + } + + /** + * Zero (possibly by unmapping) the memory backing this chunk. We must rely + * on the caller to tell us its size, which is a little unfortunate. + */ + template + SNMALLOC_FAST_PATH void zero_chunk(size_t chunk_size) + { + PAL::zero(this->chunk.unsafe_ptr(), chunk_size); + } + }; + + static const size_t PAGEMAP_METADATA_STRUCT_SIZE = +#ifdef __CHERI_PURE_CAPABILITY__ + 2 * CACHELINE_SIZE +#else + CACHELINE_SIZE +#endif + ; + + // clang-format off + /* This triggers an ICE (C1001) in MSVC, so disable it there */ +#if defined(__cpp_concepts) && !defined(_MSC_VER) + + template + concept ConceptMetadataStruct = + // Metadata structures must be standard layout (for offsetof()), + std::is_standard_layout_v && + // must be of a sufficiently small size, + sizeof(Meta) <= PAGEMAP_METADATA_STRUCT_SIZE && + // and must be pointer-interconvertable with MetaCommon. + ( + ConceptSame || + requires(Meta m) { + // Otherwise, meta must have MetaCommon field named meta_common ... + { &m.meta_common } -> ConceptSame; + // at offset zero. + (offsetof(Meta, meta_common) == 0); + } + ); + + static_assert(ConceptMetadataStruct); +# define USE_METADATA_CONCEPT +#endif + // clang-format on + + struct RemoteAllocator; + class Metaslab; + class sizeclass_t; + + /** + * Entry stored in the pagemap. + */ + class MetaEntry + { + template + friend class BuddyChunkRep; + + /** + * The pointer to the metaslab, the bottom bit is used to indicate if this + * is the first chunk in a PAL allocation, that cannot be combined with + * the preceeding chunk. + */ + uintptr_t meta{0}; + + /** + * Bit used to indicate this should not be considered part of the previous + * PAL allocation. + * + * Some platforms cannot treat different PalAllocs as a single allocation. + * This is true on CHERI as the combined permission might not be + * representable. It is also true on Windows as you cannot Commit across + * multiple continuous VirtualAllocs. + */ + static constexpr address_t BOUNDARY_BIT = 1; + + /** + * A bit-packed pointer to the owning allocator (if any), and the sizeclass + * of this chunk. The sizeclass here is itself a union between two cases: + * + * * log_2(size), at least MIN_CHUNK_BITS, for large allocations. + * + * * a value in [0, NUM_SMALL_SIZECLASSES] for small allocations. These + * may be directly passed to the sizeclass (not slab_sizeclass) functions + * of sizeclasstable.h + * + */ + uintptr_t remote_and_sizeclass{0}; + + public: + constexpr MetaEntry() = default; + + /** + * Constructor, provides the remote and sizeclass embedded in a single + * pointer-sized word. This format is not guaranteed to be stable and so + * the second argument of this must always be the return value from + * `get_remote_and_sizeclass`. + */ + SNMALLOC_FAST_PATH + MetaEntry(Metaslab* meta, uintptr_t remote_and_sizeclass) + : meta(unsafe_to_uintptr(meta)), + remote_and_sizeclass(remote_and_sizeclass) + {} + + /* See mem/metaslab.h */ + SNMALLOC_FAST_PATH + MetaEntry(Metaslab* meta, RemoteAllocator* remote, sizeclass_t sizeclass); + + /** + * Return the Metaslab metadata associated with this chunk, guarded by an + * assert that this chunk is being used as a slab (i.e., has an associated + * owning allocator). + */ + [[nodiscard]] SNMALLOC_FAST_PATH Metaslab* get_metaslab() const + { + SNMALLOC_ASSERT(get_remote() != nullptr); + return unsafe_from_uintptr(meta & ~BOUNDARY_BIT); + } + + /** + * Return the remote and sizeclass in an implementation-defined encoding. + * This is not guaranteed to be stable across snmalloc releases and so the + * only safe use for this is to pass it to the two-argument constructor of + * this class. + */ + [[nodiscard]] SNMALLOC_FAST_PATH uintptr_t get_remote_and_sizeclass() const + { + return remote_and_sizeclass; + } + + /* See mem/metaslab.h */ + [[nodiscard]] SNMALLOC_FAST_PATH RemoteAllocator* get_remote() const; + [[nodiscard]] SNMALLOC_FAST_PATH sizeclass_t get_sizeclass() const; + + MetaEntry(const MetaEntry&) = delete; + + MetaEntry& operator=(const MetaEntry& other) + { + // Don't overwrite the boundary bit with the other's + meta = (other.meta & ~BOUNDARY_BIT) | address_cast(meta & BOUNDARY_BIT); + remote_and_sizeclass = other.remote_and_sizeclass; + return *this; + } + + void set_boundary() + { + meta |= BOUNDARY_BIT; + } + + [[nodiscard]] bool is_boundary() const + { + return meta & BOUNDARY_BIT; + } + + bool clear_boundary_bit() + { + return meta &= ~BOUNDARY_BIT; + } + }; + +} // namespace snmalloc diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 48d326325..3b8e36a61 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -332,7 +332,7 @@ namespace snmalloc SNMALLOC_ASSERT( address_cast(start_of_slab) == - address_cast(chunk_record->meta_common.chunk)); + chunk_record->meta_common.chunk_address()); #if defined(__CHERI_PURE_CAPABILITY__) && !defined(SNMALLOC_CHECK_CLIENT) // Zero the whole slab. For CHERI we at least need to clear the freelist @@ -340,9 +340,9 @@ namespace snmalloc // the freelist order as for SNMALLOC_CHECK_CLIENT. Zeroing the whole slab // may be more friendly to hw because it does not involve pointer chasing // and is amenable to prefetching. - SharedStateHandle::Pal::zero( - chunk_record->meta_common.chunk.unsafe_ptr(), - snmalloc::sizeclass_to_slab_size(sizeclass)); + chunk_record->meta_common + .template zero_chunk( + snmalloc::sizeclass_to_slab_size(sizeclass)); #endif #ifdef SNMALLOC_TRACING diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index e386f16de..fdf2577e8 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -1,5 +1,6 @@ #pragma once +#include "../backend/metatypes.h" #include "../ds/helpers.h" #include "../ds/seqset.h" #include "../mem/remoteallocator.h" @@ -8,18 +9,6 @@ namespace snmalloc { - /** - * A guaranteed type-stable sub-structure of all metadata referenced by the - * Pagemap. Use-specific structures (Metaslab, ChunkRecord) are expected to - * have this at offset zero so that, even in the face of concurrent mutation - * and reuse of the memory backing that metadata, the types of these fields - * remain fixed. - */ - struct MetaCommon - { - capptr::Chunk chunk; - }; - // The Metaslab represent the status of a single slab. class alignas(CACHELINE_SIZE) Metaslab { @@ -216,140 +205,12 @@ namespace snmalloc } }; - static_assert(std::is_standard_layout_v); +#if defined(USE_METADATA_CONCEPT) + static_assert(ConceptMetadataStruct); +#endif static_assert( - offsetof(Metaslab, meta_common) == 0, - "ChunkRecord and Metaslab must share a common prefix"); - - /** - * Entry stored in the pagemap. - */ - class MetaEntry - { - template - friend class BuddyChunkRep; - - /** - * The pointer to the metaslab, the bottom bit is used to indicate if this - * is the first chunk in a PAL allocation, that cannot be combined with - * the preceeding chunk. - */ - uintptr_t meta{0}; - - /** - * Bit used to indicate this should not be considered part of the previous - * PAL allocation. - * - * Some platforms cannot treat different PalAllocs as a single allocation. - * This is true on CHERI as the combined permission might not be - * representable. It is also true on Windows as you cannot Commit across - * multiple continuous VirtualAllocs. - */ - static constexpr address_t BOUNDARY_BIT = 1; - - /** - * A bit-packed pointer to the owning allocator (if any), and the sizeclass - * of this chunk. The sizeclass here is itself a union between two cases: - * - * * log_2(size), at least MIN_CHUNK_BITS, for large allocations. - * - * * a value in [0, NUM_SMALL_SIZECLASSES] for small allocations. These - * may be directly passed to the sizeclass (not slab_sizeclass) functions of - * sizeclasstable.h - * - */ - uintptr_t remote_and_sizeclass{0}; - - public: - constexpr MetaEntry() = default; - - /** - * Constructor, provides the remote and sizeclass embedded in a single - * pointer-sized word. This format is not guaranteed to be stable and so - * the second argument of this must always be the return value from - * `get_remote_and_sizeclass`. - */ - SNMALLOC_FAST_PATH - MetaEntry(Metaslab* meta, uintptr_t remote_and_sizeclass) - : meta(unsafe_to_uintptr(meta)), - remote_and_sizeclass(remote_and_sizeclass) - {} - - SNMALLOC_FAST_PATH - MetaEntry( - Metaslab* meta, - RemoteAllocator* remote, - sizeclass_t sizeclass = sizeclass_t()) - : meta(unsafe_to_uintptr(meta)) - { - /* remote might be nullptr; cast to uintptr_t before offsetting */ - remote_and_sizeclass = pointer_offset( - unsafe_to_uintptr(remote), sizeclass.raw()); - } - - /** - * Return the Metaslab metadata associated with this chunk, guarded by an - * assert that this chunk is being used as a slab (i.e., has an associated - * owning allocator). - */ - [[nodiscard]] SNMALLOC_FAST_PATH Metaslab* get_metaslab() const - { - SNMALLOC_ASSERT(get_remote() != nullptr); - return unsafe_from_uintptr(meta & ~BOUNDARY_BIT); - } - - /** - * Return the remote and sizeclass in an implementation-defined encoding. - * This is not guaranteed to be stable across snmalloc releases and so the - * only safe use for this is to pass it to the two-argument constructor of - * this class. - */ - [[nodiscard]] SNMALLOC_FAST_PATH uintptr_t get_remote_and_sizeclass() const - { - return remote_and_sizeclass; - } - - [[nodiscard]] SNMALLOC_FAST_PATH RemoteAllocator* get_remote() const - { - return unsafe_from_uintptr( - pointer_align_down( - remote_and_sizeclass)); - } - - [[nodiscard]] SNMALLOC_FAST_PATH sizeclass_t get_sizeclass() const - { - // TODO: perhaps remove static_cast with resolution of - // https://github.com/CTSRD-CHERI/llvm-project/issues/588 - return sizeclass_t::from_raw( - static_cast(remote_and_sizeclass) & - (REMOTE_WITH_BACKEND_MARKER_ALIGN - 1)); - } - - MetaEntry(const MetaEntry&) = delete; - - MetaEntry& operator=(const MetaEntry& other) - { - // Don't overwrite the boundary bit with the other's - meta = (other.meta & ~BOUNDARY_BIT) | address_cast(meta & BOUNDARY_BIT); - remote_and_sizeclass = other.remote_and_sizeclass; - return *this; - } - - void set_boundary() - { - meta |= BOUNDARY_BIT; - } - - [[nodiscard]] bool is_boundary() const - { - return meta & BOUNDARY_BIT; - } - - bool clear_boundary_bit() - { - return meta &= ~BOUNDARY_BIT; - } - }; + sizeof(Metaslab) == PAGEMAP_METADATA_STRUCT_SIZE, + "Metaslab is expected to be the largest pagemap metadata record"); struct MetaslabCache { @@ -363,4 +224,41 @@ namespace snmalloc uint16_t unused = 0; uint16_t length = 0; }; + + /* + * MetaEntry methods that deal with RemoteAllocator* and sizeclass_t are here, + * so that the backend does not need to know the details and can, instead, + * just provide the storage space. + */ + + SNMALLOC_FAST_PATH_INLINE + MetaEntry::MetaEntry( + Metaslab* meta, + RemoteAllocator* remote, + sizeclass_t sizeclass = sizeclass_t()) + : meta(reinterpret_cast(meta)) + { + /* remote might be nullptr; cast to uintptr_t before offsetting */ + remote_and_sizeclass = + pointer_offset(reinterpret_cast(remote), sizeclass.raw()); + } + + [[nodiscard]] SNMALLOC_FAST_PATH_INLINE RemoteAllocator* + MetaEntry::get_remote() const + { + return reinterpret_cast( + pointer_align_down( + remote_and_sizeclass)); + } + + [[nodiscard]] SNMALLOC_FAST_PATH_INLINE sizeclass_t + MetaEntry::get_sizeclass() const + { + // TODO: perhaps remove static_cast with resolution of + // https://github.com/CTSRD-CHERI/llvm-project/issues/588 + return sizeclass_t::from_raw( + static_cast(remote_and_sizeclass) & + (REMOTE_WITH_BACKEND_MARKER_ALIGN - 1)); + } + } // namespace snmalloc From b5a66131bd5fa36dca2b8eed71d9a8ad30ed07cf Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Wed, 16 Mar 2022 09:58:33 +0000 Subject: [PATCH 225/302] Move BACKEND_MARKER to backend/metatypes And rename it to REMOTE_BACKEND_MARKER to scope it a bit. --- src/backend/largebuddyrange.h | 3 ++- src/backend/metatypes.h | 11 +++++++++++ src/mem/remoteallocator.h | 9 ++++----- src/mem/remotecache.h | 2 +- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/backend/largebuddyrange.h b/src/backend/largebuddyrange.h index 889f7e046..defcc5650 100644 --- a/src/backend/largebuddyrange.h +++ b/src/backend/largebuddyrange.h @@ -29,7 +29,8 @@ namespace snmalloc { SNMALLOC_ASSERT((r & (MIN_CHUNK_SIZE - 1)) == 0); // Preserve lower bits. - *ptr = r | address_cast(*ptr & (MIN_CHUNK_SIZE - 1)) | BACKEND_MARKER; + *ptr = r | address_cast(*ptr & (MIN_CHUNK_SIZE - 1)) | + MetaEntry::REMOTE_BACKEND_MARKER; } static Contents get(const Holder* ptr) diff --git a/src/backend/metatypes.h b/src/backend/metatypes.h index 1f089e0e3..b3c247358 100644 --- a/src/backend/metatypes.h +++ b/src/backend/metatypes.h @@ -123,6 +123,17 @@ namespace snmalloc uintptr_t remote_and_sizeclass{0}; public: + /** + * This bit is set in remote_and_sizeclass to discriminate between the case + * that it is in use by the frontend (0) or by the backend (1). For the + * former case, see mem/metaslab.h; for the latter, see backend/backend.h + * and backend/largebuddyrange.h. + * + * This value is statically checked by the frontend to ensure that its + * bit packing does not conflict; see mem/remoteallocator.h + */ + static constexpr address_t REMOTE_BACKEND_MARKER = 1 << 7; + constexpr MetaEntry() = default; /** diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index f8b309eab..2853c78df 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -1,5 +1,6 @@ #pragma once +#include "../backend/metatypes.h" #include "../mem/allocconfig.h" #include "../mem/freelist.h" #include "../mem/sizeclasstable.h" @@ -15,14 +16,12 @@ namespace snmalloc static constexpr size_t REMOTE_MIN_ALIGN = bits::max(CACHELINE_SIZE, SIZECLASS_REP_SIZE) << 1; - // This bit is set on the RemoteAllocator* to indicate it is - // actually being used by the backend for some other use. - static constexpr size_t BACKEND_MARKER = REMOTE_MIN_ALIGN >> 1; - // The bit above the sizeclass is always zero unless this is used // by the backend to represent another datastructure such as the buddy // allocator entries. - constexpr size_t REMOTE_WITH_BACKEND_MARKER_ALIGN = BACKEND_MARKER; + constexpr size_t REMOTE_WITH_BACKEND_MARKER_ALIGN = + MetaEntry::REMOTE_BACKEND_MARKER; + static_assert((REMOTE_MIN_ALIGN >> 1) == MetaEntry::REMOTE_BACKEND_MARKER); /** * Global key for all remote lists. diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index 5e7559971..c80944d93 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -108,7 +108,7 @@ namespace snmalloc // set implies this is used by the backend, and we should not be // deallocating memory here. snmalloc_check_client( - (address_cast(remote) & BACKEND_MARKER) == 0, + (address_cast(remote) & MetaEntry::REMOTE_BACKEND_MARKER) == 0, "Delayed detection of attempt to free internal structure."); if constexpr (SharedStateHandle::Options.QueueHeadsAreTame) { From 772e46f8784ceb3539a8d8173c95ea7c61c467e3 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Wed, 16 Mar 2022 09:57:14 +0000 Subject: [PATCH 226/302] Additional commentary and more verbose names --- src/backend/backend.h | 3 ++- src/backend/largebuddyrange.h | 32 +++++++++++++++++++++++++++++-- src/backend/metatypes.h | 36 +++++++++++++++++------------------ 3 files changed, 49 insertions(+), 22 deletions(-) diff --git a/src/backend/backend.h b/src/backend/backend.h index 528abb5d2..e81db0803 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -217,7 +217,8 @@ namespace snmalloc * * The template argument is the type of the metadata being allocated. This * allows the backend to allocate different types of metadata in different - * places or with different policies. + * places or with different policies. The default implementation, here, + * does not avail itself of this degree of freedom. */ template static capptr::Chunk diff --git a/src/backend/largebuddyrange.h b/src/backend/largebuddyrange.h index defcc5650..5b2954de2 100644 --- a/src/backend/largebuddyrange.h +++ b/src/backend/largebuddyrange.h @@ -18,17 +18,45 @@ namespace snmalloc class BuddyChunkRep { public: + /* + * The values we store in our rbtree are the addresses of (combined spans + * of) chunks of the address space; as such, bits in (MIN_CHUNK_SIZE - 1) + * are unused and so the RED_BIT is packed therein. However, in practice, + * these are not "just any" uintptr_t-s, but specifically the uintptr_t-s + * inside the Pagemap's MetaEntry structures. As such, there are some + * additional bit-swizzling concerns; see set() and get() below. + */ using Holder = uintptr_t; using Contents = uintptr_t; - static constexpr address_t RED_BIT = 2; + static constexpr address_t RED_BIT = 1 << 1; + + static_assert(RED_BIT < MIN_CHUNK_SIZE); + static_assert(RED_BIT != MetaEntry::META_BOUNDARY_BIT); + static_assert(RED_BIT != MetaEntry::REMOTE_BACKEND_MARKER); static constexpr Contents null = 0; static void set(Holder* ptr, Contents r) { SNMALLOC_ASSERT((r & (MIN_CHUNK_SIZE - 1)) == 0); - // Preserve lower bits. + /* + * Preserve lower bits, claim as backend, and update contents of holder. + * + * This is excessive at present but no harder than being more precise + * while also being future-proof. All that is strictly required would be + * to preserve META_BOUNDARY_BIT and RED_BIT in ->meta and to assert + * REMOTE_BACKEND_MARKER in ->remote_and_sizeclass (if it isn't already + * asserted). However, we don't know which Holder* we have been given, + * nor do we know whether this Holder* is completely new (and so we are + * the first reasonable opportunity to assert REMOTE_BACKEND_MARKER) or + * recycled from the frontend, and so we preserve and assert more than + * strictly necessary. + * + * The use of `address_cast` below is a CHERI-ism; otherwise both `r` and + * `*ptr & ...` are plausibly provenance-carrying values and the compiler + * balks at the ambiguity. + */ *ptr = r | address_cast(*ptr & (MIN_CHUNK_SIZE - 1)) | MetaEntry::REMOTE_BACKEND_MARKER; } diff --git a/src/backend/metatypes.h b/src/backend/metatypes.h index b3c247358..c65efaa4c 100644 --- a/src/backend/metatypes.h +++ b/src/backend/metatypes.h @@ -84,7 +84,8 @@ namespace snmalloc class sizeclass_t; /** - * Entry stored in the pagemap. + * Entry stored in the pagemap. See docs/AddressSpace.md for the full + * MetaEntry lifecycle. */ class MetaEntry { @@ -92,9 +93,11 @@ namespace snmalloc friend class BuddyChunkRep; /** - * The pointer to the metaslab, the bottom bit is used to indicate if this - * is the first chunk in a PAL allocation, that cannot be combined with - * the preceeding chunk. + * In common cases, the pointer to the metaslab. See docs/AddressSpace.md + * for additional details. + * + * The bottom bit is used to indicate if this is the first chunk in a PAL + * allocation, that cannot be combined with the preceeding chunk. */ uintptr_t meta{0}; @@ -107,18 +110,12 @@ namespace snmalloc * representable. It is also true on Windows as you cannot Commit across * multiple continuous VirtualAllocs. */ - static constexpr address_t BOUNDARY_BIT = 1; + static constexpr address_t META_BOUNDARY_BIT = 1 << 0; /** - * A bit-packed pointer to the owning allocator (if any), and the sizeclass - * of this chunk. The sizeclass here is itself a union between two cases: - * - * * log_2(size), at least MIN_CHUNK_BITS, for large allocations. - * - * * a value in [0, NUM_SMALL_SIZECLASSES] for small allocations. These - * may be directly passed to the sizeclass (not slab_sizeclass) functions - * of sizeclasstable.h - * + * In common cases, a bit-packed pointer to the owning allocator (if any), + * and the sizeclass of this chunk. See mem/metaslab.h:MetaEntryRemote for + * details of this case and docs/AddressSpace.md for further details. */ uintptr_t remote_and_sizeclass{0}; @@ -160,7 +157,7 @@ namespace snmalloc [[nodiscard]] SNMALLOC_FAST_PATH Metaslab* get_metaslab() const { SNMALLOC_ASSERT(get_remote() != nullptr); - return unsafe_from_uintptr(meta & ~BOUNDARY_BIT); + return unsafe_from_uintptr(meta & ~META_BOUNDARY_BIT); } /** @@ -183,24 +180,25 @@ namespace snmalloc MetaEntry& operator=(const MetaEntry& other) { // Don't overwrite the boundary bit with the other's - meta = (other.meta & ~BOUNDARY_BIT) | address_cast(meta & BOUNDARY_BIT); + meta = (other.meta & ~META_BOUNDARY_BIT) | + address_cast(meta & META_BOUNDARY_BIT); remote_and_sizeclass = other.remote_and_sizeclass; return *this; } void set_boundary() { - meta |= BOUNDARY_BIT; + meta |= META_BOUNDARY_BIT; } [[nodiscard]] bool is_boundary() const { - return meta & BOUNDARY_BIT; + return meta & META_BOUNDARY_BIT; } bool clear_boundary_bit() { - return meta &= ~BOUNDARY_BIT; + return meta &= ~META_BOUNDARY_BIT; } }; From 7940fee00cf0ce43c3c40584a3b3e4fb8d77cfd8 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Wed, 16 Mar 2022 13:02:26 +0000 Subject: [PATCH 227/302] Refactor MetaEntry remote_and_sizeclass Introduce a class that we can use to more completely separate the frontend encoding details from the backend. --- src/backend/backend.h | 22 ++++++---- src/backend/backend_concept.h | 4 +- src/backend/chunkallocator.h | 9 ++-- src/backend/metatypes.h | 41 +++++++------------ src/mem/corealloc.h | 39 ++++++++++-------- src/mem/localalloc.h | 29 ++++++------- src/mem/metaslab.h | 77 +++++++++++++++++++++-------------- src/mem/remotecache.h | 12 +++--- 8 files changed, 124 insertions(+), 109 deletions(-) diff --git a/src/backend/backend.h b/src/backend/backend.h index e81db0803..703d0d2c7 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -59,10 +59,14 @@ namespace snmalloc * Set template parameter to true if it not an error * to access a location that is not backed by a chunk. */ - template - SNMALLOC_FAST_PATH static const MetaEntry& get_metaentry(address_t p) + template + SNMALLOC_FAST_PATH static const Ret& get_metaentry(address_t p) { - return concretePagemap.template get(p); + static_assert( + std::is_base_of_v && sizeof(MetaEntry) == sizeof(Ret), + "Backend Pagemap get_metaentry return must look like MetaEntry"); + return static_cast( + concretePagemap.template get(p)); } /** @@ -250,15 +254,15 @@ namespace snmalloc * (remote, sizeclass, metaslab) * where metaslab, is the second element of the pair return. */ - static std::pair, Metaslab*> alloc_chunk( - LocalState& local_state, - size_t size, - RemoteAllocator* remote, - sizeclass_t sizeclass) + static std::pair, Metaslab*> + alloc_chunk(LocalState& local_state, size_t size, uintptr_t ras) { SNMALLOC_ASSERT(bits::is_pow2(size)); SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE); + SNMALLOC_ASSERT((ras & MetaEntry::REMOTE_BACKEND_MARKER) == 0); + ras &= ~MetaEntry::REMOTE_BACKEND_MARKER; + auto meta_cap = local_state.get_meta_range()->alloc_range(PAGEMAP_METADATA_STRUCT_SIZE); @@ -289,7 +293,7 @@ namespace snmalloc meta->meta_common.chunk = p; - MetaEntry t(meta, remote, sizeclass); + MetaEntry t(&meta->meta_common, ras); Pagemap::set_metaentry(address_cast(p), size, t); p = Aal::capptr_bound(p, size); diff --git a/src/backend/backend_concept.h b/src/backend/backend_concept.h index 895e19af2..c0d8f8ef0 100644 --- a/src/backend/backend_concept.h +++ b/src/backend/backend_concept.h @@ -24,10 +24,10 @@ namespace snmalloc { { Meta::set_metaentry(addr, sz, t) } -> ConceptSame; - { Meta::template get_metaentry(addr) } + { Meta::template get_metaentry(addr) } -> ConceptSame; - { Meta::template get_metaentry(addr) } + { Meta::template get_metaentry(addr) } -> ConceptSame; }; diff --git a/src/backend/chunkallocator.h b/src/backend/chunkallocator.h index 335f4f5a5..80d1be5d1 100644 --- a/src/backend/chunkallocator.h +++ b/src/backend/chunkallocator.h @@ -183,10 +183,9 @@ namespace snmalloc static std::pair, Metaslab*> alloc_chunk( typename SharedStateHandle::LocalState& local_state, ChunkAllocatorLocalState& chunk_alloc_local_state, - sizeclass_t sizeclass, chunksizeclass_t slab_sizeclass, size_t slab_size, - RemoteAllocator* remote) + uintptr_t ras) { using PAL = typename SharedStateHandle::Pal; ChunkAllocatorState& state = @@ -234,7 +233,7 @@ namespace snmalloc << " memory in stacks " << state.memory_in_stacks << std::endl; #endif - MetaEntry entry{meta, remote, sizeclass}; + MetaEntry entry{&meta->meta_common, ras}; SharedStateHandle::Pagemap::set_metaentry( address_cast(slab), slab_size, entry); return {slab, meta}; @@ -242,8 +241,8 @@ namespace snmalloc // Allocate a fresh slab as there are no available ones. // First create meta-data - auto [slab, meta] = SharedStateHandle::alloc_chunk( - &local_state, slab_size, remote, sizeclass); + auto [slab, meta] = + SharedStateHandle::alloc_chunk(&local_state, slab_size, ras); #ifdef SNMALLOC_TRACING std::cout << "Create slab:" << slab.unsafe_ptr() << " slab_sizeclass " << slab_sizeclass << " size " << slab_size << std::endl; diff --git a/src/backend/metatypes.h b/src/backend/metatypes.h index c65efaa4c..66462034a 100644 --- a/src/backend/metatypes.h +++ b/src/backend/metatypes.h @@ -79,10 +79,6 @@ namespace snmalloc #endif // clang-format on - struct RemoteAllocator; - class Metaslab; - class sizeclass_t; - /** * Entry stored in the pagemap. See docs/AddressSpace.md for the full * MetaEntry lifecycle. @@ -140,41 +136,23 @@ namespace snmalloc * `get_remote_and_sizeclass`. */ SNMALLOC_FAST_PATH - MetaEntry(Metaslab* meta, uintptr_t remote_and_sizeclass) - : meta(unsafe_to_uintptr(meta)), + MetaEntry(MetaCommon* meta, uintptr_t remote_and_sizeclass) + : meta(unsafe_to_uintptr(meta)), remote_and_sizeclass(remote_and_sizeclass) {} - /* See mem/metaslab.h */ - SNMALLOC_FAST_PATH - MetaEntry(Metaslab* meta, RemoteAllocator* remote, sizeclass_t sizeclass); - - /** - * Return the Metaslab metadata associated with this chunk, guarded by an - * assert that this chunk is being used as a slab (i.e., has an associated - * owning allocator). - */ - [[nodiscard]] SNMALLOC_FAST_PATH Metaslab* get_metaslab() const - { - SNMALLOC_ASSERT(get_remote() != nullptr); - return unsafe_from_uintptr(meta & ~META_BOUNDARY_BIT); - } - /** * Return the remote and sizeclass in an implementation-defined encoding. * This is not guaranteed to be stable across snmalloc releases and so the * only safe use for this is to pass it to the two-argument constructor of * this class. */ - [[nodiscard]] SNMALLOC_FAST_PATH uintptr_t get_remote_and_sizeclass() const + [[nodiscard]] SNMALLOC_FAST_PATH const uintptr_t& + get_remote_and_sizeclass() const { return remote_and_sizeclass; } - /* See mem/metaslab.h */ - [[nodiscard]] SNMALLOC_FAST_PATH RemoteAllocator* get_remote() const; - [[nodiscard]] SNMALLOC_FAST_PATH sizeclass_t get_sizeclass() const; - MetaEntry(const MetaEntry&) = delete; MetaEntry& operator=(const MetaEntry& other) @@ -186,6 +164,16 @@ namespace snmalloc return *this; } + /** + * Return the Metaslab metadata associated with this chunk, guarded by an + * assert that this chunk is being used as a slab (i.e., has an associated + * owning allocator). + */ + [[nodiscard]] SNMALLOC_FAST_PATH MetaCommon* get_meta() const + { + return reinterpret_cast(meta & ~META_BOUNDARY_BIT); + } + void set_boundary() { meta |= META_BOUNDARY_BIT; @@ -201,5 +189,4 @@ namespace snmalloc return meta &= ~META_BOUNDARY_BIT; } }; - } // namespace snmalloc diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 3b8e36a61..b22fbb0ac 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -403,7 +403,8 @@ namespace snmalloc * by this thread, or handling the final deallocation onto a slab, * so it can be reused by other threads. */ - SNMALLOC_SLOW_PATH void dealloc_local_object_slow(const MetaEntry& entry) + SNMALLOC_SLOW_PATH void + dealloc_local_object_slow(const MetaslabMetaEntry& entry) { // TODO: Handle message queue on this path? @@ -486,19 +487,20 @@ namespace snmalloc [local_state](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { return capptr_domesticate(local_state, p); }; - auto cb = [this, &need_post](freelist::HeadPtr msg) - SNMALLOC_FAST_PATH_LAMBDA { + auto cb = [this, + &need_post](freelist::HeadPtr msg) SNMALLOC_FAST_PATH_LAMBDA { #ifdef SNMALLOC_TRACING - std::cout << "Handling remote" << std::endl; + std::cout << "Handling remote" << std::endl; #endif - auto& entry = SharedStateHandle::Pagemap::get_metaentry( - snmalloc::address_cast(msg)); + auto& entry = + SharedStateHandle::Pagemap::template get_metaentry( + snmalloc::address_cast(msg)); - handle_dealloc_remote(entry, msg.as_void(), need_post); + handle_dealloc_remote(entry, msg.as_void(), need_post); - return true; - }; + return true; + }; if constexpr (SharedStateHandle::Options.QueueHeadsAreTame) { @@ -532,7 +534,7 @@ namespace snmalloc * need_post will be set to true, if capacity is exceeded. */ void handle_dealloc_remote( - const MetaEntry& entry, + const MetaslabMetaEntry& entry, CapPtr p, bool& need_post) { @@ -672,8 +674,10 @@ namespace snmalloc SNMALLOC_FAST_PATH void dealloc_local_object(CapPtr p) { - const MetaEntry& entry = - SharedStateHandle::Pagemap::get_metaentry(snmalloc::address_cast(p)); + // MetaEntry-s seen here are expected to have meaningful Remote pointers + auto& entry = + SharedStateHandle::Pagemap::template get_metaentry( + snmalloc::address_cast(p)); if (SNMALLOC_LIKELY(dealloc_local_object_fast(entry, p, entropy))) return; @@ -681,7 +685,7 @@ namespace snmalloc } SNMALLOC_FAST_PATH static bool dealloc_local_object_fast( - const MetaEntry& entry, + const MetaslabMetaEntry& entry, CapPtr p, LocalEntropy& entropy) { @@ -786,8 +790,8 @@ namespace snmalloc auto [slab, meta] = SharedStateHandle::alloc_chunk( get_backend_local_state(), slab_size, - public_state(), - sizeclass_t::from_small_class(sizeclass)); + MetaslabMetaEntry::encode( + public_state(), sizeclass_t::from_small_class(sizeclass))); if (slab == nullptr) { @@ -840,8 +844,9 @@ namespace snmalloc { bool need_post = true; // Always going to post, so ignore. auto n_tame = p_tame->atomic_read_next(key_global, domesticate); - auto& entry = SharedStateHandle::Pagemap::get_metaentry( - snmalloc::address_cast(p_tame)); + const MetaslabMetaEntry& entry = + SharedStateHandle::Pagemap::template get_metaentry< + MetaslabMetaEntry>(snmalloc::address_cast(p_tame)); handle_dealloc_remote(entry, p_tame.as_void(), need_post); p_tame = n_tame; } diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index d9bf18493..db32d6165 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -183,8 +183,8 @@ namespace snmalloc auto [chunk, meta] = SharedStateHandle::alloc_chunk( core_alloc->get_backend_local_state(), large_size_to_chunk_size(size), - core_alloc->public_state(), - size_to_sizeclass_full(size)); + MetaslabMetaEntry::encode( + core_alloc->public_state(), size_to_sizeclass_full(size))); // set up meta data so sizeclass is correct, and hence alloc size, and // external pointer. #ifdef SNMALLOC_TRACING @@ -266,8 +266,9 @@ namespace snmalloc std::cout << "Remote dealloc post" << p.unsafe_ptr() << " size " << alloc_size(p.unsafe_ptr()) << std::endl; #endif - const MetaEntry& entry = - SharedStateHandle::Pagemap::get_metaentry(address_cast(p)); + const MetaslabMetaEntry& entry = + SharedStateHandle::Pagemap::template get_metaentry( + address_cast(p)); local_cache.remote_dealloc_cache.template dealloc( entry.get_remote()->trunc_id(), p, key_global); post_remote_cache(); @@ -624,8 +625,9 @@ namespace snmalloc capptr::Alloc p_tame = capptr_domesticate( core_alloc->backend_state_ptr(), p_wild); - const MetaEntry& entry = - SharedStateHandle::Pagemap::get_metaentry(address_cast(p_tame)); + const MetaslabMetaEntry& entry = + SharedStateHandle::Pagemap::template get_metaentry( + address_cast(p_tame)); if (SNMALLOC_LIKELY(local_cache.remote_allocator == entry.get_remote())) { # if defined(__CHERI_PURE_CAPABILITY__) && defined(SNMALLOC_CHECK_CLIENT) @@ -714,8 +716,9 @@ namespace snmalloc // To handle this case we require the uninitialised pagemap contain an // entry for the first chunk of memory, that states it represents a // large object, so we can pull the check for null off the fast path. - const MetaEntry& entry = - SharedStateHandle::Pagemap::get_metaentry(address_cast(p_raw)); + const MetaslabMetaEntry& entry = + SharedStateHandle::Pagemap::template get_metaentry( + address_cast(p_raw)); return sizeclass_full_to_size(entry.get_sizeclass()); #endif @@ -759,9 +762,8 @@ namespace snmalloc size_t remaining_bytes(const void* p) { #ifndef SNMALLOC_PASS_THROUGH - const MetaEntry& entry = - SharedStateHandle::Pagemap::template get_metaentry( - address_cast(p)); + const MetaslabMetaEntry& entry = SharedStateHandle::Pagemap:: + template get_metaentry(address_cast(p)); auto sizeclass = entry.get_sizeclass(); return snmalloc::remaining_bytes(sizeclass, address_cast(p)); @@ -788,9 +790,8 @@ namespace snmalloc size_t index_in_object(const void* p) { #ifndef SNMALLOC_PASS_THROUGH - const MetaEntry& entry = - SharedStateHandle::Pagemap::template get_metaentry( - address_cast(p)); + const MetaslabMetaEntry& entry = SharedStateHandle::Pagemap:: + template get_metaentry(address_cast(p)); auto sizeclass = entry.get_sizeclass(); return snmalloc::index_in_object(sizeclass, address_cast(p)); diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index fdf2577e8..269511465 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -226,39 +226,56 @@ namespace snmalloc }; /* - * MetaEntry methods that deal with RemoteAllocator* and sizeclass_t are here, - * so that the backend does not need to know the details and can, instead, - * just provide the storage space. + * A convenience wrapper aroun MetaEntry with a meaningful RemoteAllocator + * pointer. This encodes a RemoteAllocator* and a sizeclass_t into a the + * uintptr_t remote_and_sizeclass field. + * + * There's a little bit of an asymmetry here. Since the backend actually sets + * the entry (when associating a metadata structure), MetaslabMetaEntry-s are + * not constructed directly; please use ::encode(). On the other hand, the + * backend's Pagemap::get_metaentry() method is templated on its return type, + * so it is relatively straightforward to view a pagemap entry as a + * MetaslabMetaEntry and then use the accessors here for decoding. */ - - SNMALLOC_FAST_PATH_INLINE - MetaEntry::MetaEntry( - Metaslab* meta, - RemoteAllocator* remote, - sizeclass_t sizeclass = sizeclass_t()) - : meta(reinterpret_cast(meta)) + struct MetaslabMetaEntry : public MetaEntry { - /* remote might be nullptr; cast to uintptr_t before offsetting */ - remote_and_sizeclass = - pointer_offset(reinterpret_cast(remote), sizeclass.raw()); - } + /// Perform the encoding. + static SNMALLOC_FAST_PATH uintptr_t + encode(RemoteAllocator* remote, sizeclass_t sizeclass) + { + /* remote might be nullptr; cast to uintptr_t before offsetting */ + return pointer_offset( + reinterpret_cast(remote), sizeclass.raw()); + } - [[nodiscard]] SNMALLOC_FAST_PATH_INLINE RemoteAllocator* - MetaEntry::get_remote() const - { - return reinterpret_cast( - pointer_align_down( - remote_and_sizeclass)); - } + [[nodiscard]] SNMALLOC_FAST_PATH RemoteAllocator* get_remote() const + { + return reinterpret_cast( + pointer_align_down( + get_remote_and_sizeclass())); + } - [[nodiscard]] SNMALLOC_FAST_PATH_INLINE sizeclass_t - MetaEntry::get_sizeclass() const - { - // TODO: perhaps remove static_cast with resolution of - // https://github.com/CTSRD-CHERI/llvm-project/issues/588 - return sizeclass_t::from_raw( - static_cast(remote_and_sizeclass) & - (REMOTE_WITH_BACKEND_MARKER_ALIGN - 1)); - } + [[nodiscard]] SNMALLOC_FAST_PATH sizeclass_t get_sizeclass() const + { + // TODO: perhaps remove static_cast with resolution of + // https://github.com/CTSRD-CHERI/llvm-project/issues/588 + return sizeclass_t::from_raw( + static_cast(get_remote_and_sizeclass()) & + (REMOTE_WITH_BACKEND_MARKER_ALIGN - 1)); + } + + /** + * Return the Metaslab metadata associated with this chunk, guarded by an + * assert that this chunk is being used as a slab (i.e., has an associated + * owning allocator). + */ + [[nodiscard]] SNMALLOC_FAST_PATH Metaslab* get_metaslab() const + { + SNMALLOC_ASSERT(get_remote() != nullptr); + return reinterpret_cast(get_meta()); + } + }; + + static_assert(sizeof(MetaslabMetaEntry) == sizeof(MetaEntry)); } // namespace snmalloc diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index c80944d93..e97df1a47 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -52,7 +52,7 @@ namespace snmalloc * * This does not require initialisation to be safely called. */ - SNMALLOC_FAST_PATH bool reserve_space(const MetaEntry& entry) + SNMALLOC_FAST_PATH bool reserve_space(const MetaslabMetaEntry& entry) { auto size = static_cast(sizeclass_full_to_size(entry.get_sizeclass())); @@ -101,8 +101,9 @@ namespace snmalloc if (!list[i].empty()) { auto [first, last] = list[i].extract_segment(key); - const MetaEntry& entry = - SharedStateHandle::Pagemap::get_metaentry(address_cast(first)); + const MetaslabMetaEntry& entry = + SharedStateHandle::Pagemap::template get_metaentry< + MetaslabMetaEntry>(address_cast(first)); auto remote = entry.get_remote(); // If the allocator is not correctly aligned, then the bit that is // set implies this is used by the backend, and we should not be @@ -141,8 +142,9 @@ namespace snmalloc // Use the next N bits to spread out remote deallocs in our own // slot. auto r = resend.take(key, domesticate); - const MetaEntry& entry = - SharedStateHandle::Pagemap::get_metaentry(address_cast(r)); + const MetaslabMetaEntry& entry = + SharedStateHandle::Pagemap::template get_metaentry< + MetaslabMetaEntry>(address_cast(r)); auto i = entry.get_remote()->trunc_id(); size_t slot = get_slot(i, post_round); list[slot].add(r, key); From a0377f62eb253192ae6d16a72a5032b143f66304 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Mon, 2 Aug 2021 21:40:18 +0100 Subject: [PATCH 228/302] Add docs/AddressSpace.md --- docs/AddressSpace.md | 157 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 docs/AddressSpace.md diff --git a/docs/AddressSpace.md b/docs/AddressSpace.md new file mode 100644 index 000000000..f72a8d077 --- /dev/null +++ b/docs/AddressSpace.md @@ -0,0 +1,157 @@ +# How snmalloc Manages Address Space + +Like any modern, high-performance allocator, `snmalloc` contains multiple layers of allocation. +We give here some notes on the internal orchestration. + +## From platform to malloc + +Consider a first, "small" allocation (typically less than a platform page); such allocations showcase more of the machinery. +For simplicity, we assume that + +- this is not an `OPEN_ENCLAVE` build, +- the `BackendAllocator` has not been told to use a `fixed_range`, +- this is not a `SNMALLOC_CHECK_CLIENT` build, and +- (as a consequence of the above) `SNMALLOC_META_PROTECTED` is not `#define`-d. + +Since this is the first allocation, all the internal caches will be empty, and so we will hit all the slow paths. +For simplicity, we gloss over much of the "lazy initialization" that would actually be implied by a first allocation. + +1. The `LocalAlloc::small_alloc` finds that it cannot satisfy the request because its `LocalCache` lacks a free list for this size class. + The request is delegated, unchanged, to `CoreAllocator::small_alloc`. + +2. The `CoreAllocator` has no active slab for this sizeclass, so `CoreAllocator::small_alloc_slow` delegates to `BackendAllocator::alloc_chunk`. + At this point, the allocation request is enlarged to one or a few chunks (a small counting number multiple of `MIN_CHUNK_SIZE`, which is typically 16KiB); see `sizeclass_to_slab_size`. + +3. `BackendAllocator::alloc_chunk` at this point splits the allocation request in two, allocating both the chunk's metadata structure (of size `PAGEMAP_METADATA_STRUCT_SIZE`) and the chunk itself (a multiple of `MIN_CHUNK_SIZE`). + Because the two exercise similar bits of machinery, we now track them in parallel in prose despite their sequential nature. + +4. The `BackendAllocator` has a chain of "range" types that it uses to manage address space. + By default (and in the case we are considering), that chain begins with a per-thread "small buddy allocator range". + + 1. For the metadata allocation, the size is (well) below `MIN_CHUNK_SIZE` and so this allocator, which by supposition is empty, attempts to `refill` itself from its parent. + This results in a request for a `MIN_CHUNK_SIZE` chunk from the parent allocator. + + 2. For the chunk allocation, the size is `MIN_CHUNK_SIZE` or larger, so this allocator immediately forwards the request to its parent. + +5. The next range allocator in the chain is a per-thread *large* buddy allocator that refills in 2 MiB granules. + (2 MiB chosen because it is a typical superpage size.) + At this point, both requests are for at least one and no more than a few times `MIN_CHUNK_SIZE` bytes. + + 1. The first request will `refill` this empty allocator by making a request for 2 MiB to its parent. + + 2. The second request will stop here, as the allocator will no longer be empty. + +6. The chain continues with a `CommitRange`, which simply forwards all allocation requests and (upon unwinding) ensures that the address space is mapped. + +7. The chain now transitions from thread-local to global; the `GlobalRange` simply serves to acquire a lock around the rest of the chain. + +8. The next entry in the chain is a `StatsRange` which serves to accumulate statistics. + We ignore this stage and continue onwards. + +9. The next entry in the chain is another *large* buddy allocator which refills at 16 MiB but can hold regions + of any size up to the entire address space. + The first request triggers a `refill`, continuing along the chain as a 16 MiB request. + (Recall that the second allocation will be handled at an earlier point on the chain.) + +10. The penultimate entry in the chain is a `PagemapRegisterRange`, which always forwards allocations along the chain. + +11. At long last, we have arrived at the last entry in the chain, a `PalRange`. + This delegates the actual allocation, of 16 MiB, to either the `reserve_aligned` or `reserve` method of the Platform Abstraction Layer (PAL). + +12. Having wound the chain onto our stack, we now unwind! + The `PagemapRegisterRange` ensures that the Pagemap entries for allocations passing through it are mapped and returns the allocation unaltered. + +13. The global large buddy allocator splits the 16 MiB refill into 8, 4, and 2 MiB regions it retains as well as returning the remaining 2 MiB back along the chain. + +14. The `StatsRange` makes its observations, the `GlobalRange` now unlocks the global component of the chain, and the `CommitRange` ensures that the allocation is mapped. + Aside from these side effects, these propagate the allocation along the chain unaltered. + +15. We now arrive back at the thread-local large buddy allocator, which takes its 2 MiB refill and breaks it down into powers of two down to the requested `MIN_CHUNK_SIZE`. + The second allocation (of the chunk), will either return or again break down one of these intermediate chunks. + +16. For the first (metadata) allocation, the thread-local *small* allocator breaks the `MIN_CHUNK_SIZE` allocation down into powers of two down to `PAGEMAP_METADATA_STRUCT_SIZE` and returns one of that size. + The second allocation will have been forwarded and so is not additionally handled here. + +Exciting, no? + +## What Can I Learn from the Pagemap? + +### Decoding a MetaEntry + +The centerpiece of `snmalloc`'s metadata is its `PageMap`, which associates each "chunk" of the address space (~16KiB; see `MIN_CHUNK_BITS`) with a `MetaEntry`. +A `MetaEntry` is a pair of pointers, suggestively named `meta` and `remote_and_sizeclass`. +In more detail, `MetaEntry`s are better represented by Sigma and Pi types, all packed into two pointer-sized words in ways that preserve pointer provenance on CHERI. + +To begin decoding, a bit (`REMOTE_BACKEND_MARKER`) in `remote_and_sizeclass` distinguishes chunks owned by frontend and backend allocators. + +For chunks owned by the *frontend* (`REMOTE_BACKEND_MARKER` not asserted), + +1. The `remote_and_sizeclass` field is a product of + + 1. A `RemoteAllocator*` indicating the `LocalAlloc` that owns the region of memory. + + 2. A "full sizeclass" value (itself a tagged sum type between large and small sizeclasses). + +2. The `meta` pointer is a bit-stuffed pair of + + 1. A pointer to a larger metadata structure with type dependent on the role of this chunk + + 2. A bit (`META_BOUNDARY_BIT`) that serves to limit chunk coalescing on platforms where that may not be possible, such as CHERI. + +See `src/backend/metatypes.h` and `src/mem/metaslab.h`. + +For chunks owned by a *backend* (`REMOTE_BACKEND_MARKER` asserted), there are again multiple possibilities. + +For chunks owned by a *small buddy allocator*, the remainder of the `MetaEntry` is zero. +That is, it appears to have small sizeclass 0 and an implausible `RemoteAllocator*`. + +For chunks owned by a *large buddy allocator*, the `MetaEntry` is instead a node in a red-black tree of all such chunks. +Its contents can be decoded as follows: + +1. The `meta` field's `META_BOUNDARY_BIT` is preserved, with the same meaning as in the frontend case, above. + +2. `meta` (resp. `remote_and_sizeclass`) includes a pointer to the left (resp. right) *chunk* of address space. + (The corresponding child *node* in this tree is found by taking the *address* of this chunk and looking up the `MetaEntry` in the Pagemap. + This trick of pointing at the child's chunk rather than at the child `MetaEntry` is particularly useful on CHERI: + it allows us to capture the authority to the chunk without needing another pointer and costs just a shift and add.) + +3. The `meta` field's `LargeBuddyRep::RED_BIT` is used to carry the red/black color of this node. + +See `src/backend/largebuddyrange.h`. + +### Encoding a MetaEntry + +We can also consider the process for generating a MetaEntry for a chunk of the address space given its state. +The following cases apply: + +1. The address is not associated with `snmalloc`: + Here, the `MetaEntry`, if it is mapped, is all zeros and so it... + * has `REMOTE_BACKEND_MARKER` clear in `remote_and_sizeclass`. + * appears to be owned by a frontend RemoteAllocator at address 0 (probably, but not certainly, `nullptr`). + * has "small" sizeclass 0, which has size 0. + * has no associated metadata structure. + +2. The address is part of a free chunk in a backend's Large Buddy Allocator: + The `MetaEntry`... + * has `REMOTE_BACKEND_MARKER` asserted in `remote_and_sizeclass`. + * has "small" sizeclass 0, which has size 0. + * the remainder of its `MetaEntry` structure will be a Large Buddy Allocator rbtree node. + * has no associated metadata structure. + +3. The address is part of a free chunk inside a backend's Small Buddy Allocator: + Here, the `MetaEntry` is zero aside from the asserted `REMOTE_BACKEND_MARKER` bit, and so it... + * has "small" sizeclass 0, which has size 0. + * has no associated metadata structure. + +4. The address is part of a live large allocation (spanning one or more 16KiB chunks): + Here, the `MetaEntry`... + * has `REMOTE_BACKEND_MARKER` clear in `remote_and_sizeclass`. + * has a *large* sizeclass value. + * has an associated `RemoteAllocator*` and `Metaslab*` metadata structure + (holding just the original chunk pointer in its `MetaCommon` substructure; + it is configured to always trigger the deallocation slow-path to skip the logic when a chunk is in use as a slab). + +5. The address, whether or not it is presently within an allocated object, is part of an active slab. Here, the `MetaEntry`.... + * encodes the *small* sizeclass of all objects in the slab. + * has a `RemoteAllocator*` referencing the owning `LocalAlloc`'s message queue. + * points to the slab's `Metaslab` structure containing additional metadata (e.g., free list). From 1e1104a11ccdd807f91b43e1a43635a304272501 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 17 Mar 2022 09:47:01 +0000 Subject: [PATCH 229/302] Fix 32bit external pointer. For 32bit external pointer it was performing a divide by size, and for things not managed by snmalloc this was causing a crash. This checks for zero, and gives the start of the address range as the start of the object. --- src/mem/sizeclasstable.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mem/sizeclasstable.h b/src/mem/sizeclasstable.h index 1569bc80e..ae5554585 100644 --- a/src/mem/sizeclasstable.h +++ b/src/mem/sizeclasstable.h @@ -357,6 +357,8 @@ namespace snmalloc } else { + if (size == 0) + return 0; return slab_start + (offset / size) * size; } } From 2ff2cdf8ffddabfc44a1882a45247cd9470e582b Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 17 Mar 2022 09:23:46 +0000 Subject: [PATCH 230/302] Add test for external pointer on stack The external pointer function should work on any memory. This checks it works for the stack. --- src/test/func/memory/memory.cc | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index 6fa16c609..8db19e307 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -344,6 +344,7 @@ void test_external_pointer_large() void test_external_pointer_dealloc_bug() { + std::cout << "Testing external pointer dealloc bug" << std::endl; auto& alloc = ThreadAlloc::get(); constexpr size_t count = MIN_CHUNK_SIZE; void* allocs[count]; @@ -364,6 +365,29 @@ void test_external_pointer_dealloc_bug() } alloc.dealloc(allocs[0]); + std::cout << "Testing external pointer dealloc bug - done" << std::endl; +} + +void test_external_pointer_stack() +{ + std::cout << "Testing external pointer stack" << std::endl; + + std::array stack; + + auto& alloc = ThreadAlloc::get(); + + for (size_t i = 0; i < stack.size(); i++) + { + if (alloc.external_pointer(&stack[i]) > &stack[i]) + { + std::cout << "Stack pointer: " << &stack[i] + << " external pointer: " << alloc.external_pointer(&stack[i]) + << std::endl; + abort(); + } + } + + std::cout << "Testing external pointer stack - done" << std::endl; } void test_alloc_16M() @@ -500,6 +524,7 @@ int main(int argc, char** argv) test_remaining_bytes(); test_static_sized_allocs(); test_calloc_large_bug(); + test_external_pointer_stack(); test_external_pointer_dealloc_bug(); test_external_pointer_large(); test_external_pointer(); From 453a7d57e9440bd11bd0f5b8be91e87ddc0ac5f0 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Wed, 16 Mar 2022 09:43:32 +0000 Subject: [PATCH 231/302] backend: clobber MetaEntry-s in dealloc_chunk Otherwise these won't get updated until the small buddy allocator hands them off to the large buddy allocator (when they morph into being rbtree nodes) and so the frontend might get confused in the interim (including risk of UAF on double-free). --- src/backend/backend.h | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/backend/backend.h b/src/backend/backend.h index 703d0d2c7..cd25e86d1 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -305,11 +305,18 @@ namespace snmalloc { auto chunk = chunk_record->meta_common.chunk; + /* + * The backend takes possession of these chunks now, by disassociating + * any existing remote allocator and metadata structure. If + * interrogated, the sizeclass reported by the MetaEntry is 0, which has + * size 0. + */ + MetaEntry t(nullptr, MetaEntry::REMOTE_BACKEND_MARKER); + Pagemap::set_metaentry(address_cast(chunk), size, t); + local_state.get_meta_range()->dealloc_range( capptr::Chunk(chunk_record), PAGEMAP_METADATA_STRUCT_SIZE); - // TODO, should we set the sizeclass to something specific here? - local_state.object_range->dealloc_range(chunk, size); } From 73be8a3786c4eb8332711caa9c2219113a772a78 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 23 Mar 2022 16:07:06 +0000 Subject: [PATCH 232/302] Fix StatsRange on OE --- src/backend/backend.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/backend/backend.h b/src/backend/backend.h index cd25e86d1..3145a1c10 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -129,8 +129,9 @@ namespace snmalloc #if defined(OPEN_ENCLAVE) // Single global buddy allocator is used on open enclave due to // the limited address space. - using GlobalR = GlobalRange>>>; + using StatsR = StatsRange>>; + using GlobalR = GlobalRange; using ObjectRange = GlobalR; using GlobalMetaRange = ObjectRange; #else From 821620133d07c8ea561c63277b68f6ee5e21adf1 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 23 Mar 2022 16:08:53 +0000 Subject: [PATCH 233/302] Remove ChunkAllocator --- src/backend/backend.h | 9 +- src/backend/chunkallocator.h | 370 ------------------- src/backend/fixedglobalconfig.h | 1 - src/backend/globalconfig.h | 1 - src/ds/spmcstack.h | 72 ---- src/mem/corealloc.h | 21 +- src/mem/pool.h | 11 +- src/test/func/domestication/domestication.cc | 8 - 8 files changed, 18 insertions(+), 475 deletions(-) delete mode 100644 src/backend/chunkallocator.h delete mode 100644 src/ds/spmcstack.h diff --git a/src/backend/backend.h b/src/backend/backend.h index 3145a1c10..4c169d6a7 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -1,7 +1,6 @@ #pragma once #include "../mem/allocconfig.h" #include "../pal/pal.h" -#include "chunkallocator.h" #include "commitrange.h" #include "commonconfig.h" #include "empty_range.h" @@ -301,10 +300,10 @@ namespace snmalloc return {p, meta}; } - static void dealloc_chunk( - LocalState& local_state, ChunkRecord* chunk_record, size_t size) + static void + dealloc_chunk(LocalState& local_state, MetaCommon& meta_common, size_t size) { - auto chunk = chunk_record->meta_common.chunk; + auto chunk = meta_common.chunk; /* * The backend takes possession of these chunks now, by disassociating @@ -316,7 +315,7 @@ namespace snmalloc Pagemap::set_metaentry(address_cast(chunk), size, t); local_state.get_meta_range()->dealloc_range( - capptr::Chunk(chunk_record), PAGEMAP_METADATA_STRUCT_SIZE); + capptr::Chunk(&meta_common), PAGEMAP_METADATA_STRUCT_SIZE); local_state.object_range->dealloc_range(chunk, size); } diff --git a/src/backend/chunkallocator.h b/src/backend/chunkallocator.h deleted file mode 100644 index 80d1be5d1..000000000 --- a/src/backend/chunkallocator.h +++ /dev/null @@ -1,370 +0,0 @@ -#pragma once - -/*** - * WARNING: This file is not currently in use. The functionality has not - * be transistioned to the new backend. It does not seem to be required - * but further testing is required before we delete it. - */ - -#include "../backend/backend_concept.h" -#include "../backend/metatypes.h" -#include "../ds/mpmcstack.h" -#include "../ds/spmcstack.h" -#include "../mem/metaslab.h" -#include "../mem/sizeclasstable.h" -#include "../pal/pal_ds.h" - -#ifdef SNMALLOC_TRACING -# include -#endif - -#include - -namespace snmalloc -{ - /** - * Used to store slabs in the unused sizes. - */ - struct ChunkRecord - { - MetaCommon meta_common; - std::atomic next; - }; -#if defined(USE_METADATA_CONCEPT) - static_assert(ConceptMetadataStruct); -#endif - - /** - * How many slab sizes that can be provided. - */ - constexpr size_t NUM_SLAB_SIZES = Pal::address_bits - MIN_CHUNK_BITS; - - /** - * Number of free stacks per chunk size that each allocator will use. - * For performance ideally a power of 2. We will return to the central - * pool anything that has not be used in the last NUM_EPOCHS - 1, where - * each epoch is separated by DecayMemoryTimerObject::PERIOD. - * I.e. if period is 500ms and num of epochs is 4, then we will return to - * the central pool anything not used for the last 1500-2000ms. - */ - constexpr size_t NUM_EPOCHS = 4; - static_assert(bits::is_pow2(NUM_EPOCHS), "Code assumes power of two."); - - class ChunkAllocatorLocalState - { - friend class ChunkAllocator; - - /** - * Stack of slabs that have been returned for reuse. - */ - ModArray>> - chunk_stack; - - /** - * Used for list of all ChunkAllocatorLocalStates. - */ - std::atomic next{nullptr}; - }; - - /** - * This is the global state required for the chunk allocator. - * It must be provided as a part of the shared state handle - * to the chunk allocator. - */ - class ChunkAllocatorState - { - friend class ChunkAllocator; - /** - * Stack of slabs that have been returned for reuse. - */ - ModArray> - decommitted_chunk_stack; - - /** - * Which is the current epoch to place dealloced chunks, and the - * first place we look for allocating chunks. - */ - alignas(CACHELINE_SIZE) std::atomic epoch{0}; - - /** - * All memory issued by this address space manager - */ - std::atomic peak_memory_usage_{0}; - - std::atomic memory_in_stacks{0}; - - std::atomic all_local{nullptr}; - - // Flag to ensure one-shot registration with the PAL for notifications. - std::atomic_flag register_decay{}; - - public: - size_t unused_memory() - { - return memory_in_stacks; - } - - size_t peak_memory_usage() - { - return peak_memory_usage_; - } - - void add_peak_memory_usage(size_t size) - { - peak_memory_usage_ += size; -#ifdef SNMALLOC_TRACING - std::cout << "peak_memory_usage_: " << peak_memory_usage_ << std::endl; -#endif - } - }; - - class ChunkAllocator - { - template - class DecayMemoryTimerObject : public PalTimerObject - { - ChunkAllocatorState* state; - - /*** - * Method for callback object to perform lazy decommit. - */ - static void process(PalTimerObject* p) - { - // Unsafe downcast here. Don't want vtable and RTTI. - auto self = reinterpret_cast(p); - ChunkAllocator::handle_decay_tick(self->state); - } - - // Specify that we notify the ChunkAllocator every 500ms. - static constexpr size_t PERIOD = 500; - - public: - DecayMemoryTimerObject(ChunkAllocatorState* state) - : PalTimerObject(&process, PERIOD), state(state) - {} - }; - - template - static void handle_decay_tick(ChunkAllocatorState* state) - { - auto new_epoch = (state->epoch + 1) % NUM_EPOCHS; - // Flush old index for all threads. - ChunkAllocatorLocalState* curr = state->all_local; - while (curr != nullptr) - { - for (size_t sc = 0; sc < NUM_SLAB_SIZES; sc++) - { - auto& old_stack = curr->chunk_stack[sc][new_epoch]; - ChunkRecord* record = old_stack.pop_all(); - while (record != nullptr) - { - auto next = record->next.load(); - - // Disable pages for this - Pal::notify_not_using( - record->meta_common.chunk.unsafe_ptr(), - slab_sizeclass_to_size(sc)); - - // Add to global state - state->decommitted_chunk_stack[sc].push(record); - - record = next; - } - } - curr = curr->next; - } - - // Advance current index - state->epoch = new_epoch; - } - - public: - template - static std::pair, Metaslab*> alloc_chunk( - typename SharedStateHandle::LocalState& local_state, - ChunkAllocatorLocalState& chunk_alloc_local_state, - chunksizeclass_t slab_sizeclass, - size_t slab_size, - uintptr_t ras) - { - using PAL = typename SharedStateHandle::Pal; - ChunkAllocatorState& state = - SharedStateHandle::get_chunk_allocator_state(&local_state); - - if (slab_sizeclass >= NUM_SLAB_SIZES) - { - // Your address space is not big enough for this allocation! - errno = ENOMEM; - return {nullptr, nullptr}; - } - - ChunkRecord* chunk_record = nullptr; - if constexpr (pal_supports) - { - // Try local cache of chunks first - for (size_t e = 0; e < NUM_EPOCHS && chunk_record == nullptr; e++) - { - chunk_record = - chunk_alloc_local_state - .chunk_stack[slab_sizeclass][(state.epoch - e) % NUM_EPOCHS] - .pop(); - } - } - - // Try global cache. - if (chunk_record == nullptr) - { - chunk_record = state.decommitted_chunk_stack[slab_sizeclass].pop(); - if (chunk_record != nullptr) - { - PAL::template notify_using( - chunk_record->meta_common.chunk.unsafe_ptr(), slab_size); - } - } - - if (chunk_record != nullptr) - { - auto slab = chunk_record->meta_common.chunk; - state.memory_in_stacks -= slab_size; - auto meta = reinterpret_cast(chunk_record); -#ifdef SNMALLOC_TRACING - std::cout << "Reuse slab:" << slab.unsafe_ptr() << " slab_sizeclass " - << slab_sizeclass << " size " << slab_size - << " memory in stacks " << state.memory_in_stacks - << std::endl; -#endif - MetaEntry entry{&meta->meta_common, ras}; - SharedStateHandle::Pagemap::set_metaentry( - address_cast(slab), slab_size, entry); - return {slab, meta}; - } - - // Allocate a fresh slab as there are no available ones. - // First create meta-data - auto [slab, meta] = - SharedStateHandle::alloc_chunk(&local_state, slab_size, ras); -#ifdef SNMALLOC_TRACING - std::cout << "Create slab:" << slab.unsafe_ptr() << " slab_sizeclass " - << slab_sizeclass << " size " << slab_size << std::endl; -#endif - - state.add_peak_memory_usage(slab_size); - state.add_peak_memory_usage(PAGEMAP_METADATA_STRUCT_SIZE); - // TODO handle bounded versus lazy pagemaps in stats - state.add_peak_memory_usage( - (slab_size / MIN_CHUNK_SIZE) * sizeof(MetaEntry)); - - return {slab, meta}; - } - - template - SNMALLOC_SLOW_PATH static void dealloc( - typename SharedStateHandle::LocalState& local_state, - ChunkAllocatorLocalState& chunk_alloc_local_state, - ChunkRecord* p, - size_t slab_sizeclass) - { - ChunkAllocatorState& state = - SharedStateHandle::get_chunk_allocator_state(&local_state); - - if constexpr (pal_supports) - { - // If we have a time source use decay based local cache. -#ifdef SNMALLOC_TRACING - std::cout << "Return slab:" << p->meta_common.chunk.unsafe_ptr() - << " slab_sizeclass " << slab_sizeclass << " size " - << slab_sizeclass_to_size(slab_sizeclass) - << " memory in stacks " << state.memory_in_stacks - << std::endl; -#endif - chunk_alloc_local_state.chunk_stack[slab_sizeclass][state.epoch].push( - p); - } - else - { - // No time source share immediately with global state. - // Disable pages for this chunk. - SharedStateHandle::Pal::notify_not_using( - p->meta_common.chunk.unsafe_ptr(), - slab_sizeclass_to_size(slab_sizeclass)); - - // Add to global state - state.decommitted_chunk_stack[slab_sizeclass].push(p); - } - - state.memory_in_stacks += slab_sizeclass_to_size(slab_sizeclass); - } - - /** - * Provide a block of meta-data with size and align. - * - * Backend allocator may use guard pages and separate area of - * address space to protect this from corruption. - */ - template< - typename U, - SNMALLOC_CONCEPT(ConceptBackendGlobals) SharedStateHandle, - typename... Args> - static U* alloc_meta_data( - typename SharedStateHandle::LocalState* local_state, Args&&... args) - { - // Cache line align - size_t size = bits::align_up(sizeof(U), 64); - - capptr::Chunk p = - SharedStateHandle::template alloc_meta_data(local_state, size); - - if (p == nullptr) - { - errno = ENOMEM; - return nullptr; - } - - return new (p.unsafe_ptr()) U(std::forward(args)...); - } - - template - static void register_local_state( - typename SharedStateHandle::LocalState& local_state, - ChunkAllocatorLocalState& chunk_alloc_local_state) - { - if constexpr (pal_supports) - { - ChunkAllocatorState& state = - SharedStateHandle::get_chunk_allocator_state(&local_state); - - // Register with the Pal to receive notifications. - if (!state.register_decay.test_and_set()) - { - auto timer = alloc_meta_data< - DecayMemoryTimerObject, - SharedStateHandle>(&local_state, &state); - if (timer != nullptr) - { - SharedStateHandle::Pal::register_timer(timer); - } - else - { - // We failed to register the notification. - // This is not catarophic, but if we can't allocate this - // state something else will fail shortly. - state.register_decay.clear(); - } - } - - // Add to the list of local states. - auto* head = state.all_local.load(); - do - { - chunk_alloc_local_state.next = head; - } while (!state.all_local.compare_exchange_strong( - head, &chunk_alloc_local_state)); - } - else - { - UNUSED(local_state); - UNUSED(chunk_alloc_local_state); - } - } - }; -} // namespace snmalloc diff --git a/src/backend/fixedglobalconfig.h b/src/backend/fixedglobalconfig.h index 9bb33ae5f..5b9cbf5d2 100644 --- a/src/backend/fixedglobalconfig.h +++ b/src/backend/fixedglobalconfig.h @@ -1,7 +1,6 @@ #pragma once #include "../backend/backend.h" -#include "../backend/chunkallocator.h" #include "../mem/corealloc.h" #include "../mem/pool.h" #include "commonconfig.h" diff --git a/src/backend/globalconfig.h b/src/backend/globalconfig.h index 3b4685694..a496aa3eb 100644 --- a/src/backend/globalconfig.h +++ b/src/backend/globalconfig.h @@ -1,7 +1,6 @@ #pragma once #include "../backend/backend.h" -#include "../backend/chunkallocator.h" #include "../mem/corealloc.h" #include "../mem/pool.h" #include "commonconfig.h" diff --git a/src/ds/spmcstack.h b/src/ds/spmcstack.h deleted file mode 100644 index 7c6ea70e3..000000000 --- a/src/ds/spmcstack.h +++ /dev/null @@ -1,72 +0,0 @@ -#pragma once - -#include "aba.h" -#include "ptrwrap.h" - -namespace snmalloc -{ - /** - * Concurrent Stack - * - * This stack supports the following clients - * (push|pop)* || pop_all* || ... || pop_all* - * - * That is a single thread that can do push and pop, and other threads - * that do pop_all. pop_all if it returns a value, returns all of the - * stack, however, it may return nullptr if it races with either a push - * or a pop. - * - * The primary use case is single-threaded access, where other threads - * can attempt to steal all the values. - */ - template - class SPMCStack - { - private: - alignas(CACHELINE_SIZE) std::atomic stack{}; - - public: - constexpr SPMCStack() = default; - - void push(T* item) - { - static_assert( - std::is_same>::value, - "T->next must be an std::atomic"); - - return push(item, item); - } - - void push(T* first, T* last) - { - T* old_head = stack.exchange(nullptr, std::memory_order_relaxed); - last->next.store(old_head, std::memory_order_relaxed); - // Assume stays null as not allowed to race with pop or other pushes. - SNMALLOC_ASSERT(stack.load() == nullptr); - stack.store(first, std::memory_order_release); - } - - T* pop() - { - if (stack.load(std::memory_order_relaxed) == nullptr) - return nullptr; - T* old_head = stack.exchange(nullptr); - if (SNMALLOC_UNLIKELY(old_head == nullptr)) - return nullptr; - - auto next = old_head->next.load(std::memory_order_relaxed); - - // Assume stays null as not allowed to race with pop or other pushes. - SNMALLOC_ASSERT(stack.load() == nullptr); - - stack.store(next, std::memory_order_release); - - return old_head; - } - - T* pop_all() - { - return stack.exchange(nullptr); - } - }; -} // namespace snmalloc diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index b22fbb0ac..bd42b4e3d 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -1,6 +1,5 @@ #pragma once -#include "../backend/chunkallocator.h" #include "../ds/defines.h" #include "allocconfig.h" #include "localcache.h" @@ -282,7 +281,7 @@ namespace snmalloc bumpptr = slab_end; } - ChunkRecord* clear_slab(Metaslab* meta, smallsizeclass_t sizeclass) + void clear_slab(Metaslab* meta, smallsizeclass_t sizeclass) { auto& key = entropy.get_free_list_key(); freelist::Iter<> fl; @@ -324,15 +323,13 @@ namespace snmalloc SNMALLOC_ASSERT( count == snmalloc::sizeclass_to_slab_object_count(sizeclass)); #endif - ChunkRecord* chunk_record = reinterpret_cast(meta); // TODO: This is a capability amplification as we are saying we // have the whole chunk. auto start_of_slab = pointer_align_down( p, snmalloc::sizeclass_to_slab_size(sizeclass)); SNMALLOC_ASSERT( - address_cast(start_of_slab) == - chunk_record->meta_common.chunk_address()); + address_cast(start_of_slab) == meta->meta_common.chunk_address()); #if defined(__CHERI_PURE_CAPABILITY__) && !defined(SNMALLOC_CHECK_CLIENT) // Zero the whole slab. For CHERI we at least need to clear the freelist @@ -340,9 +337,8 @@ namespace snmalloc // the freelist order as for SNMALLOC_CHECK_CLIENT. Zeroing the whole slab // may be more friendly to hw because it does not involve pointer chasing // and is amenable to prefetching. - chunk_record->meta_common - .template zero_chunk( - snmalloc::sizeclass_to_slab_size(sizeclass)); + meta->meta_common.template zero_chunk( + snmalloc::sizeclass_to_slab_size(sizeclass)); #endif #ifdef SNMALLOC_TRACING @@ -351,7 +347,6 @@ namespace snmalloc #else UNUSED(start_of_slab); #endif - return chunk_record; } template @@ -386,11 +381,11 @@ namespace snmalloc // TODO delay the clear to the next user of the slab, or teardown so // don't touch the cache lines at this point in snmalloc_check_client. - auto chunk_record = clear_slab(meta, sizeclass); + clear_slab(meta, sizeclass); SharedStateHandle::dealloc_chunk( get_backend_local_state(), - chunk_record, + meta->meta_common, sizeclass_to_slab_size(sizeclass)); return true; @@ -422,10 +417,8 @@ namespace snmalloc UNUSED(size); #endif - auto slab_record = reinterpret_cast(meta); - SharedStateHandle::dealloc_chunk( - get_backend_local_state(), slab_record, size); + get_backend_local_state(), meta->meta_common, size); return; } diff --git a/src/mem/pool.h b/src/mem/pool.h index c7924c63a..141a14c51 100644 --- a/src/mem/pool.h +++ b/src/mem/pool.h @@ -1,11 +1,12 @@ #pragma once -#include "../backend/chunkallocator.h" #include "../ds/flaglock.h" #include "../ds/mpmcstack.h" #include "../pal/pal_concept.h" #include "pooled.h" +#include + namespace snmalloc { /** @@ -132,15 +133,17 @@ namespace snmalloc return p; } - p = ChunkAllocator::alloc_meta_data( - nullptr, std::forward(args)...); + auto raw = + SharedStateHandle::template alloc_meta_data(nullptr, sizeof(T)); - if (p == nullptr) + if (raw == nullptr) { SharedStateHandle::Pal::error( "Failed to initialise thread local allocator."); } + p = new (raw.unsafe_ptr()) T(std::forward(args)...); + FlagLock f(pool.lock); p->list_next = pool.list; pool.list = p; diff --git a/src/test/func/domestication/domestication.cc b/src/test/func/domestication/domestication.cc index 7a5d2c077..5108960f2 100644 --- a/src/test/func/domestication/domestication.cc +++ b/src/test/func/domestication/domestication.cc @@ -24,8 +24,6 @@ namespace snmalloc private: using Backend = BackendAllocator; - SNMALLOC_REQUIRE_CONSTINIT - inline static ChunkAllocatorState chunk_allocator_state; SNMALLOC_REQUIRE_CONSTINIT inline static GlobalPoolState alloc_pool; @@ -46,12 +44,6 @@ namespace snmalloc } (); - static ChunkAllocatorState& - get_chunk_allocator_state(Backend::LocalState* = nullptr) - { - return chunk_allocator_state; - } - static GlobalPoolState& pool() { return alloc_pool; From bb82ac15e3000fdfcc474b906ff3d8b12ad9edae Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 23 Mar 2022 16:11:27 +0000 Subject: [PATCH 234/302] Add a thread id to messages Automatically prepend messages with a thread id. Makes debugging easier. --- src/pal/pal.h | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/pal/pal.h b/src/pal/pal.h index b89391479..8a0260fe3 100644 --- a/src/pal/pal.h +++ b/src/pal/pal.h @@ -168,10 +168,23 @@ namespace snmalloc Pal::error(msg.get_message()); } + static inline size_t get_tid() + { + static thread_local size_t tid{0}; + static std::atomic tid_source{1}; + + if (tid == 0) + { + tid = tid_source++; + } + return tid; + } + template inline void message(Args... args) { MessageBuilder msg{std::forward(args)...}; - Pal::message(msg.get_message()); + MessageBuilder msg_tid{"{}: {}", get_tid(), msg.get_message()}; + Pal::message(msg_tid.get_message()); } } // namespace snmalloc From bdb3183989d8544b15bf600a5018d7056d795da0 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 23 Mar 2022 16:13:44 +0000 Subject: [PATCH 235/302] Remove std::cout Now we have an allocation free formatting routine, remove std::cout from tracing. --- src/backend/backend.h | 5 ++--- src/backend/globalconfig.h | 2 +- src/ds/defines.h | 7 +++++++ src/mem/corealloc.h | 25 +++++++++++++------------ src/mem/globalalloc.h | 11 +++++------ src/mem/localalloc.h | 25 ++++++++++++------------- src/mem/threadalloc.h | 4 ++-- src/pal/pal_posix.h | 3 +-- 8 files changed, 43 insertions(+), 39 deletions(-) diff --git a/src/backend/backend.h b/src/backend/backend.h index 4c169d6a7..03f04f7b8 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -277,8 +277,7 @@ namespace snmalloc auto p = local_state.object_range->alloc_range(size); #ifdef SNMALLOC_TRACING - std::cout << "Alloc chunk: " << p.unsafe_ptr() << " (" << size << ")" - << std::endl; + message<1024>("Alloc chunk: {} ({})", p.unsafe_ptr(), size); #endif if (p == nullptr) { @@ -286,7 +285,7 @@ namespace snmalloc meta_cap, PAGEMAP_METADATA_STRUCT_SIZE); errno = ENOMEM; #ifdef SNMALLOC_TRACING - std::cout << "Out of memory" << std::endl; + message<1024>("Out of memory"); #endif return {p, nullptr}; } diff --git a/src/backend/globalconfig.h b/src/backend/globalconfig.h index a496aa3eb..31176621c 100644 --- a/src/backend/globalconfig.h +++ b/src/backend/globalconfig.h @@ -60,7 +60,7 @@ namespace snmalloc { FlagLock lock{initialisation_lock}; #ifdef SNMALLOC_TRACING - std::cout << "Run init_impl" << std::endl; + message<1024>("Run init_impl"); #endif if (initialised) diff --git a/src/ds/defines.h b/src/ds/defines.h index 0503a858f..869310d6e 100644 --- a/src/ds/defines.h +++ b/src/ds/defines.h @@ -205,6 +205,13 @@ namespace snmalloc */ template [[noreturn]] inline void report_fatal_error(Args... args); + + /** + * Forward declaration so that this can be called before the pal header is + * included. + */ + template + inline void message(Args... args); } // namespace snmalloc #ifdef SNMALLOC_CHECK_CLIENT diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index bd42b4e3d..ad6e81a26 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -342,8 +342,10 @@ namespace snmalloc #endif #ifdef SNMALLOC_TRACING - std::cout << "Slab " << start_of_slab.unsafe_ptr() - << " is unused, Object sizeclass " << sizeclass << std::endl; + message<1024>( + "Slab {} is unused, Object sizeclass {}", + start_of_slab.unsafe_ptr(), + sizeclass); #else UNUSED(start_of_slab); #endif @@ -412,7 +414,7 @@ namespace snmalloc size_t size = bits::one_at_bit(entry_sizeclass); #ifdef SNMALLOC_TRACING - std::cout << "Large deallocation: " << size << std::endl; + message<1024>("Large deallocation: {}", size); #else UNUSED(size); #endif @@ -437,7 +439,7 @@ namespace snmalloc alloc_classes[sizeclass].length++; #ifdef SNMALLOC_TRACING - std::cout << "Slab is woken up" << std::endl; + message<1024>("Slab is woken up"); #endif ticker.check_tick(); @@ -483,7 +485,7 @@ namespace snmalloc auto cb = [this, &need_post](freelist::HeadPtr msg) SNMALLOC_FAST_PATH_LAMBDA { #ifdef SNMALLOC_TRACING - std::cout << "Handling remote" << std::endl; + message<1024>("Handling remote"); #endif auto& entry = @@ -562,7 +564,7 @@ namespace snmalloc void init() { #ifdef SNMALLOC_TRACING - std::cout << "Making an allocator." << std::endl; + message<1024>("Making an allocator."); #endif // Entropy must be first, so that all data-structures can use the key // it generates. @@ -776,8 +778,7 @@ namespace snmalloc size_t slab_size = sizeclass_to_slab_size(sizeclass); #ifdef SNMALLOC_TRACING - std::cout << "rsize " << rsize << std::endl; - std::cout << "slab size " << slab_size << std::endl; + message<1024>("small_alloc_slow rsize={} slab size={}", rsize, slab_size); #endif auto [slab, meta] = SharedStateHandle::alloc_chunk( @@ -872,7 +873,7 @@ namespace snmalloc void attach(LocalCache* c) { #ifdef SNMALLOC_TRACING - std::cout << "Attach cache to " << this << std::endl; + message<1024>("Attach cache to {}", this); #endif attached_cache = c; @@ -916,7 +917,7 @@ namespace snmalloc init_message_queue(); #ifdef SNMALLOC_TRACING - std::cout << "debug_is_empty - done" << std::endl; + message<1024>("debug_is_empty - done"); #endif return sent_something; } @@ -934,7 +935,7 @@ namespace snmalloc bool debug_is_empty(bool* result) { #ifdef SNMALLOC_TRACING - std::cout << "debug_is_empty" << std::endl; + message<1024>("debug_is_empty"); #endif if (attached_cache == nullptr) { @@ -943,7 +944,7 @@ namespace snmalloc LocalCache temp(public_state()); attach(&temp); #ifdef SNMALLOC_TRACING - std::cout << "debug_is_empty - attach a cache" << std::endl; + message<1024>("debug_is_empty - attach a cache"); #endif auto sent_something = debug_is_empty_impl(result); diff --git a/src/mem/globalalloc.h b/src/mem/globalalloc.h index f32413eaa..0152f6cb2 100644 --- a/src/mem/globalalloc.h +++ b/src/mem/globalalloc.h @@ -94,7 +94,7 @@ namespace snmalloc auto* alloc = AllocPool::iterate(); # ifdef SNMALLOC_TRACING - std::cout << "debug check empty: first " << alloc << std::endl; + message<1024>("debug check empty: first {}", alloc); # endif bool done = false; bool okay = true; @@ -102,7 +102,7 @@ namespace snmalloc while (!done) { # ifdef SNMALLOC_TRACING - std::cout << "debug_check_empty: Check all allocators!" << std::endl; + message<1024>("debug_check_empty: Check all allocators!"); # endif done = true; alloc = AllocPool::iterate(); @@ -111,7 +111,7 @@ namespace snmalloc while (alloc != nullptr) { # ifdef SNMALLOC_TRACING - std::cout << "debug check empty: " << alloc << std::endl; + message<1024>("debug check empty: {}", alloc); # endif // Check that the allocator has freed all memory. // repeat the loop if empty caused message sends. @@ -119,13 +119,12 @@ namespace snmalloc { done = false; # ifdef SNMALLOC_TRACING - std::cout << "debug check empty: sent messages " << alloc - << std::endl; + message<1024>("debug check empty: sent messages {}", alloc); # endif } # ifdef SNMALLOC_TRACING - std::cout << "debug check empty: okay = " << okay << std::endl; + message<1024>("debug check empty: okay = {}", okay); # endif alloc = AllocPool::iterate(alloc); } diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index db32d6165..50a1cd27d 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -151,7 +151,7 @@ namespace snmalloc if (post_teardown) { #ifdef SNMALLOC_TRACING - std::cout << "post_teardown flush()" << std::endl; + message<1024>("post_teardown flush()"); #endif // We didn't have an allocator because the thread is being torndown. // We need to return any local state, so we don't leak it. @@ -188,8 +188,7 @@ namespace snmalloc // set up meta data so sizeclass is correct, and hence alloc size, and // external pointer. #ifdef SNMALLOC_TRACING - std::cout << "size " << size << " pow2 size " - << bits::next_pow2_bits(size) << std::endl; + message<1024>("size {} pow2size {}", size, bits::next_pow2_bits(size)); #endif // Initialise meta data for a successful large allocation. @@ -263,8 +262,10 @@ namespace snmalloc if (core_alloc != nullptr) { #ifdef SNMALLOC_TRACING - std::cout << "Remote dealloc post" << p.unsafe_ptr() << " size " - << alloc_size(p.unsafe_ptr()) << std::endl; + message<1024>( + "Remote dealloc post {} ({})", + p.unsafe_ptr(), + alloc_size(p.unsafe_ptr())); #endif const MetaslabMetaEntry& entry = SharedStateHandle::Pagemap::template get_metaentry( @@ -364,8 +365,7 @@ namespace snmalloc c->attach(&local_cache); core_alloc = c; #ifdef SNMALLOC_TRACING - std::cout << "init(): core_alloc=" << core_alloc << "@" << &local_cache - << std::endl; + message<1024>("init(): core_alloc={} @ {}", core_alloc, &local_cache); #endif // local_cache.stats.sta rt(); } @@ -407,7 +407,7 @@ namespace snmalloc // it is new to hit slow paths. core_alloc = nullptr; #ifdef SNMALLOC_TRACING - std::cout << "flush(): core_alloc=" << core_alloc << std::endl; + message<1024>("flush(): core_alloc={}", core_alloc); #endif local_cache.remote_allocator = &SharedStateHandle::unused_remote; local_cache.remote_dealloc_cache.capacity = 0; @@ -651,8 +651,8 @@ namespace snmalloc local_cache.remote_dealloc_cache.template dealloc( entry.get_remote()->trunc_id(), p_tame, key_global); # ifdef SNMALLOC_TRACING - std::cout << "Remote dealloc fast" << p_raw << " size " - << alloc_size(p_raw) << std::endl; + message<1024>( + "Remote dealloc fast {} ({})", p_raw, alloc_size(p_raw)); # endif return; } @@ -667,7 +667,7 @@ namespace snmalloc snmalloc_check_client(p_tame == nullptr, "Not allocated by snmalloc."); # ifdef SNMALLOC_TRACING - std::cout << "nullptr deallocation" << std::endl; + message<1024>("nullptr deallocation"); # endif return; #endif @@ -689,8 +689,7 @@ namespace snmalloc void teardown() { #ifdef SNMALLOC_TRACING - std::cout << "Teardown: core_alloc=" << core_alloc << "@" << &local_cache - << std::endl; + message<1024>("Teardown: core_alloc={} @ {}", core_alloc, &local_cache); #endif post_teardown = true; if (core_alloc != nullptr) diff --git a/src/mem/threadalloc.h b/src/mem/threadalloc.h index 1ab589285..bbda32d7f 100644 --- a/src/mem/threadalloc.h +++ b/src/mem/threadalloc.h @@ -139,7 +139,7 @@ namespace snmalloc static char p_teardown_val = 1; pthread_setspecific(p_key.get(), &p_teardown_val); # ifdef SNMALLOC_TRACING - std::cout << "Using pthread clean up" << std::endl; + message<1024>("Using pthread clean up"); # endif } # elif defined(SNMALLOC_USE_CXX_THREAD_DESTRUCTORS) @@ -157,7 +157,7 @@ namespace snmalloc []() { ThreadAlloc::get().teardown(); }); UNUSED(dummy); # ifdef SNMALLOC_TRACING - std::cout << "Using C++ destructor clean up" << std::endl; + message<1024>("Using C++ destructor clean up"); # endif } # endif diff --git a/src/pal/pal_posix.h b/src/pal/pal_posix.h index 242125f68..16c5ef7d2 100644 --- a/src/pal/pal_posix.h +++ b/src/pal/pal_posix.h @@ -336,8 +336,7 @@ namespace snmalloc if (p != MAP_FAILED) { #ifdef SNMALLOC_TRACING - std::cout << "Pal_posix reserved: " << p << " (" << size << ")" - << std::endl; + snmalloc::message<1024>("Pal_posix reserved: {} ({})", p, size); #endif return p; } From e77b9e28510d41fee5e7d2c80701eb376dd7cc72 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 23 Mar 2022 16:15:57 +0000 Subject: [PATCH 236/302] Add ConcurrencySafe property Ranges can be safe to call from multiple threads. This adds a constexpr field to signify if that is the case. --- src/backend/backend.h | 3 +++ src/backend/commitrange.h | 2 ++ src/backend/empty_range.h | 2 ++ src/backend/globalrange.h | 2 ++ src/backend/largebuddyrange.h | 2 ++ src/backend/pagemapregisterrange.h | 2 ++ src/backend/palrange.h | 5 +++++ src/backend/smallbuddyrange.h | 2 ++ src/backend/statsrange.h | 2 ++ src/backend/subrange.h | 2 ++ 10 files changed, 24 insertions(+) diff --git a/src/backend/backend.h b/src/backend/backend.h index 03f04f7b8..5bf8015d4 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -235,6 +235,9 @@ namespace snmalloc } else { + static_assert( + GlobalMetaRange::ConcurrencySafe, + "Global meta data range needs to be concurrency safe."); typename GlobalMetaRange::State global_state; p = global_state->alloc_range(bits::next_pow2(size)); } diff --git a/src/backend/commitrange.h b/src/backend/commitrange.h index a0032d6a7..43b1f8842 100644 --- a/src/backend/commitrange.h +++ b/src/backend/commitrange.h @@ -25,6 +25,8 @@ namespace snmalloc static constexpr bool Aligned = ParentRange::Aligned; + static constexpr bool ConcurrencySafe = ParentRange::ConcurrencySafe; + constexpr CommitRange() = default; capptr::Chunk alloc_range(size_t size) diff --git a/src/backend/empty_range.h b/src/backend/empty_range.h index 5feabd90e..36680f3a3 100644 --- a/src/backend/empty_range.h +++ b/src/backend/empty_range.h @@ -19,6 +19,8 @@ namespace snmalloc static constexpr bool Aligned = true; + static constexpr bool ConcurrencySafe = true; + constexpr EmptyRange() = default; capptr::Chunk alloc_range(size_t) diff --git a/src/backend/globalrange.h b/src/backend/globalrange.h index 7b4bb0bbe..d8ae667c2 100644 --- a/src/backend/globalrange.h +++ b/src/backend/globalrange.h @@ -37,6 +37,8 @@ namespace snmalloc static constexpr bool Aligned = ParentRange::Aligned; + static constexpr bool ConcurrencySafe = true; + constexpr GlobalRange() = default; capptr::Chunk alloc_range(size_t size) diff --git a/src/backend/largebuddyrange.h b/src/backend/largebuddyrange.h index 5b2954de2..a4a807aa2 100644 --- a/src/backend/largebuddyrange.h +++ b/src/backend/largebuddyrange.h @@ -271,6 +271,8 @@ namespace snmalloc static constexpr bool Aligned = true; + static constexpr bool ConcurrencySafe = false; + constexpr LargeBuddyRange() = default; capptr::Chunk alloc_range(size_t size) diff --git a/src/backend/pagemapregisterrange.h b/src/backend/pagemapregisterrange.h index de55b1373..4084eed86 100644 --- a/src/backend/pagemapregisterrange.h +++ b/src/backend/pagemapregisterrange.h @@ -30,6 +30,8 @@ namespace snmalloc static constexpr bool Aligned = ParentRange::Aligned; + static constexpr bool ConcurrencySafe = ParentRange::ConcurrencySafe; + capptr::Chunk alloc_range(size_t size) { auto base = state->alloc_range(size); diff --git a/src/backend/palrange.h b/src/backend/palrange.h index 82d36ce3a..5fa3fe6c8 100644 --- a/src/backend/palrange.h +++ b/src/backend/palrange.h @@ -24,6 +24,11 @@ namespace snmalloc static constexpr bool Aligned = pal_supports; + // Note we have always assumed the Pals to provide a concurrency safe + // API. If in the future this changes, then this would + // need to be changed. + static constexpr bool ConcurrencySafe = true; + constexpr PalRange() = default; capptr::Chunk alloc_range(size_t size) diff --git a/src/backend/smallbuddyrange.h b/src/backend/smallbuddyrange.h index b813b674d..4ee075867 100644 --- a/src/backend/smallbuddyrange.h +++ b/src/backend/smallbuddyrange.h @@ -175,6 +175,8 @@ namespace snmalloc static constexpr bool Aligned = true; static_assert(ParentRange::Aligned, "ParentRange must be aligned"); + static constexpr bool ConcurrencySafe = false; + constexpr SmallBuddyRange() = default; capptr::Chunk alloc_range(size_t size) diff --git a/src/backend/statsrange.h b/src/backend/statsrange.h index 0a61fefcc..069398375 100644 --- a/src/backend/statsrange.h +++ b/src/backend/statsrange.h @@ -31,6 +31,8 @@ namespace snmalloc static constexpr bool Aligned = ParentRange::Aligned; + static constexpr bool ConcurrencySafe = ParentRange::ConcurrencySafe; + constexpr StatsRange() = default; capptr::Chunk alloc_range(size_t size) diff --git a/src/backend/subrange.h b/src/backend/subrange.h index c83bdab6b..dd4824339 100644 --- a/src/backend/subrange.h +++ b/src/backend/subrange.h @@ -31,6 +31,8 @@ namespace snmalloc static constexpr bool Aligned = ParentRange::Aligned; + static constexpr bool ConcurrencySafe = ParentRange::ConcurrencySafe; + capptr::Chunk alloc_range(size_t sub_size) { SNMALLOC_ASSERT(bits::is_pow2(sub_size)); From 6424edaeaad2c3e327b367aaafdadf3fe8800400 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Thu, 24 Mar 2022 13:47:41 +0000 Subject: [PATCH 237/302] Stop playing OO games with MetaEntry David points out that the downcasts I had introduced were UB. Instead, go back to passing MetaEntry-s around and make MetaslabMetaEntry just a namespace of static methods. This partially reverts 7940fee00cf0ce43c3c40584a3b3e4fb8d77cfd8 --- src/backend/backend.h | 10 +++----- src/backend/backend_concept.h | 4 ++-- src/mem/corealloc.h | 44 ++++++++++++++++++----------------- src/mem/localalloc.h | 44 ++++++++++++++++++----------------- src/mem/metaslab.h | 37 ++++++++++++++--------------- src/mem/remotecache.h | 20 +++++++--------- 6 files changed, 78 insertions(+), 81 deletions(-) diff --git a/src/backend/backend.h b/src/backend/backend.h index 5bf8015d4..d1022a34b 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -58,14 +58,10 @@ namespace snmalloc * Set template parameter to true if it not an error * to access a location that is not backed by a chunk. */ - template - SNMALLOC_FAST_PATH static const Ret& get_metaentry(address_t p) + template + SNMALLOC_FAST_PATH static const MetaEntry& get_metaentry(address_t p) { - static_assert( - std::is_base_of_v && sizeof(MetaEntry) == sizeof(Ret), - "Backend Pagemap get_metaentry return must look like MetaEntry"); - return static_cast( - concretePagemap.template get(p)); + return concretePagemap.template get(p); } /** diff --git a/src/backend/backend_concept.h b/src/backend/backend_concept.h index c0d8f8ef0..895e19af2 100644 --- a/src/backend/backend_concept.h +++ b/src/backend/backend_concept.h @@ -24,10 +24,10 @@ namespace snmalloc { { Meta::set_metaentry(addr, sz, t) } -> ConceptSame; - { Meta::template get_metaentry(addr) } + { Meta::template get_metaentry(addr) } -> ConceptSame; - { Meta::template get_metaentry(addr) } + { Meta::template get_metaentry(addr) } -> ConceptSame; }; diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index ad6e81a26..40e08a386 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -400,17 +400,17 @@ namespace snmalloc * by this thread, or handling the final deallocation onto a slab, * so it can be reused by other threads. */ - SNMALLOC_SLOW_PATH void - dealloc_local_object_slow(const MetaslabMetaEntry& entry) + SNMALLOC_SLOW_PATH void dealloc_local_object_slow(const MetaEntry& entry) { // TODO: Handle message queue on this path? - Metaslab* meta = entry.get_metaslab(); + Metaslab* meta = FrontendMetaEntry::get_metaslab(entry); if (meta->is_large()) { // Handle large deallocation here. - size_t entry_sizeclass = entry.get_sizeclass().as_large(); + size_t entry_sizeclass = + FrontendMetaEntry::get_sizeclass(entry).as_large(); size_t size = bits::one_at_bit(entry_sizeclass); #ifdef SNMALLOC_TRACING @@ -425,7 +425,8 @@ namespace snmalloc return; } - smallsizeclass_t sizeclass = entry.get_sizeclass().as_small(); + smallsizeclass_t sizeclass = + FrontendMetaEntry::get_sizeclass(entry).as_small(); UNUSED(entropy); if (meta->is_sleeping()) @@ -488,9 +489,8 @@ namespace snmalloc message<1024>("Handling remote"); #endif - auto& entry = - SharedStateHandle::Pagemap::template get_metaentry( - snmalloc::address_cast(msg)); + auto& entry = SharedStateHandle::Pagemap::template get_metaentry( + snmalloc::address_cast(msg)); handle_dealloc_remote(entry, msg.as_void(), need_post); @@ -529,7 +529,7 @@ namespace snmalloc * need_post will be set to true, if capacity is exceeded. */ void handle_dealloc_remote( - const MetaslabMetaEntry& entry, + const MetaEntry& entry, CapPtr p, bool& need_post) { @@ -537,7 +537,8 @@ namespace snmalloc // TODO this needs to not double revoke if using MTE // TODO thread capabilities? - if (SNMALLOC_LIKELY(entry.get_remote() == public_state())) + if (SNMALLOC_LIKELY( + FrontendMetaEntry::get_remote(entry) == public_state())) { if (SNMALLOC_LIKELY( dealloc_local_object_fast(entry, p.as_void(), entropy))) @@ -553,7 +554,9 @@ namespace snmalloc need_post = true; attached_cache->remote_dealloc_cache .template dealloc( - entry.get_remote()->trunc_id(), p.as_void(), key_global); + FrontendMetaEntry::get_remote(entry)->trunc_id(), + p.as_void(), + key_global); } } @@ -670,9 +673,8 @@ namespace snmalloc dealloc_local_object(CapPtr p) { // MetaEntry-s seen here are expected to have meaningful Remote pointers - auto& entry = - SharedStateHandle::Pagemap::template get_metaentry( - snmalloc::address_cast(p)); + auto& entry = SharedStateHandle::Pagemap::template get_metaentry( + snmalloc::address_cast(p)); if (SNMALLOC_LIKELY(dealloc_local_object_fast(entry, p, entropy))) return; @@ -680,16 +682,17 @@ namespace snmalloc } SNMALLOC_FAST_PATH static bool dealloc_local_object_fast( - const MetaslabMetaEntry& entry, + const MetaEntry& entry, CapPtr p, LocalEntropy& entropy) { - auto meta = entry.get_metaslab(); + auto meta = FrontendMetaEntry::get_metaslab(entry); SNMALLOC_ASSERT(!meta->is_unused()); snmalloc_check_client( - is_start_of_object(entry.get_sizeclass(), address_cast(p)), + is_start_of_object( + FrontendMetaEntry::get_sizeclass(entry), address_cast(p)), "Not deallocating start of an object"); auto cp = p.as_static>(); @@ -784,7 +787,7 @@ namespace snmalloc auto [slab, meta] = SharedStateHandle::alloc_chunk( get_backend_local_state(), slab_size, - MetaslabMetaEntry::encode( + FrontendMetaEntry::encode( public_state(), sizeclass_t::from_small_class(sizeclass))); if (slab == nullptr) @@ -838,9 +841,8 @@ namespace snmalloc { bool need_post = true; // Always going to post, so ignore. auto n_tame = p_tame->atomic_read_next(key_global, domesticate); - const MetaslabMetaEntry& entry = - SharedStateHandle::Pagemap::template get_metaentry< - MetaslabMetaEntry>(snmalloc::address_cast(p_tame)); + const MetaEntry& entry = SharedStateHandle::Pagemap::get_metaentry( + snmalloc::address_cast(p_tame)); handle_dealloc_remote(entry, p_tame.as_void(), need_post); p_tame = n_tame; } diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 50a1cd27d..27604882e 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -183,7 +183,7 @@ namespace snmalloc auto [chunk, meta] = SharedStateHandle::alloc_chunk( core_alloc->get_backend_local_state(), large_size_to_chunk_size(size), - MetaslabMetaEntry::encode( + FrontendMetaEntry::encode( core_alloc->public_state(), size_to_sizeclass_full(size))); // set up meta data so sizeclass is correct, and hence alloc size, and // external pointer. @@ -267,11 +267,10 @@ namespace snmalloc p.unsafe_ptr(), alloc_size(p.unsafe_ptr())); #endif - const MetaslabMetaEntry& entry = - SharedStateHandle::Pagemap::template get_metaentry( - address_cast(p)); + const MetaEntry& entry = + SharedStateHandle::Pagemap::get_metaentry(address_cast(p)); local_cache.remote_dealloc_cache.template dealloc( - entry.get_remote()->trunc_id(), p, key_global); + FrontendMetaEntry::get_remote(entry)->trunc_id(), p, key_global); post_remote_cache(); return; } @@ -625,10 +624,11 @@ namespace snmalloc capptr::Alloc p_tame = capptr_domesticate( core_alloc->backend_state_ptr(), p_wild); - const MetaslabMetaEntry& entry = - SharedStateHandle::Pagemap::template get_metaentry( - address_cast(p_tame)); - if (SNMALLOC_LIKELY(local_cache.remote_allocator == entry.get_remote())) + const MetaEntry& entry = + SharedStateHandle::Pagemap::get_metaentry(address_cast(p_tame)); + if (SNMALLOC_LIKELY( + local_cache.remote_allocator == + FrontendMetaEntry::get_remote(entry))) { # if defined(__CHERI_PURE_CAPABILITY__) && defined(SNMALLOC_CHECK_CLIENT) dealloc_cheri_checks(p_tame.unsafe_ptr()); @@ -640,7 +640,8 @@ namespace snmalloc return; } - if (SNMALLOC_LIKELY(entry.get_remote() != nullptr)) + RemoteAllocator* remote = FrontendMetaEntry::get_remote(entry); + if (SNMALLOC_LIKELY(remote != nullptr)) { # if defined(__CHERI_PURE_CAPABILITY__) && defined(SNMALLOC_CHECK_CLIENT) dealloc_cheri_checks(p_tame.unsafe_ptr()); @@ -649,7 +650,7 @@ namespace snmalloc if (local_cache.remote_dealloc_cache.reserve_space(entry)) { local_cache.remote_dealloc_cache.template dealloc( - entry.get_remote()->trunc_id(), p_tame, key_global); + remote->trunc_id(), p_tame, key_global); # ifdef SNMALLOC_TRACING message<1024>( "Remote dealloc fast {} ({})", p_raw, alloc_size(p_raw)); @@ -715,11 +716,10 @@ namespace snmalloc // To handle this case we require the uninitialised pagemap contain an // entry for the first chunk of memory, that states it represents a // large object, so we can pull the check for null off the fast path. - const MetaslabMetaEntry& entry = - SharedStateHandle::Pagemap::template get_metaentry( - address_cast(p_raw)); + const MetaEntry& entry = + SharedStateHandle::Pagemap::get_metaentry(address_cast(p_raw)); - return sizeclass_full_to_size(entry.get_sizeclass()); + return sizeclass_full_to_size(FrontendMetaEntry::get_sizeclass(entry)); #endif } @@ -761,10 +761,11 @@ namespace snmalloc size_t remaining_bytes(const void* p) { #ifndef SNMALLOC_PASS_THROUGH - const MetaslabMetaEntry& entry = SharedStateHandle::Pagemap:: - template get_metaentry(address_cast(p)); + const MetaEntry& entry = + SharedStateHandle::Pagemap::template get_metaentry( + address_cast(p)); - auto sizeclass = entry.get_sizeclass(); + auto sizeclass = FrontendMetaEntry::get_sizeclass(entry); return snmalloc::remaining_bytes(sizeclass, address_cast(p)); #else return pointer_diff(p, reinterpret_cast(UINTPTR_MAX)); @@ -789,10 +790,11 @@ namespace snmalloc size_t index_in_object(const void* p) { #ifndef SNMALLOC_PASS_THROUGH - const MetaslabMetaEntry& entry = SharedStateHandle::Pagemap:: - template get_metaentry(address_cast(p)); + const MetaEntry& entry = + SharedStateHandle::Pagemap::template get_metaentry( + address_cast(p)); - auto sizeclass = entry.get_sizeclass(); + auto sizeclass = FrontendMetaEntry::get_sizeclass(entry); return snmalloc::index_in_object(sizeclass, address_cast(p)); #else return reinterpret_cast(p); diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h index 269511465..997784d15 100644 --- a/src/mem/metaslab.h +++ b/src/mem/metaslab.h @@ -226,18 +226,17 @@ namespace snmalloc }; /* - * A convenience wrapper aroun MetaEntry with a meaningful RemoteAllocator - * pointer. This encodes a RemoteAllocator* and a sizeclass_t into a the - * uintptr_t remote_and_sizeclass field. + * Define the encoding of a RemoteAllocator* and a sizeclass_t into a + * MetaEntry's uintptr_t remote_and_sizeclass field. * * There's a little bit of an asymmetry here. Since the backend actually sets - * the entry (when associating a metadata structure), MetaslabMetaEntry-s are - * not constructed directly; please use ::encode(). On the other hand, the - * backend's Pagemap::get_metaentry() method is templated on its return type, - * so it is relatively straightforward to view a pagemap entry as a - * MetaslabMetaEntry and then use the accessors here for decoding. + * the entry (when associating a metadata structure), we don't construct a + * full MetaEntry here, but rather use ::encode() to compute its + * remote_and_sizeclass value. On the decode side, we are given read-only + * access to MetaEntry-s so can directly read therefrom rather than having to + * speak in terms of uintptr_t-s. */ - struct MetaslabMetaEntry : public MetaEntry + struct FrontendMetaEntry { /// Perform the encoding. static SNMALLOC_FAST_PATH uintptr_t @@ -248,19 +247,21 @@ namespace snmalloc reinterpret_cast(remote), sizeclass.raw()); } - [[nodiscard]] SNMALLOC_FAST_PATH RemoteAllocator* get_remote() const + [[nodiscard]] static SNMALLOC_FAST_PATH RemoteAllocator* + get_remote(const MetaEntry& me) { return reinterpret_cast( pointer_align_down( - get_remote_and_sizeclass())); + me.get_remote_and_sizeclass())); } - [[nodiscard]] SNMALLOC_FAST_PATH sizeclass_t get_sizeclass() const + [[nodiscard]] static SNMALLOC_FAST_PATH sizeclass_t + get_sizeclass(const MetaEntry& me) { // TODO: perhaps remove static_cast with resolution of // https://github.com/CTSRD-CHERI/llvm-project/issues/588 return sizeclass_t::from_raw( - static_cast(get_remote_and_sizeclass()) & + static_cast(me.get_remote_and_sizeclass()) & (REMOTE_WITH_BACKEND_MARKER_ALIGN - 1)); } @@ -269,13 +270,11 @@ namespace snmalloc * assert that this chunk is being used as a slab (i.e., has an associated * owning allocator). */ - [[nodiscard]] SNMALLOC_FAST_PATH Metaslab* get_metaslab() const + [[nodiscard]] static SNMALLOC_FAST_PATH Metaslab* + get_metaslab(const MetaEntry& me) { - SNMALLOC_ASSERT(get_remote() != nullptr); - return reinterpret_cast(get_meta()); + SNMALLOC_ASSERT(get_remote(me) != nullptr); + return reinterpret_cast(me.get_meta()); } }; - - static_assert(sizeof(MetaslabMetaEntry) == sizeof(MetaEntry)); - } // namespace snmalloc diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index e97df1a47..a258a38d0 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -52,10 +52,10 @@ namespace snmalloc * * This does not require initialisation to be safely called. */ - SNMALLOC_FAST_PATH bool reserve_space(const MetaslabMetaEntry& entry) + SNMALLOC_FAST_PATH bool reserve_space(const MetaEntry& entry) { - auto size = - static_cast(sizeclass_full_to_size(entry.get_sizeclass())); + auto size = static_cast( + sizeclass_full_to_size(FrontendMetaEntry::get_sizeclass(entry))); bool result = capacity > size; if (result) @@ -101,10 +101,9 @@ namespace snmalloc if (!list[i].empty()) { auto [first, last] = list[i].extract_segment(key); - const MetaslabMetaEntry& entry = - SharedStateHandle::Pagemap::template get_metaentry< - MetaslabMetaEntry>(address_cast(first)); - auto remote = entry.get_remote(); + const MetaEntry& entry = + SharedStateHandle::Pagemap::get_metaentry(address_cast(first)); + auto remote = FrontendMetaEntry::get_remote(entry); // If the allocator is not correctly aligned, then the bit that is // set implies this is used by the backend, and we should not be // deallocating memory here. @@ -142,10 +141,9 @@ namespace snmalloc // Use the next N bits to spread out remote deallocs in our own // slot. auto r = resend.take(key, domesticate); - const MetaslabMetaEntry& entry = - SharedStateHandle::Pagemap::template get_metaentry< - MetaslabMetaEntry>(address_cast(r)); - auto i = entry.get_remote()->trunc_id(); + const MetaEntry& entry = + SharedStateHandle::Pagemap::get_metaentry(address_cast(r)); + auto i = FrontendMetaEntry::get_remote(entry)->trunc_id(); size_t slot = get_slot(i, post_round); list[slot].add(r, key); } From deac29c576ff1ec5f8aea64082d54f1070ab4262 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Sun, 13 Mar 2022 14:10:28 +0000 Subject: [PATCH 238/302] CheriBSD renamed VMEM SW perm bit CheriBSD 00d71bd4d11af448871d196f987c2ded474f3039 changes "CHERI_PERM_CHERIABI_VMMAP" to be spelled "CHERI_PERM_SW_VMEM" and deprecated the old form. Follow along with fallback so we can use older CheriBSDs. --- src/pal/pal_freebsd.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/pal/pal_freebsd.h b/src/pal/pal_freebsd.h index 66ef92565..916fc7053 100644 --- a/src/pal/pal_freebsd.h +++ b/src/pal/pal_freebsd.h @@ -3,11 +3,14 @@ #if defined(__FreeBSD__) && !defined(_KERNEL) # include "pal_bsd_aligned.h" -// On CHERI platforms, we need to know the value of CHERI_PERM_CHERIABI_VMMAP. +// On CHERI platforms, we need to know the value of CHERI_PERM_SW_VMEM. // This pollutes the global namespace a little, sadly, but I think only with // symbols that begin with CHERI_, which is as close to namespaces as C offers. # if defined(__CHERI_PURE_CAPABILITY__) # include +# if !defined(CHERI_PERM_SW_VMEM) +# define CHERI_PERM_SW_VMEM CHERI_PERM_CHERIABI_VMMAP +# endif # endif namespace snmalloc @@ -108,7 +111,7 @@ namespace snmalloc /** * On CheriBSD, exporting a pointer means stripping it of the authority to - * manage the address space it references by clearing the CHERIABI_VMMAP + * manage the address space it references by clearing the SW_VMEM * permission bit. */ template @@ -124,8 +127,7 @@ namespace snmalloc } return CapPtr>( __builtin_cheri_perms_and( - p.unsafe_ptr(), - ~static_cast(CHERI_PERM_CHERIABI_VMMAP))); + p.unsafe_ptr(), ~static_cast(CHERI_PERM_SW_VMEM))); } # endif }; From dec65d1b4a4778c76c22532370d33948b9c697b8 Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Fri, 25 Mar 2022 15:21:16 +0000 Subject: [PATCH 239/302] Make MetaCommon state public. C++ has no way of exposing it only to things that are backend implementations. Eventually this class should be made purely part of the CHERI back end. --- src/backend/metatypes.h | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/backend/metatypes.h b/src/backend/metatypes.h index 66462034a..354d8f0eb 100644 --- a/src/backend/metatypes.h +++ b/src/backend/metatypes.h @@ -15,16 +15,10 @@ namespace snmalloc * This class's data is fully private but is friends with the relevant backend * types and, thus, is "opaque" to the frontend. */ - class MetaCommon + struct MetaCommon { - friend class ChunkAllocator; - - template - friend class BackendAllocator; - capptr::Chunk chunk; - public: /** * Expose the address of, though not the authority to, our corresponding * chunk. From 6386cd7eca12b0c7cc2b2001b95ce0e0fb940aa7 Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Mon, 28 Mar 2022 11:11:31 +0100 Subject: [PATCH 240/302] Expose ranges in snmalloc_core. (#496) Also add a missing #pragma once that cause multiple includes to error. --- src/backend/commitrange.h | 3 ++- src/backend/empty_range.h | 3 ++- src/snmalloc_core.h | 10 ++++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/backend/commitrange.h b/src/backend/commitrange.h index 43b1f8842..a5b24d92a 100644 --- a/src/backend/commitrange.h +++ b/src/backend/commitrange.h @@ -1,6 +1,7 @@ #pragma once #include "../ds/ptrwrap.h" +#include "../pal/pal_consts.h" namespace snmalloc { @@ -43,4 +44,4 @@ namespace snmalloc parent->dealloc_range(base, size); } }; -} // namespace snmalloc \ No newline at end of file +} // namespace snmalloc diff --git a/src/backend/empty_range.h b/src/backend/empty_range.h index 36680f3a3..5287cd65d 100644 --- a/src/backend/empty_range.h +++ b/src/backend/empty_range.h @@ -1,3 +1,4 @@ +#pragma once #include "../ds/ptrwrap.h" namespace snmalloc @@ -28,4 +29,4 @@ namespace snmalloc return nullptr; } }; -} // namespace snmalloc \ No newline at end of file +} // namespace snmalloc diff --git a/src/snmalloc_core.h b/src/snmalloc_core.h index 7ae0a3314..5cd224f4d 100644 --- a/src/snmalloc_core.h +++ b/src/snmalloc_core.h @@ -1,5 +1,15 @@ #pragma once +#include "backend/commitrange.h" #include "backend/commonconfig.h" +#include "backend/empty_range.h" +#include "backend/globalrange.h" +#include "backend/largebuddyrange.h" #include "backend/pagemap.h" +#include "backend/pagemapregisterrange.h" +#include "backend/palrange.h" +#include "backend/range_helpers.h" +#include "backend/smallbuddyrange.h" +#include "backend/statsrange.h" +#include "backend/subrange.h" #include "mem/globalalloc.h" From ee70f952d1e33055d71d2cc4702394383daa79fb Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Mon, 28 Mar 2022 11:53:40 +0100 Subject: [PATCH 241/302] Allow POSIX PALs to overwrite writev and fsync. (#495) On FreeBSD (possibly elsewhere) the normal versions of these go via an indirection layer because they are pthread cancellation points. This indirection layer does not work correctly if pthreads can't allocate memory and so we can't get debug output until malloc is working, at least a little bit. With this version, we can call the __sys_ variants, which skip any libc / libthr interposition. --- src/pal/pal_bsd.h | 4 ++-- src/pal/pal_bsd_aligned.h | 4 ++-- src/pal/pal_freebsd.h | 12 +++++++++++- src/pal/pal_posix.h | 10 ++++++---- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/pal/pal_bsd.h b/src/pal/pal_bsd.h index a07b7c526..4689b43c0 100644 --- a/src/pal/pal_bsd.h +++ b/src/pal/pal_bsd.h @@ -10,8 +10,8 @@ namespace snmalloc * Generic *BSD PAL mixin. This provides features that are common to the BSD * family. */ - template - class PALBSD : public PALPOSIX + template + class PALBSD : public PALPOSIX { public: /** diff --git a/src/pal/pal_bsd_aligned.h b/src/pal/pal_bsd_aligned.h index c770a529a..4c88287f3 100644 --- a/src/pal/pal_bsd_aligned.h +++ b/src/pal/pal_bsd_aligned.h @@ -10,8 +10,8 @@ namespace snmalloc * This adds aligned allocation using `MAP_ALIGNED` to the generic BSD * implementation. This flag is supported by NetBSD and FreeBSD. */ - template - class PALBSD_Aligned : public PALBSD + template + class PALBSD_Aligned : public PALBSD { public: /** diff --git a/src/pal/pal_freebsd.h b/src/pal/pal_freebsd.h index 916fc7053..d24f4b7cf 100644 --- a/src/pal/pal_freebsd.h +++ b/src/pal/pal_freebsd.h @@ -13,6 +13,15 @@ # endif # endif +/** + * Direct system-call wrappers so that we can skip libthr interception, which + * won't work if malloc is broken. + * @{ + */ +extern "C" ssize_t __sys_writev(int fd, const struct iovec* iov, int iovcnt); +extern "C" int __sys_fsync(int fd); +/// @} + namespace snmalloc { /** @@ -21,7 +30,8 @@ namespace snmalloc * This adds FreeBSD-specific aligned allocation to the generic BSD * implementation. */ - class PALFreeBSD : public PALBSD_Aligned + class PALFreeBSD + : public PALBSD_Aligned { public: /** diff --git a/src/pal/pal_posix.h b/src/pal/pal_posix.h index 16c5ef7d2..add7712ef 100644 --- a/src/pal/pal_posix.h +++ b/src/pal/pal_posix.h @@ -34,11 +34,13 @@ namespace snmalloc * efficient implementation. Subclasses should provide more efficient * implementations using platform-specific functionality. * - * The template parameter for this is the subclass and is used for explicit - * up casts to allow this class to call non-virtual methods on the templated - * version. + * The first template parameter for this is the subclass and is used for + * explicit up casts to allow this class to call non-virtual methods on the + * templated version. The next two allow subclasses to provide `writev` and + * `fsync` implementations that bypass any libc machinery that might not be + * working when an early-malloc error appears. */ - template + template class PALPOSIX : public PalTimerDefaultImpl> { /** From ede7dbb3ef4b668a4476a54aab84b60d1d852a71 Mon Sep 17 00:00:00 2001 From: David CARLIER Date: Tue, 29 Mar 2022 13:58:00 +0100 Subject: [PATCH 242/302] Following up using writev/fsync direct syscalls to avoid pthread interpositions (#499) As these calls carries a pthread cancellation check and due to pthread allocation libc init timing, we directly access their syscalls instead. --- src/pal/pal_netbsd.h | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/pal/pal_netbsd.h b/src/pal/pal_netbsd.h index 377c1ccaf..75394704f 100644 --- a/src/pal/pal_netbsd.h +++ b/src/pal/pal_netbsd.h @@ -5,6 +5,16 @@ # include +/** + * We skip the pthread cancellation checkpoints by reaching directly + * the following syscalls so we avoid the possible pthread + * allocation initialization timing issues. + * @{ + */ +extern "C" ssize_t _sys_writev(int fd, const struct iovec* iov, int iovcnt); +extern "C" int _sys_fsync(int fd); +/// @} + namespace snmalloc { /** @@ -13,7 +23,7 @@ namespace snmalloc * This adds NetBSD-specific aligned allocation to the generic BSD * implementation. */ - class PALNetBSD : public PALBSD_Aligned + class PALNetBSD : public PALBSD_Aligned { public: /** From 65ee6b2a2f0ac512b51e1df3b1f6a859b07f8044 Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Fri, 1 Apr 2022 17:32:53 +0100 Subject: [PATCH 243/302] Refactor MetaSlab / MetaCommon. (#501) MetaCommon is now gone. The back end must provide a SlabMetadata, which must be a subtype of MetaSlab (i.e. MetaSlab or a subclass of MetaSlab). It may add additional state here. The MetaEntry is now templated on the concrete subclass of MetaSlab that the back-end uses. The MetaEntry still stores this as a `uintptr_t` to allow easier toggling of the boundary bit but the interfaces are all in terms of stable types now. Also some tidying of names (SharedStateHandle is now called Backend). In a follow-on PR, we can then remove the chunk field from the BackendMetadata in the non-CHERI back end and allow back ends that don't require extra state to use MetaSlab directly. Other cleanups: - Remove backend/metatypes, define the types that the front end expects in mem/metaslab. The back end may extend them but these types define part of the contract between the front and back ends. - Remove FrontendMetaEntry and fold its methods into MetaEntry. - For example purposes, the default back end now extends MetaEntry. This also ensures that nothing in the front end depends on the specific type of MetaEntry. - Some things now have more sensible names. The meta entry now operates in one of three modes: - When owned by the front end, it stores a pointer to a remote, a pointer to some MetaSlab subclass, and a sizeclass. - When owned by the back end, it stores two back-end defined values that must fit in the bits of `uintptr_t` that are not reserved for the MetaEntry itself. - When not owned by either, it can be queried as if owned by the front end. The red-black tree has been refactored to allow the holder to be a wrapper type, removing all of the Holder* and Holder& uses and treating it uniformly as a value type that can be used to access the contents. The chunk field is fone from the slab medatada. This will need to be added back in the CHERI back ends, but it's a back-end policy. The back end can choose to use it or not, depending on whether it can safely convert between an Alloc-bounded pointer and a Chunk-bounded pointer. The term 'metaslab' originated in snmalloc 1 to mean a slab of slabs. In the snmalloc2 branch it was repurposed to mean metadata about a slab. To make this clearer, all uses of metaslab are now gone and have been renamed to slab metadata. The frontend metadata classes are all prefixed Frontend and some extra invariants are checked with `static_assert`. --- src/backend/backend.h | 146 +++-- src/backend/backend_concept.h | 104 +-- src/backend/largebuddyrange.h | 121 ++-- src/backend/metatypes.h | 186 ------ src/backend/smallbuddyrange.h | 48 +- src/ds/helpers.h | 6 + src/ds/redblacktree.h | 125 +++- src/ds/seqset.h | 45 +- src/mem/corealloc.h | 216 ++++--- src/mem/localalloc.h | 100 +-- src/mem/metadata.h | 636 +++++++++++++++++++ src/mem/metaslab.h | 280 -------- src/mem/remoteallocator.h | 15 +- src/mem/remotecache.h | 44 +- src/test/func/domestication/domestication.cc | 4 +- src/test/func/redblack/redblack.cc | 75 ++- 16 files changed, 1294 insertions(+), 857 deletions(-) delete mode 100644 src/backend/metatypes.h create mode 100644 src/mem/metadata.h delete mode 100644 src/mem/metaslab.h diff --git a/src/backend/backend.h b/src/backend/backend.h index d1022a34b..6d33827fc 100644 --- a/src/backend/backend.h +++ b/src/backend/backend.h @@ -1,12 +1,12 @@ #pragma once #include "../mem/allocconfig.h" +#include "../mem/metadata.h" #include "../pal/pal.h" #include "commitrange.h" #include "commonconfig.h" #include "empty_range.h" #include "globalrange.h" #include "largebuddyrange.h" -#include "metatypes.h" #include "pagemap.h" #include "pagemapregisterrange.h" #include "palrange.h" @@ -34,23 +34,88 @@ namespace snmalloc * This class implements the standard backend for handling allocations. * It abstracts page table management and address space management. */ - template< - SNMALLOC_CONCEPT(ConceptPAL) PAL, - bool fixed_range, - typename PageMapEntry = MetaEntry> + template class BackendAllocator : public CommonConfig { public: using Pal = PAL; + using SlabMetadata = FrontendSlabMetadata; + class Pagemap { friend class BackendAllocator; + public: + /** + * Export the type stored in the pagemap. + * The following class could be replaced by: + * + * ``` + * using Entry = FrontendMetaEntry; + * ``` + * + * The full form here provides an example of how to extend the pagemap + * entries. It also guarantees that the front end never directly + * constructs meta entries, it only ever reads them or modifies them in + * place. + */ + class Entry : public FrontendMetaEntry + { + /** + * The private initialising constructor is usable only by this back end. + */ + friend class BackendAllocator; + + /** + * The private default constructor is usable only by the pagemap. + */ + friend class FlatPagemap; + + /** + * The only constructor that creates newly initialised meta entries. + * This is callable only by the back end. The front end may copy, + * query, and update these entries, but it may not create them + * directly. This contract allows the back end to store any arbitrary + * metadata in meta entries when they are first constructed. + */ + SNMALLOC_FAST_PATH + Entry(SlabMetadata* meta, uintptr_t ras) + : FrontendMetaEntry(meta, ras) + {} + + /** + * Default constructor. This must be callable from the pagemap. + */ + SNMALLOC_FAST_PATH Entry() = default; + + /** + * Copy assignment is used only by the pagemap. + */ + Entry& operator=(const Entry& other) + { + FrontendMetaEntry::operator=(other); + return *this; + } + }; + + private: SNMALLOC_REQUIRE_CONSTINIT - static inline FlatPagemap + static inline FlatPagemap concretePagemap; + /** + * Set the metadata associated with a chunk. + */ + SNMALLOC_FAST_PATH + static void set_metaentry(address_t p, size_t size, const Entry& t) + { + for (address_t a = p; a < p + size; a += MIN_CHUNK_SIZE) + { + concretePagemap.set(a, t); + } + } + public: /** * Get the metadata associated with a chunk. @@ -59,7 +124,7 @@ namespace snmalloc * to access a location that is not backed by a chunk. */ template - SNMALLOC_FAST_PATH static const MetaEntry& get_metaentry(address_t p) + SNMALLOC_FAST_PATH static const auto& get_metaentry(address_t p) { return concretePagemap.template get(p); } @@ -71,23 +136,11 @@ namespace snmalloc * to access a location that is not backed by a chunk. */ template - SNMALLOC_FAST_PATH static MetaEntry& get_metaentry_mut(address_t p) + SNMALLOC_FAST_PATH static auto& get_metaentry_mut(address_t p) { return concretePagemap.template get_mut(p); } - /** - * Set the metadata associated with a chunk. - */ - SNMALLOC_FAST_PATH - static void set_metaentry(address_t p, size_t size, const MetaEntry& t) - { - for (address_t a = p; a < p + size; a += MIN_CHUNK_SIZE) - { - concretePagemap.set(a, t); - } - } - static void register_range(address_t p, size_t sz) { concretePagemap.register_range(p, sz); @@ -246,26 +299,23 @@ namespace snmalloc /** * Returns a chunk of memory with alignment and size of `size`, and a - * metaslab block. + * block containing metadata about the slab. * * It additionally set the meta-data for this chunk of memory to * be - * (remote, sizeclass, metaslab) - * where metaslab, is the second element of the pair return. + * (remote, sizeclass, slab_metadata) + * where slab_metadata, is the second element of the pair return. */ - static std::pair, Metaslab*> + static std::pair, SlabMetadata*> alloc_chunk(LocalState& local_state, size_t size, uintptr_t ras) { SNMALLOC_ASSERT(bits::is_pow2(size)); SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE); - SNMALLOC_ASSERT((ras & MetaEntry::REMOTE_BACKEND_MARKER) == 0); - ras &= ~MetaEntry::REMOTE_BACKEND_MARKER; - auto meta_cap = - local_state.get_meta_range()->alloc_range(PAGEMAP_METADATA_STRUCT_SIZE); + local_state.get_meta_range()->alloc_range(sizeof(SlabMetadata)); - auto meta = meta_cap.template as_reinterpret().unsafe_ptr(); + auto meta = meta_cap.template as_reinterpret().unsafe_ptr(); if (meta == nullptr) { @@ -281,7 +331,7 @@ namespace snmalloc if (p == nullptr) { local_state.get_meta_range()->dealloc_range( - meta_cap, PAGEMAP_METADATA_STRUCT_SIZE); + meta_cap, sizeof(SlabMetadata)); errno = ENOMEM; #ifdef SNMALLOC_TRACING message<1024>("Out of memory"); @@ -289,32 +339,44 @@ namespace snmalloc return {p, nullptr}; } - meta->meta_common.chunk = p; - - MetaEntry t(&meta->meta_common, ras); + typename Pagemap::Entry t(meta, ras); Pagemap::set_metaentry(address_cast(p), size, t); p = Aal::capptr_bound(p, size); return {p, meta}; } - static void - dealloc_chunk(LocalState& local_state, MetaCommon& meta_common, size_t size) + static void dealloc_chunk( + LocalState& local_state, + SlabMetadata& slab_metadata, + capptr::Alloc alloc, + size_t size) { - auto chunk = meta_common.chunk; - /* * The backend takes possession of these chunks now, by disassociating * any existing remote allocator and metadata structure. If - * interrogated, the sizeclass reported by the MetaEntry is 0, which has - * size 0. + * interrogated, the sizeclass reported by the FrontendMetaEntry is 0, + * which has size 0. */ - MetaEntry t(nullptr, MetaEntry::REMOTE_BACKEND_MARKER); - Pagemap::set_metaentry(address_cast(chunk), size, t); + typename Pagemap::Entry t(nullptr, 0); + t.claim_for_backend(); + SNMALLOC_ASSERT_MSG( + Pagemap::get_metaentry(address_cast(alloc)).get_slab_metadata() == + &slab_metadata, + "Slab metadata {} passed for address {} does not match the meta entry " + "{} that is used for that address", + &slab_metadata, + address_cast(alloc), + Pagemap::get_metaentry(address_cast(alloc)).get_slab_metadata()); + Pagemap::set_metaentry(address_cast(alloc), size, t); local_state.get_meta_range()->dealloc_range( - capptr::Chunk(&meta_common), PAGEMAP_METADATA_STRUCT_SIZE); + capptr::Chunk(&slab_metadata), sizeof(SlabMetadata)); + // On non-CHERI platforms, we don't need to re-derive to get a pointer to + // the chunk. On CHERI platforms this will need to be stored in the + // SlabMetadata or similar. + capptr::Chunk chunk{alloc.unsafe_ptr()}; local_state.object_range->dealloc_range(chunk, size); } diff --git a/src/backend/backend_concept.h b/src/backend/backend_concept.h index 895e19af2..27aa8df39 100644 --- a/src/backend/backend_concept.h +++ b/src/backend/backend_concept.h @@ -1,34 +1,32 @@ #pragma once #ifdef __cpp_concepts -# include +# include "../ds/address.h" # include "../ds/concept.h" # include "../pal/pal_concept.h" -# include "../ds/address.h" + +# include namespace snmalloc { - class MetaEntry; - /** * The core of the static pagemap accessor interface: {get,set}_metadata. * - * get_metadata takes a bool-ean template parameter indicating whether it may + * get_metadata takes a boolean template parameter indicating whether it may * be accessing memory that is not known to be committed. */ template concept ConceptBackendMeta = - requires( - address_t addr, - size_t sz, - const MetaEntry& t) + requires(address_t addr, size_t sz, const typename Meta::Entry& t) { - { Meta::set_metaentry(addr, sz, t) } -> ConceptSame; - - { Meta::template get_metaentry(addr) } - -> ConceptSame; + { + Meta::template get_metaentry(addr) + } + ->ConceptSame; - { Meta::template get_metaentry(addr) } - -> ConceptSame; + { + Meta::template get_metaentry(addr) + } + ->ConceptSame; }; /** @@ -39,10 +37,27 @@ namespace snmalloc * underscore) below, which combines this and the core concept, above. */ template - concept ConceptBackendMeta_Range = - requires(address_t addr, size_t sz) + concept ConceptBackendMeta_Range = requires(address_t addr, size_t sz) { - { Meta::register_range(addr, sz) } -> ConceptSame; + { + Meta::register_range(addr, sz) + } + ->ConceptSame; + }; + + template + concept ConceptBuddyRangeMeta = + requires(address_t addr, size_t sz, const typename Meta::Entry& t) + { + { + Meta::template get_metaentry_mut(addr) + } + ->ConceptSame; + + { + Meta::template get_metaentry_mut(addr) + } + ->ConceptSame; }; /** @@ -55,7 +70,7 @@ namespace snmalloc */ template concept ConceptBackendMetaRange = - ConceptBackendMeta && ConceptBackendMeta_Range; + ConceptBackendMeta&& ConceptBackendMeta_Range; /** * The backend also defines domestication (that is, the difference between @@ -65,16 +80,18 @@ namespace snmalloc */ template concept ConceptBackendDomestication = - requires( - typename Globals::LocalState* ls, - capptr::AllocWild ptr) + requires(typename Globals::LocalState* ls, capptr::AllocWild ptr) + { { - { Globals::capptr_domesticate(ls, ptr) } - -> ConceptSame>; + Globals::capptr_domesticate(ls, ptr) + } + ->ConceptSame>; - { Globals::capptr_domesticate(ls, ptr.template as_static()) } - -> ConceptSame>; - }; + { + Globals::capptr_domesticate(ls, ptr.template as_static()) + } + ->ConceptSame>; + }; class CommonConfig; struct Flags; @@ -88,24 +105,33 @@ namespace snmalloc * * have static pagemap accessors via T::Pagemap * * define a T::LocalState type (and alias it as T::Pagemap::LocalState) * * define T::Options of type snmalloc::Flags - * * expose the global allocator pool via T::pool() + * * expose the global allocator pool via T::pool() if pool allocation is + * used. * */ template concept ConceptBackendGlobals = - std::is_base_of::value && - ConceptPAL && - ConceptBackendMetaRange && - requires() - { - typename Globals::LocalState; - - { Globals::Options } -> ConceptSameModRef; + std::is_base_of::value&& + ConceptPAL&& + ConceptBackendMetaRange&& requires() + { + typename Globals::LocalState; + { + Globals::Options + } + ->ConceptSameModRef; + } + &&( + requires() { + Globals::Options.CoreAllocIsPoolAllocated == true; typename Globals::GlobalPoolState; - { Globals::pool() } -> ConceptSame; - - }; + { + Globals::pool() + } + ->ConceptSame; + } || + requires() { Globals::Options.CoreAllocIsPoolAllocated == false; }); /** * The lazy version of the above; please see ds/concept.h and use sparingly. diff --git a/src/backend/largebuddyrange.h b/src/backend/largebuddyrange.h index a4a807aa2..a57ed6973 100644 --- a/src/backend/largebuddyrange.h +++ b/src/backend/largebuddyrange.h @@ -4,6 +4,7 @@ #include "../ds/bits.h" #include "../mem/allocconfig.h" #include "../pal/pal.h" +#include "backend_concept.h" #include "buddy.h" #include "range_helpers.h" @@ -14,7 +15,7 @@ namespace snmalloc /** * Class for using the pagemap entries for the buddy allocator. */ - template + template class BuddyChunkRep { public: @@ -23,68 +24,86 @@ namespace snmalloc * of) chunks of the address space; as such, bits in (MIN_CHUNK_SIZE - 1) * are unused and so the RED_BIT is packed therein. However, in practice, * these are not "just any" uintptr_t-s, but specifically the uintptr_t-s - * inside the Pagemap's MetaEntry structures. As such, there are some - * additional bit-swizzling concerns; see set() and get() below. + * inside the Pagemap's BackendAllocator::Entry structures. + * + * The BackendAllocator::Entry provides us with helpers that guarantee that + * we use only the bits that we are allowed to. + * @{ */ - using Holder = uintptr_t; + using Handle = MetaEntryBase::BackendStateWordRef; using Contents = uintptr_t; + ///@} - static constexpr address_t RED_BIT = 1 << 1; + /** + * The bit that we will use to mark an entry as red. + * This has constraints in two directions, it must not be one of the + * reserved bits from the perspective of the meta entry and it must not be + * a bit that is a valid part of the address of a chunk. + * @{ + */ + static constexpr address_t RED_BIT = 1 << 8; static_assert(RED_BIT < MIN_CHUNK_SIZE); - static_assert(RED_BIT != MetaEntry::META_BOUNDARY_BIT); - static_assert(RED_BIT != MetaEntry::REMOTE_BACKEND_MARKER); + static_assert(MetaEntryBase::is_backend_allowed_value( + MetaEntryBase::Word::One, RED_BIT)); + static_assert(MetaEntryBase::is_backend_allowed_value( + MetaEntryBase::Word::Two, RED_BIT)); + ///@} + /// The value of a null node, as returned by `get` static constexpr Contents null = 0; + /// The value of a null node, as stored in a `uintptr_t`. + static constexpr Contents root = 0; - static void set(Holder* ptr, Contents r) + /** + * Set the value. Preserve the red/black colour. + */ + static void set(Handle ptr, Contents r) { - SNMALLOC_ASSERT((r & (MIN_CHUNK_SIZE - 1)) == 0); - /* - * Preserve lower bits, claim as backend, and update contents of holder. - * - * This is excessive at present but no harder than being more precise - * while also being future-proof. All that is strictly required would be - * to preserve META_BOUNDARY_BIT and RED_BIT in ->meta and to assert - * REMOTE_BACKEND_MARKER in ->remote_and_sizeclass (if it isn't already - * asserted). However, we don't know which Holder* we have been given, - * nor do we know whether this Holder* is completely new (and so we are - * the first reasonable opportunity to assert REMOTE_BACKEND_MARKER) or - * recycled from the frontend, and so we preserve and assert more than - * strictly necessary. - * - * The use of `address_cast` below is a CHERI-ism; otherwise both `r` and - * `*ptr & ...` are plausibly provenance-carrying values and the compiler - * balks at the ambiguity. - */ - *ptr = r | address_cast(*ptr & (MIN_CHUNK_SIZE - 1)) | - MetaEntry::REMOTE_BACKEND_MARKER; + ptr = r | (static_cast(ptr.get()) & RED_BIT); } - static Contents get(const Holder* ptr) + /** + * Returns the value, stripping out the red/black colour. + */ + static Contents get(const Handle ptr) { - return *ptr & ~(MIN_CHUNK_SIZE - 1); + return ptr.get() & ~RED_BIT; } - static Holder& ref(bool direction, Contents k) + /** + * Returns a pointer to the tree node for the specified address. + */ + static Handle ref(bool direction, Contents k) { - MetaEntry& entry = - Pagemap::template get_metaentry_mut(address_cast(k)); + // Special case for accessing the null entry. We want to make sure + // that this is never modified by the back end, so we make it point to + // a constant entry and use the MMU to trap even in release modes. + static const Contents null_entry = 0; + if (SNMALLOC_UNLIKELY(address_cast(k) == 0)) + { + return {const_cast(&null_entry)}; + } + auto& entry = Pagemap::template get_metaentry_mut(address_cast(k)); if (direction) - return entry.meta; + return entry.get_backend_word(Pagemap::Entry::Word::One); - return entry.remote_and_sizeclass; + return entry.get_backend_word(Pagemap::Entry::Word::Two); } static bool is_red(Contents k) { - return (ref(true, k) & RED_BIT) == RED_BIT; + return (ref(true, k).get() & RED_BIT) == RED_BIT; } static void set_red(Contents k, bool new_is_red) { if (new_is_red != is_red(k)) - ref(true, k) ^= RED_BIT; + { + auto v = ref(true, k); + v = v.get() ^ RED_BIT; + } + SNMALLOC_ASSERT(is_red(k) == new_is_red); } static Contents offset(Contents k, size_t size) @@ -117,6 +136,24 @@ namespace snmalloc return k; } + /** + * Convert the pointer wrapper into something that the snmalloc debug + * printing code can print. + */ + static address_t printable(Handle k) + { + return k.printable_address(); + } + + /** + * Returns the name for use in debugging traces. Not used in normal builds + * (release or debug), only when tracing is enabled. + */ + static const char* name() + { + return "BuddyChunkRep"; + } + static bool can_consolidate(Contents k, size_t size) { // Need to know both entries exist in the pagemap. @@ -125,7 +162,7 @@ namespace snmalloc // The buddy could be in a part of the pagemap that has // not been registered and thus could segfault on access. auto larger = bits::max(k, buddy(k, size)); - MetaEntry& entry = + auto& entry = Pagemap::template get_metaentry_mut(address_cast(larger)); return !entry.is_boundary(); } @@ -135,7 +172,7 @@ namespace snmalloc typename ParentRange, size_t REFILL_SIZE_BITS, size_t MAX_SIZE_BITS, - SNMALLOC_CONCEPT(ConceptBackendMeta) Pagemap, + SNMALLOC_CONCEPT(ConceptBuddyRangeMeta) Pagemap, bool Consolidate = true> class LargeBuddyRange { @@ -192,7 +229,9 @@ namespace snmalloc // Tag first entry so we don't consolidate it. if (first) { - Pagemap::get_metaentry_mut(address_cast(base)).set_boundary(); + auto& entry = Pagemap::get_metaentry_mut(address_cast(base)); + entry.claim_for_backend(); + entry.set_boundary(); } } else @@ -316,4 +355,4 @@ namespace snmalloc dealloc_overflow(overflow); } }; -} // namespace snmalloc \ No newline at end of file +} // namespace snmalloc diff --git a/src/backend/metatypes.h b/src/backend/metatypes.h deleted file mode 100644 index 354d8f0eb..000000000 --- a/src/backend/metatypes.h +++ /dev/null @@ -1,186 +0,0 @@ -#pragma once -#include "../ds/concept.h" -#include "../mem/allocconfig.h" /* TODO: CACHELINE_SIZE */ -#include "../pal/pal_concept.h" - -namespace snmalloc -{ - /** - * A guaranteed type-stable sub-structure of all metadata referenced by the - * Pagemap. Use-specific structures (Metaslab, ChunkRecord) are expected to - * have this at offset zero so that, even in the face of concurrent mutation - * and reuse of the memory backing that metadata, the types of these fields - * remain fixed. - * - * This class's data is fully private but is friends with the relevant backend - * types and, thus, is "opaque" to the frontend. - */ - struct MetaCommon - { - capptr::Chunk chunk; - - /** - * Expose the address of, though not the authority to, our corresponding - * chunk. - */ - [[nodiscard]] SNMALLOC_FAST_PATH address_t chunk_address() - { - return address_cast(this->chunk); - } - - /** - * Zero (possibly by unmapping) the memory backing this chunk. We must rely - * on the caller to tell us its size, which is a little unfortunate. - */ - template - SNMALLOC_FAST_PATH void zero_chunk(size_t chunk_size) - { - PAL::zero(this->chunk.unsafe_ptr(), chunk_size); - } - }; - - static const size_t PAGEMAP_METADATA_STRUCT_SIZE = -#ifdef __CHERI_PURE_CAPABILITY__ - 2 * CACHELINE_SIZE -#else - CACHELINE_SIZE -#endif - ; - - // clang-format off - /* This triggers an ICE (C1001) in MSVC, so disable it there */ -#if defined(__cpp_concepts) && !defined(_MSC_VER) - - template - concept ConceptMetadataStruct = - // Metadata structures must be standard layout (for offsetof()), - std::is_standard_layout_v && - // must be of a sufficiently small size, - sizeof(Meta) <= PAGEMAP_METADATA_STRUCT_SIZE && - // and must be pointer-interconvertable with MetaCommon. - ( - ConceptSame || - requires(Meta m) { - // Otherwise, meta must have MetaCommon field named meta_common ... - { &m.meta_common } -> ConceptSame; - // at offset zero. - (offsetof(Meta, meta_common) == 0); - } - ); - - static_assert(ConceptMetadataStruct); -# define USE_METADATA_CONCEPT -#endif - // clang-format on - - /** - * Entry stored in the pagemap. See docs/AddressSpace.md for the full - * MetaEntry lifecycle. - */ - class MetaEntry - { - template - friend class BuddyChunkRep; - - /** - * In common cases, the pointer to the metaslab. See docs/AddressSpace.md - * for additional details. - * - * The bottom bit is used to indicate if this is the first chunk in a PAL - * allocation, that cannot be combined with the preceeding chunk. - */ - uintptr_t meta{0}; - - /** - * Bit used to indicate this should not be considered part of the previous - * PAL allocation. - * - * Some platforms cannot treat different PalAllocs as a single allocation. - * This is true on CHERI as the combined permission might not be - * representable. It is also true on Windows as you cannot Commit across - * multiple continuous VirtualAllocs. - */ - static constexpr address_t META_BOUNDARY_BIT = 1 << 0; - - /** - * In common cases, a bit-packed pointer to the owning allocator (if any), - * and the sizeclass of this chunk. See mem/metaslab.h:MetaEntryRemote for - * details of this case and docs/AddressSpace.md for further details. - */ - uintptr_t remote_and_sizeclass{0}; - - public: - /** - * This bit is set in remote_and_sizeclass to discriminate between the case - * that it is in use by the frontend (0) or by the backend (1). For the - * former case, see mem/metaslab.h; for the latter, see backend/backend.h - * and backend/largebuddyrange.h. - * - * This value is statically checked by the frontend to ensure that its - * bit packing does not conflict; see mem/remoteallocator.h - */ - static constexpr address_t REMOTE_BACKEND_MARKER = 1 << 7; - - constexpr MetaEntry() = default; - - /** - * Constructor, provides the remote and sizeclass embedded in a single - * pointer-sized word. This format is not guaranteed to be stable and so - * the second argument of this must always be the return value from - * `get_remote_and_sizeclass`. - */ - SNMALLOC_FAST_PATH - MetaEntry(MetaCommon* meta, uintptr_t remote_and_sizeclass) - : meta(unsafe_to_uintptr(meta)), - remote_and_sizeclass(remote_and_sizeclass) - {} - - /** - * Return the remote and sizeclass in an implementation-defined encoding. - * This is not guaranteed to be stable across snmalloc releases and so the - * only safe use for this is to pass it to the two-argument constructor of - * this class. - */ - [[nodiscard]] SNMALLOC_FAST_PATH const uintptr_t& - get_remote_and_sizeclass() const - { - return remote_and_sizeclass; - } - - MetaEntry(const MetaEntry&) = delete; - - MetaEntry& operator=(const MetaEntry& other) - { - // Don't overwrite the boundary bit with the other's - meta = (other.meta & ~META_BOUNDARY_BIT) | - address_cast(meta & META_BOUNDARY_BIT); - remote_and_sizeclass = other.remote_and_sizeclass; - return *this; - } - - /** - * Return the Metaslab metadata associated with this chunk, guarded by an - * assert that this chunk is being used as a slab (i.e., has an associated - * owning allocator). - */ - [[nodiscard]] SNMALLOC_FAST_PATH MetaCommon* get_meta() const - { - return reinterpret_cast(meta & ~META_BOUNDARY_BIT); - } - - void set_boundary() - { - meta |= META_BOUNDARY_BIT; - } - - [[nodiscard]] bool is_boundary() const - { - return meta & META_BOUNDARY_BIT; - } - - bool clear_boundary_bit() - { - return meta &= ~META_BOUNDARY_BIT; - } - }; -} // namespace snmalloc diff --git a/src/backend/smallbuddyrange.h b/src/backend/smallbuddyrange.h index 4ee075867..2500e1f2c 100644 --- a/src/backend/smallbuddyrange.h +++ b/src/backend/smallbuddyrange.h @@ -1,6 +1,7 @@ #pragma once #include "../ds/address.h" +#include "../mem/allocconfig.h" #include "../pal/pal.h" #include "range_helpers.h" @@ -22,13 +23,14 @@ namespace snmalloc class BuddyInplaceRep { public: - using Holder = capptr::Chunk; + using Handle = capptr::Chunk*; using Contents = capptr::Chunk; static constexpr Contents null = nullptr; + static constexpr Contents root = nullptr; static constexpr address_t MASK = 1; - static void set(Holder* ptr, Contents r) + static void set(Handle ptr, Contents r) { SNMALLOC_ASSERT((address_cast(r) & MASK) == 0); if (r == nullptr) @@ -40,44 +42,45 @@ namespace snmalloc .template as_static(); } - static Contents get(Holder* ptr) + static Contents get(Handle ptr) { return pointer_align_down<2, FreeChunk>((*ptr).as_void()); } - static Holder& ref(bool direction, Contents r) + static Handle ref(bool direction, Contents r) { if (direction) - return r->left; + return &r->left; - return r->right; + return &r->right; } static bool is_red(Contents k) { if (k == nullptr) return false; - return (address_cast(ref(false, k)) & MASK) == MASK; + return (address_cast(*ref(false, k)) & MASK) == MASK; } static void set_red(Contents k, bool new_is_red) { if (new_is_red != is_red(k)) { - auto& r = ref(false, k); - auto old_addr = pointer_align_down<2, FreeChunk>(r.as_void()); + auto r = ref(false, k); + auto old_addr = pointer_align_down<2, FreeChunk>(r->as_void()); if (new_is_red) { if (old_addr == nullptr) - r = capptr::Chunk(reinterpret_cast(MASK)); + *r = capptr::Chunk(reinterpret_cast(MASK)); else - r = pointer_offset(old_addr, MASK).template as_static(); + *r = pointer_offset(old_addr, MASK).template as_static(); } else { - r = old_addr; + *r = old_addr; } + SNMALLOC_ASSERT(is_red(k) == new_is_red); } } @@ -115,6 +118,25 @@ namespace snmalloc return address_cast(k); } + /** + * Return the holder in some format suitable for printing by snmalloc's + * debug log mechanism. Used only when used in tracing mode, not normal + * debug or release builds. Raw pointers are printable already, so this is + * the identity function. + */ + static Handle printable(Handle k) + { + return k; + } + + /** + * Return a name for use in tracing mode. Unused in any other context. + */ + static const char* name() + { + return "BuddyInplaceRep"; + } + static bool can_consolidate(Contents k, size_t size) { UNUSED(k, size); @@ -225,4 +247,4 @@ namespace snmalloc add_range(base, size); } }; -} // namespace snmalloc \ No newline at end of file +} // namespace snmalloc diff --git a/src/ds/helpers.h b/src/ds/helpers.h index dd4d557d2..f599aacb8 100644 --- a/src/ds/helpers.h +++ b/src/ds/helpers.h @@ -471,4 +471,10 @@ namespace snmalloc return buffer.data(); } }; + + /** + * Convenience type that has no fields / methods. + */ + struct Empty + {}; } // namespace snmalloc diff --git a/src/ds/redblacktree.h b/src/ds/redblacktree.h index 10eb55f58..3d87f9e3f 100644 --- a/src/ds/redblacktree.h +++ b/src/ds/redblacktree.h @@ -11,16 +11,40 @@ namespace snmalloc { #ifdef __cpp_concepts + /** + * The representation must define two types. `Contents` defines some + * identifier that can be mapped to a node as a value type. `Handle` defines + * a reference to the storage, which can be used to update it. + * + * Conceptually, `Contents` is a node ID and `Handle` is a pointer to a node + * ID. + */ template concept RBRepTypes = requires() { - typename Rep::Holder; + typename Rep::Handle; typename Rep::Contents; }; + /** + * The representation must define operations on the holder and contents + * types. It must be able to 'dereference' a holder with `get`, assign to it + * with `set`, set and query the red/black colour of a node with `set_red` and + * `is_red`. + * + * The `ref` method provides uniform access to the children of a node, + * returning a holder pointing to either the left or right child, depending on + * the direction parameter. + * + * The backend must also provide two constant values. + * `Rep::null` defines a value that, if returned from `get`, indicates a null + * value. `Rep::root` defines a value that, if constructed directly, indicates + * a null value and can therefore be used as the initial raw bit pattern of + * the root node. + */ template concept RBRepMethods = - requires(typename Rep::Holder* hp, typename Rep::Contents k, bool b) + requires(typename Rep::Handle hp, typename Rep::Contents k, bool b) { { Rep::get(hp) @@ -41,7 +65,20 @@ namespace snmalloc { Rep::ref(b, k) } - ->ConceptSame; + ->ConceptSame; + { + Rep::null + } + ->ConceptSameModRef; + { + typename Rep::Handle + { + const_cast< + std::remove_const_t>*>( + &Rep::root) + } + } + ->ConceptSame; }; template @@ -70,33 +107,44 @@ namespace snmalloc bool TRACE = false> class RBTree { - using H = typename Rep::Holder; + using H = typename Rep::Handle; using K = typename Rep::Contents; // Container that behaves like a C++ Ref type to enable assignment // to treat left, right and root uniformly. class ChildRef { - H* ptr; + H ptr; public: ChildRef() : ptr(nullptr) {} - ChildRef(H& p) : ptr(&p) {} + ChildRef(H p) : ptr(p) {} + + ChildRef(const ChildRef& other) = default; operator K() { return Rep::get(ptr); } - ChildRef& operator=(const K& t) + ChildRef& operator=(const ChildRef& other) = default; + + ChildRef& operator=(const K t) { // Use representations assigment, so we update the correct bits - // color and other things way also be stored in the Holder. + // color and other things way also be stored in the Handle. Rep::set(ptr, t); return *this; } + /** + * Comparison operators. Note that these are nominal comparisons: + * they compare the identities of the references rather than the values + * referenced. + * comparison of the values held in these child references. + * @{ + */ bool operator==(const ChildRef t) const { return ptr == t.ptr; @@ -106,20 +154,26 @@ namespace snmalloc { return ptr != t.ptr; } + ///@} - H* addr() + bool is_null() { - return ptr; + return Rep::get(ptr) == Rep::null; } - bool is_null() + /** + * Return the reference in some printable format defined by the + * representation. + */ + auto printable() { - return Rep::get(ptr) == Rep::null; + return Rep::printable(ptr); } }; // Root field of the tree - H root{}; + typename std::remove_const_t> + root{Rep::root}; static ChildRef get_dir(bool direction, K k) { @@ -128,7 +182,7 @@ namespace snmalloc ChildRef get_root() { - return {root}; + return {H{&root}}; } void invariant() @@ -194,6 +248,23 @@ namespace snmalloc { ChildRef node; bool dir = false; + + /** + * Update the step to point to a new node and direction. + */ + void set(ChildRef r, bool direction) + { + node = r; + dir = direction; + } + + /** + * Update the step to point to a new node and direction. + */ + void set(typename Rep::Handle r, bool direction) + { + set(ChildRef(r), direction); + } }; public: @@ -207,9 +278,9 @@ namespace snmalloc std::array path; size_t length = 0; - RBPath(typename Rep::Holder& root) : path{} + RBPath(typename Rep::Handle root) : path{} { - path[0] = {root, false}; + path[0].set(root, false); length = 1; } @@ -258,7 +329,7 @@ namespace snmalloc auto next = get_dir(direction, curr()); if (next.is_null()) return false; - path[length] = {next, direction}; + path[length].set(next, direction); length++; return true; } @@ -269,7 +340,7 @@ namespace snmalloc bool move_inc_null(bool direction) { auto next = get_dir(direction, curr()); - path[length] = {next, direction}; + path[length].set(next, direction); length++; return !(next.is_null()); } @@ -319,8 +390,8 @@ namespace snmalloc { message<1024>( " -> {} @ {} ({})", - K(path[i].node), - path[i].node.addr(), + Rep::printable(K(path[i].node)), + path[i].node.printable(), path[i].dir); } } @@ -337,7 +408,7 @@ namespace snmalloc { if constexpr (TRACE) { - message<100>("-------"); + message<100>("------- {}", Rep::name()); message<1024>(msg); path.print(); print(base); @@ -378,11 +449,11 @@ namespace snmalloc "{}\\_{}{}{}@{} ({})", indent, colour, - curr, + Rep::printable((K(curr))), reset, - curr.addr(), + curr.printable(), depth); - if ((get_dir(true, curr) != 0) || (get_dir(false, curr) != 0)) + if (!(get_dir(true, curr).is_null() && get_dir(false, curr).is_null())) { auto s_indent = std::string(indent); print(get_dir(true, curr), (s_indent + "|").c_str(), depth + 1); @@ -465,7 +536,7 @@ namespace snmalloc // we are searching for nearby red elements so we can rotate the tree to // rebalance. The following slides nicely cover the case analysis below // https://www.cs.purdue.edu/homes/ayg/CS251/slides/chap13c.pdf - while (path.curr() != ChildRef(root)) + while (path.curr() != ChildRef(H{&root})) { K parent = path.parent(); bool cur_dir = path.curr_dir(); @@ -574,7 +645,7 @@ namespace snmalloc } // Insert an element at the given path. - void insert_path(RBPath path, K value) + void insert_path(RBPath& path, K value) { SNMALLOC_ASSERT(path.curr().is_null()); path.curr() = value; @@ -704,7 +775,7 @@ namespace snmalloc RBPath get_root_path() { - return RBPath(root); + return RBPath(H{&root}); } }; } // namespace snmalloc diff --git a/src/ds/seqset.h b/src/ds/seqset.h index da5eb46ab..510961c32 100644 --- a/src/ds/seqset.h +++ b/src/ds/seqset.h @@ -20,12 +20,32 @@ namespace snmalloc template class SeqSet { + /** + * This sequence structure is intrusive, in that it requires the use of a + * `next` field in the elements it manages, but, unlike some other intrusive + * designs, it does not require the use of a `container_of`-like construct, + * because its pointers point to the element, not merely the intrusive + * member. + * + * In some cases, the next pointer is provided by a superclass but the list + * is templated over the subclass. The `SeqSet` enforces the invariant that + * only instances of the subclass can be added to the list and so can safely + * down-cast the type of `.next` to `T*`. As such, we require only that the + * `next` field is a pointer to `T` or some superclass of `T`. + * %{ + */ + using NextPtr = decltype(std::declval().next); + static_assert( + std::is_base_of_v, T>, + "T->next must be a queue pointer to T"); + ///@} + /** * Field representation for Fifo behaviour. */ struct FieldFifo { - T* head{nullptr}; + NextPtr head{nullptr}; }; /** @@ -33,14 +53,10 @@ namespace snmalloc */ struct FieldLifo { - T* head{nullptr}; - T** end{&head}; + NextPtr head{nullptr}; + NextPtr* end{&head}; }; - static_assert( - std::is_same::value, - "T->next must be a queue pointer to T"); - /** * Field indirection to actual representation. * Different numbers of fields are required for the @@ -90,7 +106,10 @@ namespace snmalloc else v.head = v.head->next; } - return result; + // This cast is safe if the ->next pointers in all of the objects in the + // list are managed by this class because object types are checked on + // insertion. + return static_cast(result); } /** @@ -107,7 +126,7 @@ namespace snmalloc if (is_empty()) return; - T** prev = &(v.head); + NextPtr* prev = &(v.head); while (true) { @@ -117,11 +136,11 @@ namespace snmalloc break; } - T* curr = *prev; + NextPtr curr = *prev; // Note must read curr->next before calling `f` as `f` is allowed to // mutate that field. - T* next = curr->next; - if (f(curr)) + NextPtr next = curr->next; + if (f(static_cast(curr))) { // Remove element; *prev = next; @@ -165,7 +184,7 @@ namespace snmalloc */ SNMALLOC_FAST_PATH const T* peek() { - return v.head; + return static_cast(v.head); } }; } // namespace snmalloc diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 40e08a386..a1fc5c5e4 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -3,7 +3,7 @@ #include "../ds/defines.h" #include "allocconfig.h" #include "localcache.h" -#include "metaslab.h" +#include "metadata.h" #include "pool.h" #include "remotecache.h" #include "sizeclasstable.h" @@ -11,16 +11,6 @@ namespace snmalloc { - /** - * Empty class used as the superclass for `CoreAllocator` when it does not - * opt into pool allocation. This class exists because `std::conditional` - * (or other equivalent features in C++) can choose between options for - * superclasses but they cannot choose whether a class has a superclass. - * Setting the superclass to an empty class is equivalent to no superclass. - */ - class NotPoolAllocated - {}; - /** * The core, stateful, part of a memory allocator. Each `LocalAllocator` * owns one `CoreAllocator` once it is initialised. @@ -43,19 +33,39 @@ namespace snmalloc * provided externally, then it must be set explicitly with * `init_message_queue`. */ - template + template class CoreAllocator : public std::conditional_t< - SharedStateHandle::Options.CoreAllocIsPoolAllocated, - Pooled>, - NotPoolAllocated> + Backend::Options.CoreAllocIsPoolAllocated, + Pooled>, + Empty> { - template + template friend class LocalAllocator; + /** + * Define local names for specialised versions of various types that are + * specialised for the back-end that we are using. + * @{ + */ + using BackendSlabMetadata = typename Backend::SlabMetadata; + using PagemapEntry = typename Backend::Pagemap::Entry; + /// }@ + /** * Per size class list of active slabs for this allocator. */ - MetaslabCache alloc_classes[NUM_SMALL_SIZECLASSES]; + struct SlabMetadataCache + { +#ifdef SNMALLOC_CHECK_CLIENT + SeqSet available; +#else + // This is slightly faster in some cases, + // but makes memory reuse more predictable. + SeqSet available; +#endif + uint16_t unused = 0; + uint16_t length = 0; + } alloc_classes[NUM_SMALL_SIZECLASSES]; /** * Local entropy source and current version of keys for @@ -68,7 +78,7 @@ namespace snmalloc * allocator */ std::conditional_t< - SharedStateHandle::Options.IsQueueInline, + Backend::Options.IsQueueInline, RemoteAllocator, RemoteAllocator*> remote_alloc; @@ -76,7 +86,7 @@ namespace snmalloc /** * The type used local state. This is defined by the back end. */ - using LocalState = typename SharedStateHandle::LocalState; + using LocalState = typename Backend::LocalState; /** * A local area of address space managed by this allocator. @@ -85,7 +95,7 @@ namespace snmalloc * externally. */ std::conditional_t< - SharedStateHandle::Options.CoreAllocOwnsLocalState, + Backend::Options.CoreAllocOwnsLocalState, LocalState, LocalState*> backend_state; @@ -99,7 +109,7 @@ namespace snmalloc /** * Ticker to query the clock regularly at a lower cost. */ - Ticker ticker; + Ticker ticker; /** * The message queue needs to be accessible from other threads @@ -109,7 +119,7 @@ namespace snmalloc */ auto* public_state() { - if constexpr (SharedStateHandle::Options.IsQueueInline) + if constexpr (Backend::Options.IsQueueInline) { return &remote_alloc; } @@ -124,7 +134,7 @@ namespace snmalloc */ LocalState* backend_state_ptr() { - if constexpr (SharedStateHandle::Options.CoreAllocOwnsLocalState) + if constexpr (Backend::Options.CoreAllocOwnsLocalState) { return &backend_state; } @@ -186,10 +196,10 @@ namespace snmalloc SNMALLOC_ASSERT(attached_cache != nullptr); auto domesticate = [this](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { - return capptr_domesticate(backend_state_ptr(), p); + return capptr_domesticate(backend_state_ptr(), p); }; // Use attached cache, and fill it if it is empty. - return attached_cache->template alloc( + return attached_cache->template alloc( domesticate, size, [&](smallsizeclass_t sizeclass, freelist::Iter<>* fl) { @@ -199,7 +209,7 @@ namespace snmalloc static SNMALLOC_FAST_PATH void alloc_new_list( capptr::Chunk& bumpptr, - Metaslab* meta, + BackendSlabMetadata* meta, size_t rsize, size_t slab_size, LocalEntropy& entropy) @@ -281,17 +291,18 @@ namespace snmalloc bumpptr = slab_end; } - void clear_slab(Metaslab* meta, smallsizeclass_t sizeclass) + capptr::Alloc + clear_slab(BackendSlabMetadata* meta, smallsizeclass_t sizeclass) { auto& key = entropy.get_free_list_key(); freelist::Iter<> fl; auto more = meta->free_queue.close(fl, key); UNUSED(more); auto local_state = backend_state_ptr(); - auto domesticate = - [local_state](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { - return capptr_domesticate(local_state, p); - }; + auto domesticate = [local_state](freelist::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(local_state, p); + }; capptr::Alloc p = finish_alloc_no_zero(fl.take(key, domesticate), sizeclass); @@ -328,17 +339,13 @@ namespace snmalloc auto start_of_slab = pointer_align_down( p, snmalloc::sizeclass_to_slab_size(sizeclass)); - SNMALLOC_ASSERT( - address_cast(start_of_slab) == meta->meta_common.chunk_address()); - #if defined(__CHERI_PURE_CAPABILITY__) && !defined(SNMALLOC_CHECK_CLIENT) // Zero the whole slab. For CHERI we at least need to clear the freelist // pointers to avoid leaking capabilities but we do not need to do it in // the freelist order as for SNMALLOC_CHECK_CLIENT. Zeroing the whole slab // may be more friendly to hw because it does not involve pointer chasing // and is amenable to prefetching. - meta->meta_common.template zero_chunk( - snmalloc::sizeclass_to_slab_size(sizeclass)); + // FIXME: This should be a back-end method guarded on a feature flag. #endif #ifdef SNMALLOC_TRACING @@ -346,21 +353,18 @@ namespace snmalloc "Slab {} is unused, Object sizeclass {}", start_of_slab.unsafe_ptr(), sizeclass); -#else - UNUSED(start_of_slab); #endif + return start_of_slab; } template SNMALLOC_SLOW_PATH void dealloc_local_slabs(smallsizeclass_t sizeclass) { // Return unused slabs of sizeclass_t back to global allocator - alloc_classes[sizeclass].available.filter([this, - sizeclass](Metaslab* meta) { + alloc_classes[sizeclass].available.filter([this, sizeclass](auto* meta) { auto domesticate = [this](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { - auto res = - capptr_domesticate(backend_state_ptr(), p); + auto res = capptr_domesticate(backend_state_ptr(), p); #ifdef SNMALLOC_TRACING if (res.unsafe_ptr() != p.unsafe_ptr()) printf( @@ -383,11 +387,12 @@ namespace snmalloc // TODO delay the clear to the next user of the slab, or teardown so // don't touch the cache lines at this point in snmalloc_check_client. - clear_slab(meta, sizeclass); + auto start = clear_slab(meta, sizeclass); - SharedStateHandle::dealloc_chunk( + Backend::dealloc_chunk( get_backend_local_state(), - meta->meta_common, + *meta, + start, sizeclass_to_slab_size(sizeclass)); return true; @@ -400,17 +405,17 @@ namespace snmalloc * by this thread, or handling the final deallocation onto a slab, * so it can be reused by other threads. */ - SNMALLOC_SLOW_PATH void dealloc_local_object_slow(const MetaEntry& entry) + SNMALLOC_SLOW_PATH void + dealloc_local_object_slow(capptr::Alloc p, const PagemapEntry& entry) { // TODO: Handle message queue on this path? - Metaslab* meta = FrontendMetaEntry::get_metaslab(entry); + auto* meta = entry.get_slab_metadata(); if (meta->is_large()) { // Handle large deallocation here. - size_t entry_sizeclass = - FrontendMetaEntry::get_sizeclass(entry).as_large(); + size_t entry_sizeclass = entry.get_sizeclass().as_large(); size_t size = bits::one_at_bit(entry_sizeclass); #ifdef SNMALLOC_TRACING @@ -419,14 +424,12 @@ namespace snmalloc UNUSED(size); #endif - SharedStateHandle::dealloc_chunk( - get_backend_local_state(), meta->meta_common, size); + Backend::dealloc_chunk(get_backend_local_state(), *meta, p, size); return; } - smallsizeclass_t sizeclass = - FrontendMetaEntry::get_sizeclass(entry).as_small(); + smallsizeclass_t sizeclass = entry.get_sizeclass().as_small(); UNUSED(entropy); if (meta->is_sleeping()) @@ -479,25 +482,25 @@ namespace snmalloc { bool need_post = false; auto local_state = backend_state_ptr(); - auto domesticate = - [local_state](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { - return capptr_domesticate(local_state, p); - }; + auto domesticate = [local_state](freelist::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(local_state, p); + }; auto cb = [this, &need_post](freelist::HeadPtr msg) SNMALLOC_FAST_PATH_LAMBDA { #ifdef SNMALLOC_TRACING message<1024>("Handling remote"); #endif - auto& entry = SharedStateHandle::Pagemap::template get_metaentry( - snmalloc::address_cast(msg)); + auto& entry = + Backend::Pagemap::template get_metaentry(snmalloc::address_cast(msg)); handle_dealloc_remote(entry, msg.as_void(), need_post); return true; }; - if constexpr (SharedStateHandle::Options.QueueHeadsAreTame) + if constexpr (Backend::Options.QueueHeadsAreTame) { /* * The front of the queue has already been validated; just change the @@ -529,7 +532,7 @@ namespace snmalloc * need_post will be set to true, if capacity is exceeded. */ void handle_dealloc_remote( - const MetaEntry& entry, + const PagemapEntry& entry, CapPtr p, bool& need_post) { @@ -537,14 +540,13 @@ namespace snmalloc // TODO this needs to not double revoke if using MTE // TODO thread capabilities? - if (SNMALLOC_LIKELY( - FrontendMetaEntry::get_remote(entry) == public_state())) + if (SNMALLOC_LIKELY(entry.get_remote() == public_state())) { if (SNMALLOC_LIKELY( dealloc_local_object_fast(entry, p.as_void(), entropy))) return; - dealloc_local_object_slow(entry); + dealloc_local_object_slow(p, entry); } else { @@ -554,9 +556,7 @@ namespace snmalloc need_post = true; attached_cache->remote_dealloc_cache .template dealloc( - FrontendMetaEntry::get_remote(entry)->trunc_id(), - p.as_void(), - key_global); + entry.get_remote()->trunc_id(), p.as_void(), key_global); } } @@ -572,12 +572,12 @@ namespace snmalloc // Entropy must be first, so that all data-structures can use the key // it generates. // This must occur before any freelists are constructed. - entropy.init(); + entropy.init(); // Ignoring stats for now. // stats().start(); - if constexpr (SharedStateHandle::Options.IsQueueInline) + if constexpr (Backend::Options.IsQueueInline) { init_message_queue(); message_queue().invariant(); @@ -607,7 +607,7 @@ namespace snmalloc * SFINAE disabled if the allocator does not own the local state. */ template< - typename Config = SharedStateHandle, + typename Config = Backend, typename = std::enable_if_t> CoreAllocator(LocalCache* cache) : attached_cache(cache) { @@ -619,7 +619,7 @@ namespace snmalloc * state. SFINAE disabled if the allocator does own the local state. */ template< - typename Config = SharedStateHandle, + typename Config = Backend, typename = std::enable_if_t> CoreAllocator(LocalCache* cache, LocalState* backend = nullptr) : backend_state(backend), attached_cache(cache) @@ -631,7 +631,7 @@ namespace snmalloc * If the message queue is not inline, provide it. This will then * configure the message queue for use. */ - template + template std::enable_if_t init_message_queue(RemoteAllocator* q) { remote_alloc = q; @@ -650,7 +650,7 @@ namespace snmalloc // stats().remote_post(); // TODO queue not in line! bool sent_something = attached_cache->remote_dealloc_cache - .post( + .post( backend_state_ptr(), public_state()->trunc_id(), key_global); return sent_something; @@ -672,27 +672,27 @@ namespace snmalloc SNMALLOC_FAST_PATH void dealloc_local_object(CapPtr p) { - // MetaEntry-s seen here are expected to have meaningful Remote pointers - auto& entry = SharedStateHandle::Pagemap::template get_metaentry( - snmalloc::address_cast(p)); + // PagemapEntry-s seen here are expected to have meaningful Remote + // pointers + auto& entry = + Backend::Pagemap::template get_metaentry(snmalloc::address_cast(p)); if (SNMALLOC_LIKELY(dealloc_local_object_fast(entry, p, entropy))) return; - dealloc_local_object_slow(entry); + dealloc_local_object_slow(p, entry); } SNMALLOC_FAST_PATH static bool dealloc_local_object_fast( - const MetaEntry& entry, + const PagemapEntry& entry, CapPtr p, LocalEntropy& entropy) { - auto meta = FrontendMetaEntry::get_metaslab(entry); + auto meta = entry.get_slab_metadata(); SNMALLOC_ASSERT(!meta->is_unused()); snmalloc_check_client( - is_start_of_object( - FrontendMetaEntry::get_sizeclass(entry), address_cast(p)), + is_start_of_object(entry.get_sizeclass(), address_cast(p)), "Not deallocating start of an object"); auto cp = p.as_static>(); @@ -734,11 +734,11 @@ namespace snmalloc if (meta->needed() == 0) alloc_classes[sizeclass].unused--; - auto domesticate = [this]( - freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { - return capptr_domesticate(backend_state_ptr(), p); - }; - auto [p, still_active] = Metaslab::alloc_free_list( + auto domesticate = + [this](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(backend_state_ptr(), p); + }; + auto [p, still_active] = BackendSlabMetadata::alloc_free_list( domesticate, meta, fast_free_list, entropy, sizeclass); if (still_active) @@ -747,7 +747,7 @@ namespace snmalloc sl.insert(meta); } - auto r = finish_alloc(p, sizeclass); + auto r = finish_alloc(p, sizeclass); return ticker.check_tick(r); } return small_alloc_slow(sizeclass, fast_free_list); @@ -760,7 +760,7 @@ namespace snmalloc SNMALLOC_FAST_PATH LocalState& get_backend_local_state() { - if constexpr (SharedStateHandle::Options.CoreAllocOwnsLocalState) + if constexpr (Backend::Options.CoreAllocOwnsLocalState) { return backend_state; } @@ -784,10 +784,10 @@ namespace snmalloc message<1024>("small_alloc_slow rsize={} slab size={}", rsize, slab_size); #endif - auto [slab, meta] = SharedStateHandle::alloc_chunk( + auto [slab, meta] = Backend::alloc_chunk( get_backend_local_state(), slab_size, - FrontendMetaEntry::encode( + PagemapEntry::encode( public_state(), sizeclass_t::from_small_class(sizeclass))); if (slab == nullptr) @@ -803,9 +803,9 @@ namespace snmalloc auto domesticate = [this](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { - return capptr_domesticate(backend_state_ptr(), p); + return capptr_domesticate(backend_state_ptr(), p); }; - auto [p, still_active] = Metaslab::alloc_free_list( + auto [p, still_active] = BackendSlabMetadata::alloc_free_list( domesticate, meta, fast_free_list, entropy, sizeclass); if (still_active) @@ -814,7 +814,7 @@ namespace snmalloc alloc_classes[sizeclass].available.insert(meta); } - auto r = finish_alloc(p, sizeclass); + auto r = finish_alloc(p, sizeclass); return ticker.check_tick(r); } @@ -827,10 +827,10 @@ namespace snmalloc { SNMALLOC_ASSERT(attached_cache != nullptr); auto local_state = backend_state_ptr(); - auto domesticate = - [local_state](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { - return capptr_domesticate(local_state, p); - }; + auto domesticate = [local_state](freelist::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(local_state, p); + }; if (destroy_queue) { @@ -841,8 +841,8 @@ namespace snmalloc { bool need_post = true; // Always going to post, so ignore. auto n_tame = p_tame->atomic_read_next(key_global, domesticate); - const MetaEntry& entry = SharedStateHandle::Pagemap::get_metaentry( - snmalloc::address_cast(p_tame)); + const PagemapEntry& entry = + Backend::Pagemap::get_metaentry(snmalloc::address_cast(p_tame)); handle_dealloc_remote(entry, p_tame.as_void(), need_post); p_tame = n_tame; } @@ -855,10 +855,9 @@ namespace snmalloc handle_message_queue([]() {}); } - auto posted = - attached_cache->flush( - backend_state_ptr(), - [&](capptr::Alloc p) { dealloc_local_object(p); }); + auto posted = attached_cache->flush( + backend_state_ptr(), + [&](capptr::Alloc p) { dealloc_local_object(p); }); // We may now have unused slabs, return to the global allocator. for (smallsizeclass_t sizeclass = 0; sizeclass < NUM_SMALL_SIZECLASSES; @@ -896,8 +895,8 @@ namespace snmalloc bool debug_is_empty_impl(bool* result) { auto test = [&result](auto& queue) { - queue.filter([&result](auto metaslab) { - if (metaslab->needed() != 0) + queue.filter([&result](auto slab_metadata) { + if (slab_metadata->needed() != 0) { if (result != nullptr) *result = false; @@ -963,9 +962,6 @@ namespace snmalloc /** * Use this alias to access the pool of allocators throughout snmalloc. */ - template - using AllocPool = Pool< - CoreAllocator, - SharedStateHandle, - SharedStateHandle::pool>; + template + using AllocPool = Pool, Backend, Backend::pool>; } // namespace snmalloc diff --git a/src/mem/localalloc.h b/src/mem/localalloc.h index 27604882e..bbe0781db 100644 --- a/src/mem/localalloc.h +++ b/src/mem/localalloc.h @@ -59,20 +59,27 @@ namespace snmalloc * core allocator must be provided externally by invoking the `init` method * on this class *before* any allocation-related methods are called. */ - template + template class LocalAllocator { public: - using StateHandle = SharedStateHandle; + using StateHandle = Backend; private: - using CoreAlloc = CoreAllocator; + /** + * Define local names for specialised versions of various types that are + * specialised for the back-end that we are using. + * @{ + */ + using CoreAlloc = CoreAllocator; + using PagemapEntry = typename Backend::Pagemap::Entry; + /// }@ // Free list per small size class. These are used for // allocation on the fast path. This part of the code is inspired by // mimalloc. // Also contains remote deallocation cache. - LocalCache local_cache{&SharedStateHandle::unused_remote}; + LocalCache local_cache{&Backend::unused_remote}; // Underlying allocator for most non-fast path operations. CoreAlloc* core_alloc{nullptr}; @@ -116,7 +123,7 @@ namespace snmalloc SNMALLOC_SLOW_PATH decltype(auto) lazy_init(Action action, Args... args) { SNMALLOC_ASSERT(core_alloc == nullptr); - if constexpr (!SharedStateHandle::Options.LocalAllocSupportsLazyInit) + if constexpr (!Backend::Options.LocalAllocSupportsLazyInit) { SNMALLOC_CHECK( false && @@ -129,7 +136,7 @@ namespace snmalloc else { // Initialise the thread local allocator - if constexpr (SharedStateHandle::Options.CoreAllocOwnsLocalState) + if constexpr (Backend::Options.CoreAllocOwnsLocalState) { init(); } @@ -141,7 +148,7 @@ namespace snmalloc // Must be called at least once per thread. // A pthread implementation only calls the thread destruction handle // if the key has been set. - SharedStateHandle::register_clean_up(); + Backend::register_clean_up(); // Perform underlying operation auto r = action(core_alloc, args...); @@ -180,10 +187,10 @@ namespace snmalloc return check_init([&](CoreAlloc* core_alloc) { // Grab slab of correct size // Set remote as large allocator remote. - auto [chunk, meta] = SharedStateHandle::alloc_chunk( + auto [chunk, meta] = Backend::alloc_chunk( core_alloc->get_backend_local_state(), large_size_to_chunk_size(size), - FrontendMetaEntry::encode( + PagemapEntry::encode( core_alloc->public_state(), size_to_sizeclass_full(size))); // set up meta data so sizeclass is correct, and hence alloc size, and // external pointer. @@ -197,7 +204,7 @@ namespace snmalloc if (zero_mem == YesZero && chunk.unsafe_ptr() != nullptr) { - SharedStateHandle::Pal::template zero( + Backend::Pal::template zero( chunk.unsafe_ptr(), bits::next_pow2(size)); } @@ -208,11 +215,10 @@ namespace snmalloc template SNMALLOC_FAST_PATH capptr::Alloc small_alloc(size_t size) { - auto domesticate = [this](freelist::QueuePtr p) - SNMALLOC_FAST_PATH_LAMBDA { - return capptr_domesticate( - core_alloc->backend_state_ptr(), p); - }; + auto domesticate = [this]( + freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(core_alloc->backend_state_ptr(), p); + }; auto slowpath = [&]( smallsizeclass_t sizeclass, freelist::Iter<>* fl) SNMALLOC_FAST_PATH_LAMBDA { @@ -236,7 +242,7 @@ namespace snmalloc sizeclass); }; - return local_cache.template alloc( + return local_cache.template alloc( domesticate, size, slowpath); } @@ -267,10 +273,10 @@ namespace snmalloc p.unsafe_ptr(), alloc_size(p.unsafe_ptr())); #endif - const MetaEntry& entry = - SharedStateHandle::Pagemap::get_metaentry(address_cast(p)); + const PagemapEntry& entry = + Backend::Pagemap::get_metaentry(address_cast(p)); local_cache.remote_dealloc_cache.template dealloc( - FrontendMetaEntry::get_remote(entry)->trunc_id(), p, key_global); + entry.get_remote()->trunc_id(), p, key_global); post_remote_cache(); return; } @@ -297,13 +303,13 @@ namespace snmalloc } /** - * Call `SharedStateHandle::is_initialised()` if it is implemented, + * Call `Backend::is_initialised()` if it is implemented, * unconditionally returns true otherwise. */ SNMALLOC_FAST_PATH bool is_initialised() { - return call_is_initialised(nullptr, 0); + return call_is_initialised(nullptr, 0); } /** @@ -326,13 +332,13 @@ namespace snmalloc {} /** - * Call `SharedStateHandle::ensure_init()` if it is implemented, do + * Call `Backend::ensure_init()` if it is implemented, do * nothing otherwise. */ SNMALLOC_FAST_PATH void ensure_init() { - call_ensure_init(nullptr, 0); + call_ensure_init(nullptr, 0); } public: @@ -377,7 +383,7 @@ namespace snmalloc // Initialise the global allocator structures ensure_init(); // Grab an allocator for this thread. - init(AllocPool::acquire(&(this->local_cache))); + init(AllocPool::acquire(&(this->local_cache))); } // Return all state in the fast allocator and release the underlying @@ -397,9 +403,9 @@ namespace snmalloc // Detach underlying allocator core_alloc->attached_cache = nullptr; // Return underlying allocator to the system. - if constexpr (SharedStateHandle::Options.CoreAllocOwnsLocalState) + if constexpr (Backend::Options.CoreAllocOwnsLocalState) { - AllocPool::release(core_alloc); + AllocPool::release(core_alloc); } // Set up thread local allocator to look like @@ -408,7 +414,7 @@ namespace snmalloc #ifdef SNMALLOC_TRACING message<1024>("flush(): core_alloc={}", core_alloc); #endif - local_cache.remote_allocator = &SharedStateHandle::unused_remote; + local_cache.remote_allocator = &Backend::unused_remote; local_cache.remote_dealloc_cache.capacity = 0; } } @@ -621,14 +627,12 @@ namespace snmalloc * well-formedness) of this pointer. The remainder of the logic will * deal with the object's extent. */ - capptr::Alloc p_tame = capptr_domesticate( - core_alloc->backend_state_ptr(), p_wild); + capptr::Alloc p_tame = + capptr_domesticate(core_alloc->backend_state_ptr(), p_wild); - const MetaEntry& entry = - SharedStateHandle::Pagemap::get_metaentry(address_cast(p_tame)); - if (SNMALLOC_LIKELY( - local_cache.remote_allocator == - FrontendMetaEntry::get_remote(entry))) + const PagemapEntry& entry = + Backend::Pagemap::get_metaentry(address_cast(p_tame)); + if (SNMALLOC_LIKELY(local_cache.remote_allocator == entry.get_remote())) { # if defined(__CHERI_PURE_CAPABILITY__) && defined(SNMALLOC_CHECK_CLIENT) dealloc_cheri_checks(p_tame.unsafe_ptr()); @@ -636,11 +640,11 @@ namespace snmalloc if (SNMALLOC_LIKELY(CoreAlloc::dealloc_local_object_fast( entry, p_tame, local_cache.entropy))) return; - core_alloc->dealloc_local_object_slow(entry); + core_alloc->dealloc_local_object_slow(p_tame, entry); return; } - RemoteAllocator* remote = FrontendMetaEntry::get_remote(entry); + RemoteAllocator* remote = entry.get_remote(); if (SNMALLOC_LIKELY(remote != nullptr)) { # if defined(__CHERI_PURE_CAPABILITY__) && defined(SNMALLOC_CHECK_CLIENT) @@ -706,7 +710,7 @@ namespace snmalloc #else // TODO What's the domestication policy here? At the moment we just // probe the pagemap with the raw address, without checks. There could - // be implicit domestication through the `SharedStateHandle::Pagemap` or + // be implicit domestication through the `Backend::Pagemap` or // we could just leave well enough alone. // Note that alloc_size should return 0 for nullptr. @@ -716,10 +720,10 @@ namespace snmalloc // To handle this case we require the uninitialised pagemap contain an // entry for the first chunk of memory, that states it represents a // large object, so we can pull the check for null off the fast path. - const MetaEntry& entry = - SharedStateHandle::Pagemap::get_metaentry(address_cast(p_raw)); + const PagemapEntry& entry = + Backend::Pagemap::get_metaentry(address_cast(p_raw)); - return sizeclass_full_to_size(FrontendMetaEntry::get_sizeclass(entry)); + return sizeclass_full_to_size(entry.get_sizeclass()); #endif } @@ -761,11 +765,10 @@ namespace snmalloc size_t remaining_bytes(const void* p) { #ifndef SNMALLOC_PASS_THROUGH - const MetaEntry& entry = - SharedStateHandle::Pagemap::template get_metaentry( - address_cast(p)); + const PagemapEntry& entry = + Backend::Pagemap::template get_metaentry(address_cast(p)); - auto sizeclass = FrontendMetaEntry::get_sizeclass(entry); + auto sizeclass = entry.get_sizeclass(); return snmalloc::remaining_bytes(sizeclass, address_cast(p)); #else return pointer_diff(p, reinterpret_cast(UINTPTR_MAX)); @@ -774,7 +777,7 @@ namespace snmalloc bool check_bounds(const void* p, size_t s) { - if (SNMALLOC_LIKELY(SharedStateHandle::Pagemap::is_initialised())) + if (SNMALLOC_LIKELY(Backend::Pagemap::is_initialised())) { return remaining_bytes(p) >= s; } @@ -790,11 +793,10 @@ namespace snmalloc size_t index_in_object(const void* p) { #ifndef SNMALLOC_PASS_THROUGH - const MetaEntry& entry = - SharedStateHandle::Pagemap::template get_metaentry( - address_cast(p)); + const PagemapEntry& entry = + Backend::Pagemap::template get_metaentry(address_cast(p)); - auto sizeclass = FrontendMetaEntry::get_sizeclass(entry); + auto sizeclass = entry.get_sizeclass(); return snmalloc::index_in_object(sizeclass, address_cast(p)); #else return reinterpret_cast(p); diff --git a/src/mem/metadata.h b/src/mem/metadata.h new file mode 100644 index 000000000..765e2989f --- /dev/null +++ b/src/mem/metadata.h @@ -0,0 +1,636 @@ +#pragma once + +#include "../backend/backend_concept.h" +#include "../ds/helpers.h" +#include "../ds/seqset.h" +#include "freelist.h" +#include "sizeclasstable.h" + +namespace snmalloc +{ + struct RemoteAllocator; + + /** + * Remotes need to be aligned enough that the bottom bits have enough room for + * all the size classes, both large and small. An additional bit is required + * to separate backend uses. + */ + static constexpr size_t REMOTE_MIN_ALIGN = + bits::max(CACHELINE_SIZE, SIZECLASS_REP_SIZE) << 1; + + /** + * Base class for the templated FrontendMetaEntry. This exists to avoid + * needing a template parameter to access constants that are independent of + * the template parameter and contains all of the state that is agnostic to + * the types used for storing per-slab metadata. This class should never be + * instantiated directly (and its protected constructor guarantees that), + * only the templated subclass should be use. The subclass provides + * convenient accessors. + * + * A back end may also subclass `FrontendMetaEntry` to provide other + * back-end-specific information. The front end never directly instantiates + * these. + */ + class MetaEntryBase + { + protected: + /** + * This bit is set in remote_and_sizeclass to discriminate between the case + * that it is in use by the frontend (0) or by the backend (1). For the + * former case, see other methods on this and the subclass + * `FrontendMetaEntry`; for the latter, see backend/backend.h and + * backend/largebuddyrange.h. + * + * This value is statically checked by the frontend to ensure that its + * bit packing does not conflict; see mem/remoteallocator.h + */ + static constexpr address_t REMOTE_BACKEND_MARKER = 1 << 7; + + /** + * Bit used to indicate this should not be considered part of the previous + * PAL allocation. + * + * Some platforms cannot treat different PalAllocs as a single allocation. + * This is true on CHERI as the combined permission might not be + * representable. It is also true on Windows as you cannot Commit across + * multiple continuous VirtualAllocs. + */ + static constexpr address_t META_BOUNDARY_BIT = 1 << 0; + + /** + * The bit above the sizeclass is always zero unless this is used + * by the backend to represent another datastructure such as the buddy + * allocator entries. + */ + static constexpr size_t REMOTE_WITH_BACKEND_MARKER_ALIGN = + MetaEntryBase::REMOTE_BACKEND_MARKER; + static_assert( + (REMOTE_MIN_ALIGN >> 1) == MetaEntryBase::REMOTE_BACKEND_MARKER); + + /** + * In common cases, the pointer to the slab metadata. See + * docs/AddressSpace.md for additional details. + * + * The bottom bit is used to indicate if this is the first chunk in a PAL + * allocation, that cannot be combined with the preceeding chunk. + */ + uintptr_t meta{0}; + + /** + * In common cases, a bit-packed pointer to the owning allocator (if any), + * and the sizeclass of this chunk. See `encode` for + * details of this case and docs/AddressSpace.md for further details. + */ + uintptr_t remote_and_sizeclass{0}; + + /** + * Constructor from two pointer-sized words. The subclass is responsible + * for ensuring that accesses to these are type-safe. + */ + constexpr MetaEntryBase(uintptr_t m, uintptr_t ras) + : meta(m), remote_and_sizeclass(ras) + {} + + /** + * Default constructor, zero initialises. + */ + constexpr MetaEntryBase() : MetaEntryBase(0, 0) {} + + /** + * When a meta entry is in use by the back end, it exposes two words of + * state. The low bits in both are reserved. Bits in this bitmask must + * not be set by the back end in either word. + * + * During a major release, this constraint may be weakened, allowing the + * back end to set more bits. We don't currently use all of these bits in + * both words, but we reserve them all to make access uniform. If more + * bits are required by a back end then we could make this asymmetric. + * + * `REMOTE_BACKEND_MARKER` is the highest bit that we reserve, so this is + * currently every bit including that bit and all lower bits. + */ + static constexpr address_t BACKEND_RESERVED_MASK = + (REMOTE_BACKEND_MARKER << 1) - 1; + + public: + /** + * Does the back end currently own this entry? Note that freshly + * allocated entries are owned by the front end until explicitly + * claimed by the back end and so this will return `false` if neither + * the front nor back end owns this entry. + */ + [[nodiscard]] bool is_backend_owned() const + { + return (REMOTE_BACKEND_MARKER & remote_and_sizeclass) == + REMOTE_BACKEND_MARKER; + } + + /** + * Returns true if this metaentry has not been claimed by the front or back + * ends. + */ + [[nodiscard]] bool is_unowned() const + { + return (meta == 0) && (remote_and_sizeclass == 0); + } + + /** + * Encode the remote and the sizeclass. + */ + [[nodiscard]] static SNMALLOC_FAST_PATH uintptr_t + encode(RemoteAllocator* remote, sizeclass_t sizeclass) + { + /* remote might be nullptr; cast to uintptr_t before offsetting */ + return pointer_offset( + reinterpret_cast(remote), sizeclass.raw()); + } + + /** + * Return the remote and sizeclass in an implementation-defined encoding. + * This is not guaranteed to be stable across snmalloc releases and so the + * only safe use for this is to pass it to the two-argument constructor of + * this class. + */ + [[nodiscard]] SNMALLOC_FAST_PATH uintptr_t get_remote_and_sizeclass() const + { + return remote_and_sizeclass; + } + + /** + * Explicit assignment operator, copies the data preserving the boundary bit + * in the target if it is set. + */ + MetaEntryBase& operator=(const MetaEntryBase& other) + { + // Don't overwrite the boundary bit with the other's + meta = (other.meta & ~META_BOUNDARY_BIT) | + address_cast(meta & META_BOUNDARY_BIT); + remote_and_sizeclass = other.remote_and_sizeclass; + return *this; + } + + /** + * On some platforms, allocations originating from the OS may not be + * combined. The boundary bit indicates whether this is meta entry + * corresponds to the first chunk in such a range and so may not be combined + * with anything before it in the address space. + * @{ + */ + void set_boundary() + { + meta |= META_BOUNDARY_BIT; + } + + [[nodiscard]] bool is_boundary() const + { + return meta & META_BOUNDARY_BIT; + } + + bool clear_boundary_bit() + { + return meta &= ~META_BOUNDARY_BIT; + } + ///@} + + /** + * Returns the remote. + * + * If the meta entry is owned by the back end then this returns an + * undefined value and will abort in debug builds. + */ + [[nodiscard]] SNMALLOC_FAST_PATH RemoteAllocator* get_remote() const + { + SNMALLOC_ASSERT(!is_backend_owned()); + return reinterpret_cast( + pointer_align_down( + get_remote_and_sizeclass())); + } + + /** + * Return the sizeclass. + * + * This can be called irrespective of whether the corresponding meta entry + * is owned by the front or back end (and is, for example, called by + * `external_pointer`). In the future, it may provide some stronger + * guarantees on the value that is returned in this case. + */ + [[nodiscard]] SNMALLOC_FAST_PATH sizeclass_t get_sizeclass() const + { + // TODO: perhaps remove static_cast with resolution of + // https://github.com/CTSRD-CHERI/llvm-project/issues/588 + return sizeclass_t::from_raw( + static_cast(get_remote_and_sizeclass()) & + (REMOTE_WITH_BACKEND_MARKER_ALIGN - 1)); + } + + /** + * Claim the meta entry for use by the back end. This preserves the + * boundary bit, if it is set, but otherwise resets the meta entry to a + * pristine state. + */ + void claim_for_backend() + { + meta = is_boundary() ? META_BOUNDARY_BIT : 0; + remote_and_sizeclass = REMOTE_BACKEND_MARKER; + } + + /** + * When used by the back end, the two words in a meta entry have no + * semantics defined by the front end and are identified by enumeration + * values. + */ + enum class Word + { + /** + * The first word. + */ + One, + + /** + * The second word. + */ + Two + }; + + static constexpr bool is_backend_allowed_value(Word, uintptr_t val) + { + return (val & BACKEND_RESERVED_MASK) == 0; + } + + /** + * Proxy class that allows setting and reading back the bits in each word + * that are exposed for the back end. + * + * The back end must not keep instances of this class after returning the + * corresponding meta entry to the front end. + */ + class BackendStateWordRef + { + /** + * A pointer to the relevant word. + */ + uintptr_t* val; + + public: + /** + * Constructor, wraps a `uintptr_t`. Note that this may be used outside + * of the meta entry by code wishing to provide uniform storage to things + * that are either in a meta entry or elsewhere. + */ + constexpr BackendStateWordRef(uintptr_t* v) : val(v) {} + + /** + * Copy constructor. Aliases the underlying storage. Note that this is + * not thread safe: two `BackendStateWordRef` instances sharing access to + * the same storage must not be used from different threads without + * explicit synchronisation. + */ + constexpr BackendStateWordRef(const BackendStateWordRef& other) = default; + + /** + * Read the value. This zeroes any bits in the underlying storage that + * the back end is not permitted to access. + */ + [[nodiscard]] uintptr_t get() const + { + return (*val) & ~BACKEND_RESERVED_MASK; + } + + /** + * Default copy assignment. See the copy constructor for constraints on + * using this. + */ + BackendStateWordRef& + operator=(const BackendStateWordRef& other) = default; + + /** + * Assignment operator. Zeroes the bits in the provided value that the + * back end is not permitted to use and then stores the result in the + * value that this class manages. + */ + BackendStateWordRef& operator=(uintptr_t v) + { + SNMALLOC_ASSERT_MSG( + ((v & BACKEND_RESERVED_MASK) == 0), + "The back end is not permitted to use the low bits in the meta " + "entry. ({} & {}) == {}.", + v, + BACKEND_RESERVED_MASK, + (v & BACKEND_RESERVED_MASK)); + *val = v | (static_cast(*val) & BACKEND_RESERVED_MASK); + return *this; + } + + /** + * Comparison operator. Performs address comparison *not* value + * comparison. + */ + bool operator!=(const BackendStateWordRef& other) const + { + return val != other.val; + } + + /** + * Returns the address of the underlying storage in a form that can be + * passed to `snmalloc::message` for printing. + */ + address_t printable_address() + { + return address_cast(val); + } + }; + + /** + * Get a proxy that allows the back end to read from and write to (some bits + * of) a word in the meta entry. The meta entry must either be unowned or + * explicitly claimed by the back end before calling this. + */ + BackendStateWordRef get_backend_word(Word w) + { + if (!is_backend_owned()) + { + SNMALLOC_ASSERT_MSG( + is_unowned(), + "Meta entry is owned by the front end. Meta: {}, " + "remote_and_sizeclass:{}", + meta, + remote_and_sizeclass); + claim_for_backend(); + } + return {w == Word::One ? &meta : &remote_and_sizeclass}; + } + }; + + /** + * The FrontendSlabMetadata represent the metadata associated with a single + * slab. + */ + class alignas(CACHELINE_SIZE) FrontendSlabMetadata + { + public: + /** + * Used to link slab metadata together in various other data-structures. + * This is intended to be used with `SeqSet` and so may actually hold a + * subclass of this class provided by the back end. The `SeqSet` is + * responsible for maintaining that invariant. While an instance of this + * class is in a `SeqSet`, the `next` field should not be assigned to by + * anything that doesn't enforce the invariant that `next` stores a `T*`, + * where `T` is a subclass of `FrontendSlabMetadata`. + */ + FrontendSlabMetadata* next{nullptr}; + + constexpr FrontendSlabMetadata() = default; + + /** + * Data-structure for building the free list for this slab. + */ +#ifdef SNMALLOC_CHECK_CLIENT + freelist::Builder free_queue; +#else + freelist::Builder free_queue; +#endif + + /** + * The number of deallocation required until we hit a slow path. This + * counts down in two different ways that are handled the same on the + * fast path. The first is + * - deallocations until the slab has sufficient entries to be considered + * useful to allocate from. This could be as low as 1, or when we have + * a requirement for entropy then it could be much higher. + * - deallocations until the slab is completely unused. This is needed + * to be detected, so that the statistics can be kept up to date, and + * potentially return memory to the a global pool of slabs/chunks. + */ + uint16_t needed_ = 0; + + /** + * Flag that is used to indicate that the slab is currently not active. + * I.e. it is not in a CoreAllocator cache for the appropriate sizeclass. + */ + bool sleeping_ = false; + + /** + * Flag to indicate this is actually a large allocation rather than a slab + * of small allocations. + */ + bool large_ = false; + + uint16_t& needed() + { + return needed_; + } + + bool& sleeping() + { + return sleeping_; + } + + /** + * Initialise FrontendSlabMetadata for a slab. + */ + void initialise(smallsizeclass_t sizeclass) + { + free_queue.init(); + // Set up meta data as if the entire slab has been turned into a free + // list. This means we don't have to check for special cases where we have + // returned all the elements, but this is a slab that is still being bump + // allocated from. Hence, the bump allocator slab will never be returned + // for use in another size class. + set_sleeping(sizeclass, 0); + + large_ = false; + } + + /** + * Make this a chunk represent a large allocation. + * + * Set needed so immediately moves to slow path. + */ + void initialise_large() + { + // We will push to this just to make the fast path clean. + free_queue.init(); + + // Flag to detect that it is a large alloc on the slow path + large_ = true; + + // Jump to slow path on first deallocation. + needed() = 1; + } + + /** + * Updates statistics for adding an entry to the free list, if the + * slab is either + * - empty adding the entry to the free list, or + * - was full before the subtraction + * this returns true, otherwise returns false. + */ + bool return_object() + { + return (--needed()) == 0; + } + + bool is_unused() + { + return needed() == 0; + } + + bool is_sleeping() + { + return sleeping(); + } + + bool is_large() + { + return large_; + } + + /** + * Try to set this slab metadata to sleep. If the remaining elements are + * fewer than the threshold, then it will actually be set to the sleeping + * state, and will return true, otherwise it will return false. + */ + SNMALLOC_FAST_PATH bool + set_sleeping(smallsizeclass_t sizeclass, uint16_t remaining) + { + auto threshold = threshold_for_waking_slab(sizeclass); + if (remaining >= threshold) + { + // Set needed to at least one, possibly more so we only use + // a slab when it has a reasonable amount of free elements + auto allocated = sizeclass_to_slab_object_count(sizeclass); + needed() = allocated - remaining; + sleeping() = false; + return false; + } + + sleeping() = true; + needed() = threshold - remaining; + return true; + } + + SNMALLOC_FAST_PATH void set_not_sleeping(smallsizeclass_t sizeclass) + { + auto allocated = sizeclass_to_slab_object_count(sizeclass); + needed() = allocated - threshold_for_waking_slab(sizeclass); + + // Design ensures we can't move from full to empty. + // There are always some more elements to free at this + // point. This is because the threshold is always less + // than the count for the slab + SNMALLOC_ASSERT(needed() != 0); + + sleeping() = false; + } + + /** + * Allocates a free list from the meta data. + * + * Returns a freshly allocated object of the correct size, and a bool that + * specifies if the slab metadata should be placed in the queue for that + * sizeclass. + * + * If Randomisation is not used, it will always return false for the second + * component, but with randomisation, it may only return part of the + * available objects for this slab metadata. + */ + template + static SNMALLOC_FAST_PATH std::pair + alloc_free_list( + Domesticator domesticate, + FrontendSlabMetadata* meta, + freelist::Iter<>& fast_free_list, + LocalEntropy& entropy, + smallsizeclass_t sizeclass) + { + auto& key = entropy.get_free_list_key(); + + std::remove_reference_t tmp_fl; + auto remaining = meta->free_queue.close(tmp_fl, key); + auto p = tmp_fl.take(key, domesticate); + fast_free_list = tmp_fl; + +#ifdef SNMALLOC_CHECK_CLIENT + entropy.refresh_bits(); +#else + UNUSED(entropy); +#endif + + // This marks the slab as sleeping, and sets a wakeup + // when sufficient deallocations have occurred to this slab. + // Takes how many deallocations were not grabbed on this call + // This will be zero if there is no randomisation. + auto sleeping = meta->set_sleeping(sizeclass, remaining); + + return {p, !sleeping}; + } + }; + + /** + * Entry stored in the pagemap. See docs/AddressSpace.md for the full + * FrontendMetaEntry lifecycle. + */ + template + class FrontendMetaEntry : public MetaEntryBase + { + /** + * Ensure that the template parameter is valid. + */ + static_assert( + std::is_convertible_v, + "The front end requires that the back end provides slab metadata that is " + "compatible with the front-end's structure"); + + public: + constexpr FrontendMetaEntry() = default; + + /** + * Constructor, provides the remote and sizeclass embedded in a single + * pointer-sized word. This format is not guaranteed to be stable and so + * the second argument of this must always be the return value from + * `get_remote_and_sizeclass`. + */ + SNMALLOC_FAST_PATH + FrontendMetaEntry(BackendSlabMetadata* meta, uintptr_t remote_and_sizeclass) + : MetaEntryBase( + unsafe_to_uintptr(meta), remote_and_sizeclass) + { + SNMALLOC_ASSERT_MSG( + (REMOTE_BACKEND_MARKER & remote_and_sizeclass) == 0, + "Setting a backend-owned value ({}) via the front-end interface is not " + "allowed", + remote_and_sizeclass); + remote_and_sizeclass &= ~REMOTE_BACKEND_MARKER; + } + + /** + * Implicit copying of meta entries is almost certainly a bug and so the + * copy constructor is deleted to statically catch these problems. + */ + FrontendMetaEntry(const FrontendMetaEntry&) = delete; + + /** + * Explicit assignment operator, copies the data preserving the boundary bit + * in the target if it is set. + */ + FrontendMetaEntry& operator=(const FrontendMetaEntry& other) + { + MetaEntryBase::operator=(other); + return *this; + } + + /** + * Return the FrontendSlabMetadata metadata associated with this chunk, + * guarded by an assert that this chunk is being used as a slab (i.e., has + * an associated owning allocator). + */ + [[nodiscard]] SNMALLOC_FAST_PATH BackendSlabMetadata* + get_slab_metadata() const + { + SNMALLOC_ASSERT(get_remote() != nullptr); + return unsafe_from_uintptr( + meta & ~META_BOUNDARY_BIT); + } + }; + +} // namespace snmalloc diff --git a/src/mem/metaslab.h b/src/mem/metaslab.h deleted file mode 100644 index 997784d15..000000000 --- a/src/mem/metaslab.h +++ /dev/null @@ -1,280 +0,0 @@ -#pragma once - -#include "../backend/metatypes.h" -#include "../ds/helpers.h" -#include "../ds/seqset.h" -#include "../mem/remoteallocator.h" -#include "freelist.h" -#include "sizeclasstable.h" - -namespace snmalloc -{ - // The Metaslab represent the status of a single slab. - class alignas(CACHELINE_SIZE) Metaslab - { - public: - MetaCommon meta_common; - - // Used to link metaslabs together in various other data-structures. - Metaslab* next{nullptr}; - - constexpr Metaslab() = default; - - /** - * Data-structure for building the free list for this slab. - */ -#ifdef SNMALLOC_CHECK_CLIENT - freelist::Builder free_queue; -#else - freelist::Builder free_queue; -#endif - - /** - * The number of deallocation required until we hit a slow path. This - * counts down in two different ways that are handled the same on the - * fast path. The first is - * - deallocations until the slab has sufficient entries to be considered - * useful to allocate from. This could be as low as 1, or when we have - * a requirement for entropy then it could be much higher. - * - deallocations until the slab is completely unused. This is needed - * to be detected, so that the statistics can be kept up to date, and - * potentially return memory to the a global pool of slabs/chunks. - */ - uint16_t needed_ = 0; - - /** - * Flag that is used to indicate that the slab is currently not active. - * I.e. it is not in a CoreAllocator cache for the appropriate sizeclass. - */ - bool sleeping_ = false; - - /** - * Flag to indicate this is actually a large allocation rather than a slab - * of small allocations. - */ - bool large_ = false; - - uint16_t& needed() - { - return needed_; - } - - bool& sleeping() - { - return sleeping_; - } - - /** - * Initialise Metaslab for a slab. - */ - void initialise(smallsizeclass_t sizeclass) - { - free_queue.init(); - // Set up meta data as if the entire slab has been turned into a free - // list. This means we don't have to check for special cases where we have - // returned all the elements, but this is a slab that is still being bump - // allocated from. Hence, the bump allocator slab will never be returned - // for use in another size class. - set_sleeping(sizeclass, 0); - - large_ = false; - } - - /** - * Make this a chunk represent a large allocation. - * - * Set needed so immediately moves to slow path. - */ - void initialise_large() - { - // We will push to this just to make the fast path clean. - free_queue.init(); - - // Flag to detect that it is a large alloc on the slow path - large_ = true; - - // Jump to slow path on first deallocation. - needed() = 1; - } - - /** - * Updates statistics for adding an entry to the free list, if the - * slab is either - * - empty adding the entry to the free list, or - * - was full before the subtraction - * this returns true, otherwise returns false. - */ - bool return_object() - { - return (--needed()) == 0; - } - - bool is_unused() - { - return needed() == 0; - } - - bool is_sleeping() - { - return sleeping(); - } - - bool is_large() - { - return large_; - } - - /** - * Try to set this metaslab to sleep. If the remaining elements are fewer - * than the threshold, then it will actually be set to the sleeping state, - * and will return true, otherwise it will return false. - */ - SNMALLOC_FAST_PATH bool - set_sleeping(smallsizeclass_t sizeclass, uint16_t remaining) - { - auto threshold = threshold_for_waking_slab(sizeclass); - if (remaining >= threshold) - { - // Set needed to at least one, possibly more so we only use - // a slab when it has a reasonable amount of free elements - auto allocated = sizeclass_to_slab_object_count(sizeclass); - needed() = allocated - remaining; - sleeping() = false; - return false; - } - - sleeping() = true; - needed() = threshold - remaining; - return true; - } - - SNMALLOC_FAST_PATH void set_not_sleeping(smallsizeclass_t sizeclass) - { - auto allocated = sizeclass_to_slab_object_count(sizeclass); - needed() = allocated - threshold_for_waking_slab(sizeclass); - - // Design ensures we can't move from full to empty. - // There are always some more elements to free at this - // point. This is because the threshold is always less - // than the count for the slab - SNMALLOC_ASSERT(needed() != 0); - - sleeping() = false; - } - - /** - * Allocates a free list from the meta data. - * - * Returns a freshly allocated object of the correct size, and a bool that - * specifies if the metaslab should be placed in the queue for that - * sizeclass. - * - * If Randomisation is not used, it will always return false for the second - * component, but with randomisation, it may only return part of the - * available objects for this metaslab. - */ - template - static SNMALLOC_FAST_PATH std::pair - alloc_free_list( - Domesticator domesticate, - Metaslab* meta, - freelist::Iter<>& fast_free_list, - LocalEntropy& entropy, - smallsizeclass_t sizeclass) - { - auto& key = entropy.get_free_list_key(); - - std::remove_reference_t tmp_fl; - auto remaining = meta->free_queue.close(tmp_fl, key); - auto p = tmp_fl.take(key, domesticate); - fast_free_list = tmp_fl; - -#ifdef SNMALLOC_CHECK_CLIENT - entropy.refresh_bits(); -#else - UNUSED(entropy); -#endif - - // This marks the slab as sleeping, and sets a wakeup - // when sufficient deallocations have occurred to this slab. - // Takes how many deallocations were not grabbed on this call - // This will be zero if there is no randomisation. - auto sleeping = meta->set_sleeping(sizeclass, remaining); - - return {p, !sleeping}; - } - }; - -#if defined(USE_METADATA_CONCEPT) - static_assert(ConceptMetadataStruct); -#endif - static_assert( - sizeof(Metaslab) == PAGEMAP_METADATA_STRUCT_SIZE, - "Metaslab is expected to be the largest pagemap metadata record"); - - struct MetaslabCache - { -#ifdef SNMALLOC_CHECK_CLIENT - SeqSet available; -#else - // This is slightly faster in some cases, - // but makes memory reuse more predictable. - SeqSet available; -#endif - uint16_t unused = 0; - uint16_t length = 0; - }; - - /* - * Define the encoding of a RemoteAllocator* and a sizeclass_t into a - * MetaEntry's uintptr_t remote_and_sizeclass field. - * - * There's a little bit of an asymmetry here. Since the backend actually sets - * the entry (when associating a metadata structure), we don't construct a - * full MetaEntry here, but rather use ::encode() to compute its - * remote_and_sizeclass value. On the decode side, we are given read-only - * access to MetaEntry-s so can directly read therefrom rather than having to - * speak in terms of uintptr_t-s. - */ - struct FrontendMetaEntry - { - /// Perform the encoding. - static SNMALLOC_FAST_PATH uintptr_t - encode(RemoteAllocator* remote, sizeclass_t sizeclass) - { - /* remote might be nullptr; cast to uintptr_t before offsetting */ - return pointer_offset( - reinterpret_cast(remote), sizeclass.raw()); - } - - [[nodiscard]] static SNMALLOC_FAST_PATH RemoteAllocator* - get_remote(const MetaEntry& me) - { - return reinterpret_cast( - pointer_align_down( - me.get_remote_and_sizeclass())); - } - - [[nodiscard]] static SNMALLOC_FAST_PATH sizeclass_t - get_sizeclass(const MetaEntry& me) - { - // TODO: perhaps remove static_cast with resolution of - // https://github.com/CTSRD-CHERI/llvm-project/issues/588 - return sizeclass_t::from_raw( - static_cast(me.get_remote_and_sizeclass()) & - (REMOTE_WITH_BACKEND_MARKER_ALIGN - 1)); - } - - /** - * Return the Metaslab metadata associated with this chunk, guarded by an - * assert that this chunk is being used as a slab (i.e., has an associated - * owning allocator). - */ - [[nodiscard]] static SNMALLOC_FAST_PATH Metaslab* - get_metaslab(const MetaEntry& me) - { - SNMALLOC_ASSERT(get_remote(me) != nullptr); - return reinterpret_cast(me.get_meta()); - } - }; -} // namespace snmalloc diff --git a/src/mem/remoteallocator.h b/src/mem/remoteallocator.h index 2853c78df..e851a0af9 100644 --- a/src/mem/remoteallocator.h +++ b/src/mem/remoteallocator.h @@ -1,8 +1,8 @@ #pragma once -#include "../backend/metatypes.h" #include "../mem/allocconfig.h" #include "../mem/freelist.h" +#include "../mem/metadata.h" #include "../mem/sizeclasstable.h" #include @@ -10,19 +10,6 @@ namespace snmalloc { - // Remotes need to be aligned enough that the bottom bits have enough room for - // all the size classes, both large and small. An additional bit is required - // to separate backend uses. - static constexpr size_t REMOTE_MIN_ALIGN = - bits::max(CACHELINE_SIZE, SIZECLASS_REP_SIZE) << 1; - - // The bit above the sizeclass is always zero unless this is used - // by the backend to represent another datastructure such as the buddy - // allocator entries. - constexpr size_t REMOTE_WITH_BACKEND_MARKER_ALIGN = - MetaEntry::REMOTE_BACKEND_MARKER; - static_assert((REMOTE_MIN_ALIGN >> 1) == MetaEntry::REMOTE_BACKEND_MARKER); - /** * Global key for all remote lists. */ diff --git a/src/mem/remotecache.h b/src/mem/remotecache.h index a258a38d0..116665687 100644 --- a/src/mem/remotecache.h +++ b/src/mem/remotecache.h @@ -1,10 +1,10 @@ #pragma once -#include "../mem/allocconfig.h" -#include "../mem/freelist.h" -#include "../mem/metaslab.h" -#include "../mem/remoteallocator.h" -#include "../mem/sizeclasstable.h" +#include "allocconfig.h" +#include "freelist.h" +#include "metadata.h" +#include "remoteallocator.h" +#include "sizeclasstable.h" #include #include @@ -52,10 +52,11 @@ namespace snmalloc * * This does not require initialisation to be safely called. */ - SNMALLOC_FAST_PATH bool reserve_space(const MetaEntry& entry) + template + SNMALLOC_FAST_PATH bool reserve_space(const Entry& entry) { - auto size = static_cast( - sizeclass_full_to_size(FrontendMetaEntry::get_sizeclass(entry))); + auto size = + static_cast(sizeclass_full_to_size(entry.get_sizeclass())); bool result = capacity > size; if (result) @@ -75,19 +76,19 @@ namespace snmalloc list[get_slot(target_id, 0)].add(r, key); } - template + template bool post( - typename SharedStateHandle::LocalState* local_state, + typename Backend::LocalState* local_state, RemoteAllocator::alloc_id_t id, const FreeListKey& key) { SNMALLOC_ASSERT(initialised); size_t post_round = 0; bool sent_something = false; - auto domesticate = - [local_state](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { - return capptr_domesticate(local_state, p); - }; + auto domesticate = [local_state](freelist::QueuePtr p) + SNMALLOC_FAST_PATH_LAMBDA { + return capptr_domesticate(local_state, p); + }; while (true) { @@ -101,16 +102,16 @@ namespace snmalloc if (!list[i].empty()) { auto [first, last] = list[i].extract_segment(key); - const MetaEntry& entry = - SharedStateHandle::Pagemap::get_metaentry(address_cast(first)); - auto remote = FrontendMetaEntry::get_remote(entry); + const auto& entry = + Backend::Pagemap::get_metaentry(address_cast(first)); + auto remote = entry.get_remote(); // If the allocator is not correctly aligned, then the bit that is // set implies this is used by the backend, and we should not be // deallocating memory here. snmalloc_check_client( - (address_cast(remote) & MetaEntry::REMOTE_BACKEND_MARKER) == 0, + !entry.is_backend_owned(), "Delayed detection of attempt to free internal structure."); - if constexpr (SharedStateHandle::Options.QueueHeadsAreTame) + if constexpr (Backend::Options.QueueHeadsAreTame) { auto domesticate_nop = [](freelist::QueuePtr p) { return freelist::HeadPtr(p.unsafe_ptr()); @@ -141,9 +142,8 @@ namespace snmalloc // Use the next N bits to spread out remote deallocs in our own // slot. auto r = resend.take(key, domesticate); - const MetaEntry& entry = - SharedStateHandle::Pagemap::get_metaentry(address_cast(r)); - auto i = FrontendMetaEntry::get_remote(entry)->trunc_id(); + const auto& entry = Backend::Pagemap::get_metaentry(address_cast(r)); + auto i = entry.get_remote()->trunc_id(); size_t slot = get_slot(i, post_round); list[slot].add(r, key); } diff --git a/src/test/func/domestication/domestication.cc b/src/test/func/domestication/domestication.cc index 5108960f2..64c74dd14 100644 --- a/src/test/func/domestication/domestication.cc +++ b/src/test/func/domestication/domestication.cc @@ -146,8 +146,8 @@ int main() * * - RemoteAllocator::dequeue domesticating the stub's next pointer (p) * - * - Metaslab::alloc_free_list, domesticating the successor object in the - * newly minted freelist::Iter (i.e., the thing that would be allocated + * - FrontendMetaData::alloc_free_list, domesticating the successor object + * in the newly minted freelist::Iter (i.e., the thing that would be allocated * after q). */ static constexpr size_t expected_count = diff --git a/src/test/func/redblack/redblack.cc b/src/test/func/redblack/redblack.cc index 590133130..208acea28 100644 --- a/src/test/func/redblack/redblack.cc +++ b/src/test/func/redblack/redblack.cc @@ -13,7 +13,7 @@ #include "ds/redblacktree.h" #include "snmalloc.h" -struct Wrapper +struct NodeRef { // The redblack tree is going to be used inside the pagemap, // and the redblack tree cannot use all the bits. Applying an offset @@ -21,7 +21,33 @@ struct Wrapper // the representation. static constexpr size_t offset = 10000; - size_t value = offset << 1; + size_t* ptr; + constexpr NodeRef(size_t* p) : ptr(p) {} + constexpr NodeRef() : ptr(nullptr) {} + constexpr NodeRef(const NodeRef& other) : ptr(other.ptr) {} + constexpr NodeRef(NodeRef&& other) : ptr(other.ptr) {} + + bool operator!=(const NodeRef& other) const + { + return ptr != other.ptr; + } + NodeRef& operator=(const NodeRef& other) + { + ptr = other.ptr; + return *this; + } + void set(uint16_t val) + { + *ptr = ((size_t(val) + offset) << 1) + (*ptr & 1); + } + explicit operator uint16_t() + { + return uint16_t((*ptr >> 1) - offset); + } + explicit operator size_t*() + { + return ptr; + } }; // Simple representation that is like the pagemap. @@ -29,8 +55,8 @@ struct Wrapper // We shift the fields up to make room for the colour. struct node { - Wrapper left; - Wrapper right; + size_t left; + size_t right; }; inline static node array[2048]; @@ -38,40 +64,41 @@ inline static node array[2048]; class Rep { public: - using key = size_t; + using key = uint16_t; static constexpr key null = 0; + static constexpr size_t root{NodeRef::offset << 1}; - using Holder = Wrapper; - using Contents = size_t; + using Handle = NodeRef; + using Contents = uint16_t; - static void set(Holder* ptr, Contents r) + static void set(Handle ptr, Contents r) { - ptr->value = ((r + Wrapper::offset) << 1) + (ptr->value & 1); + ptr.set(r); } - static Contents get(Holder* ptr) + static Contents get(Handle ptr) { - return (ptr->value >> 1) - Wrapper::offset; + return static_cast(ptr); } - static Holder& ref(bool direction, key k) + static Handle ref(bool direction, key k) { if (direction) - return array[k].left; + return {&array[k].left}; else - return array[k].right; + return {&array[k].right}; } static bool is_red(key k) { - return (array[k].left.value & 1) == 1; + return (array[k].left & 1) == 1; } static void set_red(key k, bool new_is_red) { if (new_is_red != is_red(k)) - array[k].left.value ^= 1; + array[k].left ^= 1; } static bool compare(key k1, key k2) @@ -88,6 +115,16 @@ class Rep { return k; } + + static size_t* printable(NodeRef k) + { + return static_cast(k); + } + + static const char* name() + { + return "TestRep"; + } }; template @@ -112,9 +149,9 @@ void test(size_t size, unsigned int seed) for (auto j = batch; j > 0; j--) { auto index = 1 + rand.next() % size; - if (tree.insert_elem(index)) + if (tree.insert_elem(Rep::key(index))) { - entries.push_back(index); + entries.push_back(Rep::key(index)); } } } @@ -189,4 +226,4 @@ int main(int argc, char** argv) // Trace particular example test(size, seed); return 0; -} \ No newline at end of file +} From f6e9796bbcf481e72c641980c6fad1df5f88958d Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Wed, 6 Apr 2022 09:59:33 +0100 Subject: [PATCH 244/302] Introduce header layering (#503) See src/snmalloc/README.md for an explanation of the layers. Some other cleanups on the way: Fine-grained stats support is now gone. It's been broken for two years, it depends on iostream (which then causes linker failures with libstdc++) and it's collecting the wrong stats for the new design. After discussion with @mjp41, it's better to remove it and introduce new stats support later, rather than keep broken code in the main branch. Tracing was controlled with a preprocessor macro, now there's also a CMake option. --- .github/workflows/main.yml | 2 +- CMakeLists.txt | 29 +- src/ds/csv.h | 55 --- src/mem/allocstats.h | 428 ------------------ src/override/override.h | 28 -- src/snmalloc.h | 22 - src/snmalloc/README.md | 40 ++ src/{ => snmalloc}/aal/aal.h | 19 +- src/{ => snmalloc}/aal/aal_arm.h | 0 src/{ => snmalloc}/aal/aal_cheri.h | 2 +- src/{ => snmalloc}/aal/aal_concept.h | 30 +- src/{ => snmalloc}/aal/aal_consts.h | 0 src/{ => snmalloc}/aal/aal_powerpc.h | 0 src/{ => snmalloc}/aal/aal_riscv.h | 0 src/{ => snmalloc}/aal/aal_sparc.h | 0 src/{ => snmalloc}/aal/aal_x86.h | 0 src/{ => snmalloc}/aal/aal_x86_sgx.h | 0 src/{ds => snmalloc/aal}/address.h | 4 +- src/{ => snmalloc}/backend/backend.h | 16 +- .../backend/fixedglobalconfig.h | 3 - src/{ => snmalloc}/backend/globalconfig.h | 31 +- .../backend_helpers/backend_helpers.h | 14 + .../backend_helpers}/buddy.h | 6 +- .../backend_helpers}/commitrange.h | 3 +- .../backend_helpers}/commonconfig.h | 74 +-- .../backend_helpers}/empty_range.h | 2 +- .../backend_helpers}/globalrange.h | 6 +- .../backend_helpers}/largebuddyrange.h | 7 +- .../backend_helpers}/pagemap.h | 6 +- .../backend_helpers}/pagemapregisterrange.h | 3 +- .../backend_helpers}/palrange.h | 3 +- .../backend_helpers}/range_helpers.h | 4 +- .../backend_helpers}/smallbuddyrange.h | 2 - .../backend_helpers}/statsrange.h | 0 .../backend_helpers}/subrange.h | 4 +- src/{ => snmalloc}/ds/aba.h | 3 +- src/{mem => snmalloc/ds}/allocconfig.h | 3 - src/snmalloc/ds/ds.h | 11 + src/{ => snmalloc}/ds/flaglock.h | 3 +- src/{ => snmalloc}/ds/mpmcstack.h | 3 +- src/snmalloc/ds/singleton.h | 51 +++ src/{ds => snmalloc/ds_core}/bits.h | 52 ++- src/{ds => snmalloc/ds_core}/concept.h | 0 src/{ds => snmalloc/ds_core}/defines.h | 0 src/snmalloc/ds_core/ds_core.h | 16 + src/{ds => snmalloc/ds_core}/helpers.h | 41 +- src/{ds => snmalloc/ds_core}/ptrwrap.h | 4 +- src/{ds => snmalloc/ds_core}/redblacktree.h | 3 - src/{ds => snmalloc/ds_core}/seqset.h | 4 +- src/{mem => snmalloc/global}/bounds_checks.h | 4 +- src/snmalloc/global/global.h | 4 + src/{mem => snmalloc/global}/memcpy.h | 5 +- src/{mem => snmalloc/global}/scopedalloc.h | 1 + src/{mem => snmalloc/global}/threadalloc.h | 3 +- .../mem}/backend_concept.h | 4 +- src/snmalloc/mem/backend_wrappers.h | 85 ++++ src/{ => snmalloc}/mem/corealloc.h | 3 +- src/{ => snmalloc}/mem/entropy.h | 1 - src/{ => snmalloc}/mem/external_alloc.h | 0 src/{ => snmalloc}/mem/freelist.h | 3 +- src/{ => snmalloc}/mem/globalalloc.h | 45 +- src/{ => snmalloc}/mem/localalloc.h | 5 +- src/{ => snmalloc}/mem/localcache.h | 11 +- src/snmalloc/mem/mem.h | 16 + src/{ => snmalloc}/mem/metadata.h | 4 +- src/{ => snmalloc}/mem/pool.h | 4 +- src/{ => snmalloc}/mem/pooled.h | 3 +- src/{ => snmalloc}/mem/remoteallocator.h | 8 +- src/{ => snmalloc}/mem/remotecache.h | 3 +- src/{ => snmalloc}/mem/sizeclasstable.h | 5 +- src/{ => snmalloc}/mem/ticker.h | 2 +- .../override/jemalloc_compat.cc | 0 .../override/malloc-extensions.cc | 0 .../override/malloc-extensions.h | 0 src/{ => snmalloc}/override/malloc.cc | 0 src/{ => snmalloc}/override/memcpy.cc | 2 - src/{ => snmalloc}/override/new.cc | 0 src/snmalloc/override/override.h | 15 + src/{ => snmalloc}/override/rust.cc | 0 src/{ => snmalloc}/pal/pal.h | 16 +- src/{ => snmalloc}/pal/pal_apple.h | 0 src/{ => snmalloc}/pal/pal_bsd.h | 0 src/{ => snmalloc}/pal/pal_bsd_aligned.h | 0 src/{ => snmalloc}/pal/pal_concept.h | 93 ++-- src/{ => snmalloc}/pal/pal_consts.h | 2 +- src/{ => snmalloc}/pal/pal_dragonfly.h | 0 src/{ => snmalloc}/pal/pal_ds.h | 3 +- src/{ => snmalloc}/pal/pal_freebsd.h | 0 src/{ => snmalloc}/pal/pal_freebsd_kernel.h | 2 +- src/{ => snmalloc}/pal/pal_haiku.h | 0 src/{ => snmalloc}/pal/pal_linux.h | 2 +- src/{ => snmalloc}/pal/pal_netbsd.h | 0 src/{ => snmalloc}/pal/pal_noalloc.h | 0 src/{ => snmalloc}/pal/pal_open_enclave.h | 0 src/{ => snmalloc}/pal/pal_openbsd.h | 0 src/{ => snmalloc}/pal/pal_plain.h | 2 +- src/{ => snmalloc}/pal/pal_posix.h | 5 +- src/{ => snmalloc}/pal/pal_solaris.h | 0 src/{ => snmalloc}/pal/pal_timer_default.h | 0 src/{ => snmalloc}/pal/pal_windows.h | 3 +- src/snmalloc/snmalloc.h | 10 + src/snmalloc/snmalloc_core.h | 3 + src/snmalloc/snmalloc_front.h | 1 + src/snmalloc_core.h | 15 - src/snmalloc_front.h | 2 - src/test/func/bits/bits.cc | 4 +- src/test/func/domestication/domestication.cc | 6 +- .../func/external_pagemap/external_pagemap.cc | 2 +- .../func/first_operation/first_operation.cc | 2 +- src/test/func/fixed_region/fixed_region.cc | 4 +- src/test/func/jemalloc/jemalloc.cc | 4 +- src/test/func/malloc/malloc.cc | 2 +- src/test/func/memcpy/func-memcpy.cc | 5 +- src/test/func/memory/memory.cc | 2 +- src/test/func/memory_usage/memory_usage.cc | 4 +- src/test/func/pagemap/pagemap.cc | 4 +- src/test/func/pool/pool.cc | 2 +- src/test/func/redblack/redblack.cc | 7 +- src/test/func/release-rounding/rounding.cc | 2 +- src/test/func/sizeclass/sizeclass.cc | 4 +- src/test/func/statistics/stats.cc | 2 +- src/test/func/teardown/teardown.cc | 2 +- .../thread_alloc_external.cc | 6 +- src/test/func/two_alloc_types/alloc1.cc | 10 +- src/test/func/two_alloc_types/alloc2.cc | 6 +- src/test/func/two_alloc_types/main.cc | 3 +- src/test/perf/contention/contention.cc | 2 +- .../perf/external_pointer/externalpointer.cc | 2 +- src/test/perf/low_memory/low-memory.cc | 4 +- src/test/perf/memcpy/memcpy.cc | 2 +- src/test/perf/singlethread/singlethread.cc | 2 +- src/test/setup.h | 5 +- 132 files changed, 545 insertions(+), 987 deletions(-) delete mode 100644 src/ds/csv.h delete mode 100644 src/mem/allocstats.h delete mode 100644 src/override/override.h delete mode 100644 src/snmalloc.h create mode 100644 src/snmalloc/README.md rename src/{ => snmalloc}/aal/aal.h (95%) rename src/{ => snmalloc}/aal/aal_arm.h (100%) rename src/{ => snmalloc}/aal/aal_cheri.h (98%) rename src/{ => snmalloc}/aal/aal_concept.h (68%) rename src/{ => snmalloc}/aal/aal_consts.h (100%) rename src/{ => snmalloc}/aal/aal_powerpc.h (100%) rename src/{ => snmalloc}/aal/aal_riscv.h (100%) rename src/{ => snmalloc}/aal/aal_sparc.h (100%) rename src/{ => snmalloc}/aal/aal_x86.h (100%) rename src/{ => snmalloc}/aal/aal_x86_sgx.h (100%) rename src/{ds => snmalloc/aal}/address.h (99%) rename src/{ => snmalloc}/backend/backend.h (96%) rename src/{ => snmalloc}/backend/fixedglobalconfig.h (95%) rename src/{ => snmalloc}/backend/globalconfig.h (79%) create mode 100644 src/snmalloc/backend_helpers/backend_helpers.h rename src/{backend => snmalloc/backend_helpers}/buddy.h (96%) rename src/{backend => snmalloc/backend_helpers}/commitrange.h (94%) rename src/{backend => snmalloc/backend_helpers}/commonconfig.h (66%) rename src/{backend => snmalloc/backend_helpers}/empty_range.h (93%) rename src/{backend => snmalloc/backend_helpers}/globalrange.h (91%) rename src/{backend => snmalloc/backend_helpers}/largebuddyrange.h (98%) rename src/{backend => snmalloc/backend_helpers}/pagemap.h (98%) rename src/{backend => snmalloc/backend_helpers}/pagemapregisterrange.h (94%) rename src/{backend => snmalloc/backend_helpers}/palrange.h (96%) rename src/{backend => snmalloc/backend_helpers}/range_helpers.h (95%) rename src/{backend => snmalloc/backend_helpers}/smallbuddyrange.h (99%) rename src/{backend => snmalloc/backend_helpers}/statsrange.h (100%) rename src/{backend => snmalloc/backend_helpers}/subrange.h (96%) rename src/{ => snmalloc}/ds/aba.h (99%) rename src/{mem => snmalloc/ds}/allocconfig.h (98%) create mode 100644 src/snmalloc/ds/ds.h rename src/{ => snmalloc}/ds/flaglock.h (98%) rename src/{ => snmalloc}/ds/mpmcstack.h (97%) create mode 100644 src/snmalloc/ds/singleton.h rename src/{ds => snmalloc/ds_core}/bits.h (87%) rename src/{ds => snmalloc/ds_core}/concept.h (100%) rename src/{ds => snmalloc/ds_core}/defines.h (100%) create mode 100644 src/snmalloc/ds_core/ds_core.h rename src/{ds => snmalloc/ds_core}/helpers.h (90%) rename src/{ds => snmalloc/ds_core}/ptrwrap.h (99%) rename src/{ds => snmalloc/ds_core}/redblacktree.h (99%) rename src/{ds => snmalloc/ds_core}/seqset.h (98%) rename src/{mem => snmalloc/global}/bounds_checks.h (98%) create mode 100644 src/snmalloc/global/global.h rename src/{mem => snmalloc/global}/memcpy.h (99%) rename src/{mem => snmalloc/global}/scopedalloc.h (98%) rename src/{mem => snmalloc/global}/threadalloc.h (99%) rename src/{backend => snmalloc/mem}/backend_concept.h (97%) create mode 100644 src/snmalloc/mem/backend_wrappers.h rename src/{ => snmalloc}/mem/corealloc.h (99%) rename src/{ => snmalloc}/mem/entropy.h (99%) rename src/{ => snmalloc}/mem/external_alloc.h (100%) rename src/{ => snmalloc}/mem/freelist.h (99%) rename src/{ => snmalloc}/mem/globalalloc.h (72%) rename src/{ => snmalloc}/mem/localalloc.h (99%) rename src/{ => snmalloc}/mem/localcache.h (90%) create mode 100644 src/snmalloc/mem/mem.h rename src/{ => snmalloc}/mem/metadata.h (99%) rename src/{ => snmalloc}/mem/pool.h (98%) rename src/{ => snmalloc}/mem/pooled.h (90%) rename src/{ => snmalloc}/mem/remoteallocator.h (98%) rename src/{ => snmalloc}/mem/remotecache.h (99%) rename src/{ => snmalloc}/mem/sizeclasstable.h (99%) rename src/{ => snmalloc}/mem/ticker.h (98%) rename src/{ => snmalloc}/override/jemalloc_compat.cc (100%) rename src/{ => snmalloc}/override/malloc-extensions.cc (100%) rename src/{ => snmalloc}/override/malloc-extensions.h (100%) rename src/{ => snmalloc}/override/malloc.cc (100%) rename src/{ => snmalloc}/override/memcpy.cc (90%) rename src/{ => snmalloc}/override/new.cc (100%) create mode 100644 src/snmalloc/override/override.h rename src/{ => snmalloc}/override/rust.cc (100%) rename src/{ => snmalloc}/pal/pal.h (89%) rename src/{ => snmalloc}/pal/pal_apple.h (100%) rename src/{ => snmalloc}/pal/pal_bsd.h (100%) rename src/{ => snmalloc}/pal/pal_bsd_aligned.h (100%) rename src/{ => snmalloc}/pal/pal_concept.h (58%) rename src/{ => snmalloc}/pal/pal_consts.h (98%) rename src/{ => snmalloc}/pal/pal_dragonfly.h (100%) rename src/{ => snmalloc}/pal/pal_ds.h (98%) rename src/{ => snmalloc}/pal/pal_freebsd.h (100%) rename src/{ => snmalloc}/pal/pal_freebsd_kernel.h (98%) rename src/{ => snmalloc}/pal/pal_haiku.h (100%) rename src/{ => snmalloc}/pal/pal_linux.h (99%) rename src/{ => snmalloc}/pal/pal_netbsd.h (100%) rename src/{ => snmalloc}/pal/pal_noalloc.h (100%) rename src/{ => snmalloc}/pal/pal_open_enclave.h (100%) rename src/{ => snmalloc}/pal/pal_openbsd.h (100%) rename src/{ => snmalloc}/pal/pal_plain.h (95%) rename src/{ => snmalloc}/pal/pal_posix.h (99%) rename src/{ => snmalloc}/pal/pal_solaris.h (100%) rename src/{ => snmalloc}/pal/pal_timer_default.h (100%) rename src/{ => snmalloc}/pal/pal_windows.h (99%) create mode 100644 src/snmalloc/snmalloc.h create mode 100644 src/snmalloc/snmalloc_core.h create mode 100644 src/snmalloc/snmalloc_front.h delete mode 100644 src/snmalloc_core.h delete mode 100644 src/snmalloc_front.h diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 27a528c4b..6a0ba7217 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -287,7 +287,7 @@ jobs: git diff --exit-code - name: Run clang-tidy run: | - clang-tidy-9 src/override/malloc.cc -header-filter="`pwd`/*" -warnings-as-errors='*' -export-fixes=tidy.fail -- -std=c++17 -mcx16 -DSNMALLOC_PLATFORM_HAS_GETENTROPY=0 + clang-tidy-9 src/snmalloc/override/malloc.cc -header-filter="`pwd`/*" -warnings-as-errors='*' -export-fixes=tidy.fail -- -std=c++17 -mcx16 -DSNMALLOC_PLATFORM_HAS_GETENTROPY=0 if [ -f tidy.fail ] ; then cat tidy.fail exit 1 diff --git a/CMakeLists.txt b/CMakeLists.txt index 339f4f69a..db1e5c545 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,10 +12,10 @@ include(CMakeDependentOption) option(SNMALLOC_HEADER_ONLY_LIBRARY "Use snmalloc has a header-only library" OFF) # Options that apply globally -option(USE_SNMALLOC_STATS "Track allocation stats" OFF) option(SNMALLOC_CI_BUILD "Disable features not sensible for CI" OFF) option(SNMALLOC_QEMU_WORKAROUND "Disable using madvise(DONT_NEED) to zero memory on Linux" Off) option(SNMALLOC_USE_CXX17 "Build as C++17 for legacy support." OFF) +option(SNMALLOC_TRACING "Enable large quantities of debug output." OFF) option(SNMALLOC_NO_REALLOCARRAY "Build without reallocarray exported" ON) option(SNMALLOC_NO_REALLOCARR "Build without reallocarr exported" ON) # Options that apply only if we're not building the header-only library @@ -198,8 +198,8 @@ function(add_as_define FLAG) target_compile_definitions(snmalloc INTERFACE $<$:${FLAG}>) endfunction() -add_as_define(USE_SNMALLOC_STATS) add_as_define(SNMALLOC_QEMU_WORKAROUND) +add_as_define(SNMALLOC_TRACING) add_as_define(SNMALLOC_CI_BUILD) add_as_define(SNMALLOC_PLATFORM_HAS_GETENTROPY) if (SNMALLOC_NO_REALLOCARRAY) @@ -328,9 +328,9 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) endfunction() - set(SHIM_FILES src/override/new.cc) + set(SHIM_FILES src/snmalloc/override/new.cc) if (SNMALLOC_MEMCPY_BOUNDS) - list(APPEND SHIM_FILES src/override/memcpy.cc) + list(APPEND SHIM_FILES src/snmalloc/override/memcpy.cc) endif () if (SNMALLOC_STATIC_LIBRARY) @@ -346,8 +346,8 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) endif() if(SNMALLOC_RUST_SUPPORT) - add_shim(snmallocshim-rust STATIC src/override/rust.cc) - add_shim(snmallocshim-checks-rust STATIC src/override/rust.cc) + add_shim(snmallocshim-rust STATIC src/snmalloc/override/rust.cc) + add_shim(snmallocshim-checks-rust STATIC src/snmalloc/override/rust.cc) target_compile_definitions(snmallocshim-checks-rust PRIVATE SNMALLOC_CHECK_CLIENT) endif() @@ -426,9 +426,6 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) set_tests_properties(${TESTNAME} PROPERTIES PROCESSORS 4) endif() endif() - # if (${TEST_CATEGORY} MATCHES "func") - # target_compile_definitions(${TESTNAME} PRIVATE -DUSE_SNMALLOC_STATS) - # endif () endforeach() endforeach() endforeach() @@ -441,12 +438,12 @@ install(TARGETS snmalloc EXPORT snmallocConfig) install(TARGETS EXPORT snmallocConfig DESTINATION ${CMAKE_INSTALL_LIBDIR} PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/snmalloc) -install(DIRECTORY src/aal DESTINATION include/snmalloc) -install(DIRECTORY src/ds DESTINATION include/snmalloc) -install(DIRECTORY src/override DESTINATION include/snmalloc) -install(DIRECTORY src/backend DESTINATION include/snmalloc) -install(DIRECTORY src/mem DESTINATION include/snmalloc) -install(DIRECTORY src/pal DESTINATION include/snmalloc) +install(DIRECTORY src/snmalloc/aal DESTINATION include/snmalloc) +install(DIRECTORY src/snmalloc/ds DESTINATION include/snmalloc) +install(DIRECTORY src/snmalloc/override DESTINATION include/snmalloc) +install(DIRECTORY src/snmalloc/backend DESTINATION include/snmalloc) +install(DIRECTORY src/snmalloc/mem DESTINATION include/snmalloc) +install(DIRECTORY src/snmalloc/pal DESTINATION include/snmalloc) install(FILES src/test/measuretime.h src/test/opt.h @@ -455,7 +452,7 @@ install(FILES src/test/xoroshiro.h DESTINATION include/snmalloc/test ) -install(FILES src/snmalloc.h;src/snmalloc_core.h;src/snmalloc_front.h DESTINATION include/snmalloc) +install(FILES src/snmalloc/snmalloc.h;src/snmalloc/snmalloc_core.h;src/snmalloc/snmalloc_front.h DESTINATION include/snmalloc) install(EXPORT snmallocConfig FILE snmalloc-config.cmake diff --git a/src/ds/csv.h b/src/ds/csv.h deleted file mode 100644 index 63419efbd..000000000 --- a/src/ds/csv.h +++ /dev/null @@ -1,55 +0,0 @@ -#pragma once - -#include -#include - -namespace snmalloc -{ - class CSVStream - { - private: - std::ostream* out; - bool first = true; - - public: - class Endl - {}; - - Endl endl; - - CSVStream(std::ostream* o) : out(o) {} - - void preprint() - { - if (!first) - { - *out << ", "; - } - else - { - first = false; - } - } - - CSVStream& operator<<(const std::string& str) - { - preprint(); - *out << str; - return *this; - } - - CSVStream& operator<<(uint64_t u) - { - preprint(); - *out << u; - return *this; - } - - CSVStream& operator<<(Endl) - { - *out << std::endl; - first = true; - return *this; - } - }; -} // namespace snmalloc \ No newline at end of file diff --git a/src/mem/allocstats.h b/src/mem/allocstats.h deleted file mode 100644 index 2d3ae452d..000000000 --- a/src/mem/allocstats.h +++ /dev/null @@ -1,428 +0,0 @@ -#pragma once - -#include "../ds/bits.h" -#include "../mem/sizeclasstable.h" - -#include - -#ifdef USE_SNMALLOC_STATS -# include "../ds/csv.h" - -# include -# include -#endif - -namespace snmalloc -{ - template - struct AllocStats - { - constexpr AllocStats() = default; - - struct CurrentMaxPair - { - size_t current{0}; - size_t max{0}; - size_t used{0}; - - void inc() - { - current++; - used++; - if (current > max) - max++; - } - - void dec() - { - // Split stats means this is not true. - // TODO reestablish checks, when we sanitise the stats. - // SNMALLOC_ASSERT(current > 0); - current--; - } - - bool is_empty() - { - return current == 0; - } - - bool is_unused() - { - return max == 0; - } - - void add(CurrentMaxPair& that) - { - current += that.current; - max += that.max; - used += that.used; - } -#ifdef USE_SNMALLOC_STATS - void print(CSVStream& csv, size_t multiplier = 1) - { - csv << current * multiplier << max * multiplier << used * multiplier; - } -#endif - }; - - struct Stats - { - constexpr Stats() = default; - - CurrentMaxPair count; - CurrentMaxPair slab_count; - uint64_t time{0}; - uint64_t ticks{0}; - double online_average{0}; - - bool is_empty() - { - return count.is_empty(); - } - - void add(Stats& that) - { - count.add(that.count); - slab_count.add(that.slab_count); - } - - void addToRunningAverage() - { - uint64_t now = Aal::tick(); - - if (slab_count.current != 0) - { - double occupancy = static_cast(count.current) / - static_cast(slab_count.current); - uint64_t duration = now - time; - - if (ticks == 0) - online_average = occupancy; - else - online_average += - ((occupancy - online_average) * static_cast(duration)) / - static_cast(ticks); - - ticks += duration; - } - - time = now; - } - -#ifdef USE_SNMALLOC_STATS - void - print(CSVStream& csv, size_t multiplier = 1, size_t slab_multiplier = 1) - { - // Keep in sync with header lower down - count.print(csv, multiplier); - slab_count.print(csv, slab_multiplier); - size_t average = - static_cast(online_average * static_cast(multiplier)); - - csv << average << (slab_multiplier - average) * slab_count.max - << csv.endl; - } -#endif - }; - -#ifdef USE_SNMALLOC_STATS - static constexpr size_t BUCKETS_BITS = 4; - static constexpr size_t BUCKETS = 1 << BUCKETS_BITS; - static constexpr size_t TOTAL_BUCKETS = - bits::to_exp_mant_const( - bits::one_at_bit(bits::ADDRESS_BITS - 1)); - - Stats sizeclass[N]; - - size_t large_pop_count[LARGE_N] = {0}; - size_t large_push_count[LARGE_N] = {0}; - - size_t remote_freed = 0; - size_t remote_posted = 0; - size_t remote_received = 0; - size_t superslab_push_count = 0; - size_t superslab_pop_count = 0; - size_t superslab_fresh_count = 0; - size_t segment_count = 0; - size_t bucketed_requests[TOTAL_BUCKETS] = {}; -#endif - - void alloc_request(size_t size) - { - UNUSED(size); - -#ifdef USE_SNMALLOC_STATS - auto index = (size == 0) ? 0 : bits::to_exp_mant(size); - SNMALLOC_ASSERT(index < TOTAL_BUCKETS); - bucketed_requests[index]++; -#endif - } - - bool is_empty() - { -#ifdef USE_SNMALLOC_STATS - for (size_t i = 0; i < N; i++) - { - if (!sizeclass[i].is_empty()) - return false; - } - - for (size_t i = 0; i < LARGE_N; i++) - { - if (large_push_count[i] != large_pop_count[i]) - return false; - } - - return (remote_freed == remote_posted); -#else - return true; -#endif - } - - void sizeclass_alloc(smallsizeclass_t sc) - { - UNUSED(sc); - -#ifdef USE_SNMALLOC_STATS - sizeclass[sc].addToRunningAverage(); - sizeclass[sc].count.inc(); -#endif - } - - void sizeclass_dealloc(smallsizeclass_t sc) - { - UNUSED(sc); - -#ifdef USE_SNMALLOC_STATS - sizeclass[sc].addToRunningAverage(); - sizeclass[sc].count.dec(); -#endif - } - - void large_alloc(size_t sc) - { - UNUSED(sc); - -#ifdef USE_SNMALLOC_STATS - SNMALLOC_ASSUME(sc < LARGE_N); - large_pop_count[sc]++; -#endif - } - - void sizeclass_alloc_slab(smallsizeclass_t sc) - { - UNUSED(sc); - -#ifdef USE_SNMALLOC_STATS - sizeclass[sc].addToRunningAverage(); - sizeclass[sc].slab_count.inc(); -#endif - } - - void sizeclass_dealloc_slab(smallsizeclass_t sc) - { - UNUSED(sc); - -#ifdef USE_SNMALLOC_STATS - sizeclass[sc].addToRunningAverage(); - sizeclass[sc].slab_count.dec(); -#endif - } - - void large_dealloc(size_t sc) - { - UNUSED(sc); - -#ifdef USE_SNMALLOC_STATS - large_push_count[sc]++; -#endif - } - - void segment_create() - { -#ifdef USE_SNMALLOC_STATS - segment_count++; -#endif - } - - void superslab_pop() - { -#ifdef USE_SNMALLOC_STATS - superslab_pop_count++; -#endif - } - - void superslab_push() - { -#ifdef USE_SNMALLOC_STATS - superslab_push_count++; -#endif - } - - void superslab_fresh() - { -#ifdef USE_SNMALLOC_STATS - superslab_fresh_count++; -#endif - } - - void remote_free(smallsizeclass_t sc) - { - UNUSED(sc); - -#ifdef USE_SNMALLOC_STATS - remote_freed += sizeclass_to_size(sc); -#endif - } - - void remote_post() - { -#ifdef USE_SNMALLOC_STATS - remote_posted = remote_freed; -#endif - } - - void remote_receive(smallsizeclass_t sc) - { - UNUSED(sc); - -#ifdef USE_SNMALLOC_STATS - remote_received += sizeclass_to_size(sc); -#endif - } - - void add(AllocStats& that) - { - UNUSED(that); - -#ifdef USE_SNMALLOC_STATS - for (size_t i = 0; i < N; i++) - sizeclass[i].add(that.sizeclass[i]); - - for (size_t i = 0; i < LARGE_N; i++) - { - large_push_count[i] += that.large_push_count[i]; - large_pop_count[i] += that.large_pop_count[i]; - } - - for (size_t i = 0; i < TOTAL_BUCKETS; i++) - bucketed_requests[i] += that.bucketed_requests[i]; - - remote_freed += that.remote_freed; - remote_posted += that.remote_posted; - remote_received += that.remote_received; - superslab_pop_count += that.superslab_pop_count; - superslab_push_count += that.superslab_push_count; - superslab_fresh_count += that.superslab_fresh_count; - segment_count += that.segment_count; -#endif - } - -#ifdef USE_SNMALLOC_STATS - template - void print(std::ostream& o, uint64_t dumpid = 0, uint64_t allocatorid = 0) - { - UNUSED(o, dumpid, allocatorid); - - CSVStream csv(&o); - - if (dumpid == 0) - { - // Output headers for initial dump - // Keep in sync with data dump - csv << "GlobalStats" - << "DumpID" - << "AllocatorID" - << "Remote freed" - << "Remote posted" - << "Remote received" - << "Superslab pop" - << "Superslab push" - << "Superslab fresh" - << "Segments" << csv.endl; - - csv << "BucketedStats" - << "DumpID" - << "AllocatorID" - << "Size group" - << "Size" - << "Current count" - << "Max count" - << "Total Allocs" - << "Current Slab bytes" - << "Max Slab bytes" - << "Total slab allocs" - << "Average Slab Usage" - << "Average wasted space" << csv.endl; - - csv << "LargeBucketedStats" - << "DumpID" - << "AllocatorID" - << "Size group" - << "Size" - << "Push count" - << "Pop count" << csv.endl; - - csv << "AllocSizes" - << "DumpID" - << "AllocatorID" - << "ClassID" - << "Low size" - << "High size" - << "Count" << csv.endl; - } - - for (smallsizeclass_t i = 0; i < N; i++) - { - if (sizeclass[i].count.is_unused()) - continue; - - sizeclass[i].addToRunningAverage(); - - csv << "BucketedStats" << dumpid << allocatorid << i - << sizeclass_to_size(i); - - sizeclass[i].print(csv, sizeclass_to_size(i)); - } - - // for (uint8_t i = 0; i < LARGE_N; i++) - // { - // if ((large_push_count[i] == 0) && (large_pop_count[i] == 0)) - // continue; - - // csv << "LargeBucketedStats" << dumpid << allocatorid << (i + N) - // << large_sizeclass_to_size(i) << large_push_count[i] - // << large_pop_count[i] << csv.endl; - // } - - size_t low = 0; - size_t high = 0; - - for (size_t i = 0; i < TOTAL_BUCKETS; i++) - { - low = high + 1; - high = bits::from_exp_mant(i); - - if (bucketed_requests[i] == 0) - continue; - - csv << "AllocSizes" << dumpid << allocatorid << i << low << high - << bucketed_requests[i] << csv.endl; - } - - csv << "GlobalStats" << dumpid << allocatorid << remote_freed - << remote_posted << remote_received << superslab_pop_count - << superslab_push_count << superslab_fresh_count << segment_count - << csv.endl; - } -#endif - - void start() - { -#ifdef USE_SNMALLOC_STATS - for (size_t i = 0; i < N; i++) - sizeclass[i].time = Aal::tick(); -#endif - } - }; -} // namespace snmalloc diff --git a/src/override/override.h b/src/override/override.h deleted file mode 100644 index a9aedbe9d..000000000 --- a/src/override/override.h +++ /dev/null @@ -1,28 +0,0 @@ -#pragma once - -// Core implementation of snmalloc independent of the configuration mode -#include "../snmalloc_core.h" - -#ifndef SNMALLOC_PROVIDE_OWN_CONFIG -# include "../backend/globalconfig.h" -// The default configuration for snmalloc is used if alternative not defined -namespace snmalloc -{ - using Alloc = snmalloc::LocalAllocator; -} // namespace snmalloc -#endif - -// User facing API surface, needs to know what `Alloc` is. -#include "../snmalloc_front.h" - -#ifndef SNMALLOC_EXPORT -# define SNMALLOC_EXPORT -#endif -#ifdef SNMALLOC_STATIC_LIBRARY_PREFIX -# define __SN_CONCAT(a, b) a##b -# define __SN_EVALUATE(a, b) __SN_CONCAT(a, b) -# define SNMALLOC_NAME_MANGLE(a) \ - __SN_EVALUATE(SNMALLOC_STATIC_LIBRARY_PREFIX, a) -#elif !defined(SNMALLOC_NAME_MANGLE) -# define SNMALLOC_NAME_MANGLE(a) a -#endif diff --git a/src/snmalloc.h b/src/snmalloc.h deleted file mode 100644 index e1d90d402..000000000 --- a/src/snmalloc.h +++ /dev/null @@ -1,22 +0,0 @@ -#pragma once - -// Core implementation of snmalloc independent of the configuration mode -#include "snmalloc_core.h" - -// If you define SNMALLOC_PROVIDE_OWN_CONFIG then you must provide your own -// definition of `snmalloc::Alloc` and include `snmalloc_front.h` before -// including any files that include `snmalloc.h` and consume the global -// allocation APIs. -#ifndef SNMALLOC_PROVIDE_OWN_CONFIG -// Default implementation of global state -# include "backend/globalconfig.h" - -// The default configuration for snmalloc -namespace snmalloc -{ - using Alloc = snmalloc::LocalAllocator; -} - -// User facing API surface, needs to know what `Alloc` is. -# include "snmalloc_front.h" -#endif diff --git a/src/snmalloc/README.md b/src/snmalloc/README.md new file mode 100644 index 000000000..0366a5e77 --- /dev/null +++ b/src/snmalloc/README.md @@ -0,0 +1,40 @@ +Include hierarchy +----------------- + +The `snmalloc/` include path contains all of the snmalloc headers. +These are arranged in a hierarchy such that each of the directories may include ones below, in the following order, starting at the bottom: + + - `ds_core/` provides core data structures that depend on the C++ implementation and nothing else. + This directory includes a number of things that abstract over different language extensions (for example, different built-in function names in different compilers). + - `aal/` provides the architecture abstraction layer (AAL). + This layer provides abstractions over CPU-specific intrinsics and defines things such as the virtual address-space size. + There is a single AAL for an snmalloc instantiation. + - `pal/` provides the platform abstraction layer (PAL). + This exposes OS- or environment-specific abstractions into the rest of the code. + An snmalloc instantiation may use more than one PAL, including ones provided by the user. + - `ds/` includes data structures that may depend on platform services or on features specific to the current CPU. + - `mem/` provides the core allocator abstractions. + The code here is templated over a back-end, which defines a particular embedding of snmalloc. + - `backend_helpers/` provides helper classes for use in defining a back end. + This includes data structures such as pagemap implementations (efficient maps from a chunk address to associated metadata) and buddy allocators for managing address-space ranges. + - `backend/` provides some example implementations for snmalloc embeddings that provide a global memory allocator for an address space. + Users may ignore this entirely and use the types in `mem/` with a custom back end to expose an snmalloc instance with specific behaviour. + Layers above this can be used with a custom configuration by defining `SNMALLOC_PROVIDE_OWN_CONFIG` and exporting a type as `snmalloc::Alloc` that defines the type of an `snmalloc::LocalAllocator` template specialisation. + - `global/` provides some front-end components that assume that snmalloc is available in a global configuration. + - `override/` builds on top of `global/` to provide specific implementations with compatibility with external specifications (for example C `malloc`, C++ `operator new`, jemalloc's `*allocx`, or Rust's `std::alloc`). + +Each layer until `backend_helpers/` provides a single header with the same name as the directory. +Files in higher layers should depend only on the single-file version. +This allows specific files to be moved to a lower layer if appropriate, without too much code churn. + +There is only one exception to this rule: `backend/globalconfig.h`. +This file defines either the default configuration *or* nothing, depending on whether the user has defined `SNMALLOC_PROVIDE_OWN_CONFIG`. +The layers above the back end should include only this file, so that there is a single interception point for externally defined back ends. + +External code should include only the following files: + + - `snmalloc/snmalloc_core.h` includes everything up to `backend_helpers`. + This provides the building blocks required to assemble an snmalloc instance, but does not assume any global configuration. + - `snmalloc/snmalloc_front.h` assumes a global configuration (either user-provided or the default from `snmalloc/backend/globalconfig.h` and exposes all of the functionality that depends on both. + - `snmalloc/snmalloc.h` is a convenience wrapper that includes both of the above files. + - `snmalloc/override/*.cc` can be compiled as-is or included after `snmalloc/snmalloc_core.h` and a custom global allocator definition to provide specific languages' global memory allocator APIs with a custom snmalloc embedding. diff --git a/src/aal/aal.h b/src/snmalloc/aal/aal.h similarity index 95% rename from src/aal/aal.h rename to src/snmalloc/aal/aal.h index fcff4dc80..413c284b0 100644 --- a/src/aal/aal.h +++ b/src/snmalloc/aal/aal.h @@ -1,7 +1,12 @@ +/** + * The snmalloc architecture abstraction layer. This defines + * CPU-architecture-specific functionality. + * + * Files in this directory may depend on `ds_core` and each other, but nothing + * else in snmalloc. + */ #pragma once -#include "../ds/concept.h" -#include "../ds/defines.h" -#include "../ds/ptrwrap.h" +#include "../ds_core/ds_core.h" #include "aal_concept.h" #include "aal_consts.h" @@ -242,10 +247,6 @@ namespace snmalloc constexpr static bool aal_supports = (AAL::aal_features & F) == F; } // namespace snmalloc -#if defined(_MSC_VER) && defined(SNMALLOC_VA_BITS_32) -# include -#endif - #ifdef __POINTER_WIDTH__ # if ((__POINTER_WIDTH__ == 64) && !defined(SNMALLOC_VA_BITS_64)) || \ ((__POINTER_WIDTH__ == 32) && !defined(SNMALLOC_VA_BITS_32)) @@ -262,3 +263,7 @@ static_assert(sizeof(size_t) == 4); #elif defined(SNMALLOC_VA_BITS_64) static_assert(sizeof(size_t) == 8); #endif + +// Included after the AAL has been defined, depends on the AAL's notion of an +// address +#include "address.h" diff --git a/src/aal/aal_arm.h b/src/snmalloc/aal/aal_arm.h similarity index 100% rename from src/aal/aal_arm.h rename to src/snmalloc/aal/aal_arm.h diff --git a/src/aal/aal_cheri.h b/src/snmalloc/aal/aal_cheri.h similarity index 98% rename from src/aal/aal_cheri.h rename to src/snmalloc/aal/aal_cheri.h index 1a7906e6b..4774dde67 100644 --- a/src/aal/aal_cheri.h +++ b/src/snmalloc/aal/aal_cheri.h @@ -1,6 +1,6 @@ #pragma once -#include "../ds/defines.h" +#include "../ds_core/ds_core.h" #include diff --git a/src/aal/aal_concept.h b/src/snmalloc/aal/aal_concept.h similarity index 68% rename from src/aal/aal_concept.h rename to src/snmalloc/aal/aal_concept.h index 15eed19bc..6269d9629 100644 --- a/src/aal/aal_concept.h +++ b/src/snmalloc/aal/aal_concept.h @@ -1,8 +1,7 @@ #pragma once #ifdef __cpp_concepts -# include "../ds/concept.h" -# include "../ds/ptrwrap.h" +# include "../ds_core/ds_core.h" # include "aal_consts.h" # include @@ -27,9 +26,12 @@ namespace snmalloc * AALs provide a prefetch operation. */ template - concept ConceptAAL_prefetch = requires(void *ptr) + concept ConceptAAL_prefetch = requires(void* ptr) { - { AAL::prefetch(ptr) } noexcept -> ConceptSame; + { + AAL::prefetch(ptr) + } + noexcept->ConceptSame; }; /** @@ -38,29 +40,31 @@ namespace snmalloc template concept ConceptAAL_tick = requires() { - { AAL::tick() } noexcept -> ConceptSame; + { + AAL::tick() + } + noexcept->ConceptSame; }; template concept ConceptAAL_capptr_methods = - requires(capptr::Chunk auth, capptr::AllocFull ret, size_t sz) + requires(capptr::Chunk auth, capptr::AllocFull ret, size_t sz) { /** * Produce a pointer with reduced authority from a more privilged pointer. * The resulting pointer will have base at auth's address and length of * exactly sz. auth+sz must not exceed auth's limit. */ - { AAL::template capptr_bound(auth, sz) } - noexcept - -> ConceptSame>; + { + AAL::template capptr_bound(auth, sz) + } + noexcept->ConceptSame>; }; template concept ConceptAAL = - ConceptAAL_static_members && - ConceptAAL_prefetch && - ConceptAAL_tick && - ConceptAAL_capptr_methods; + ConceptAAL_static_members&& ConceptAAL_prefetch&& + ConceptAAL_tick&& ConceptAAL_capptr_methods; } // namespace snmalloc #endif diff --git a/src/aal/aal_consts.h b/src/snmalloc/aal/aal_consts.h similarity index 100% rename from src/aal/aal_consts.h rename to src/snmalloc/aal/aal_consts.h diff --git a/src/aal/aal_powerpc.h b/src/snmalloc/aal/aal_powerpc.h similarity index 100% rename from src/aal/aal_powerpc.h rename to src/snmalloc/aal/aal_powerpc.h diff --git a/src/aal/aal_riscv.h b/src/snmalloc/aal/aal_riscv.h similarity index 100% rename from src/aal/aal_riscv.h rename to src/snmalloc/aal/aal_riscv.h diff --git a/src/aal/aal_sparc.h b/src/snmalloc/aal/aal_sparc.h similarity index 100% rename from src/aal/aal_sparc.h rename to src/snmalloc/aal/aal_sparc.h diff --git a/src/aal/aal_x86.h b/src/snmalloc/aal/aal_x86.h similarity index 100% rename from src/aal/aal_x86.h rename to src/snmalloc/aal/aal_x86.h diff --git a/src/aal/aal_x86_sgx.h b/src/snmalloc/aal/aal_x86_sgx.h similarity index 100% rename from src/aal/aal_x86_sgx.h rename to src/snmalloc/aal/aal_x86_sgx.h diff --git a/src/ds/address.h b/src/snmalloc/aal/address.h similarity index 99% rename from src/ds/address.h rename to src/snmalloc/aal/address.h index a6a161a7e..1f528f980 100644 --- a/src/ds/address.h +++ b/src/snmalloc/aal/address.h @@ -1,7 +1,5 @@ #pragma once -#include "../pal/pal_consts.h" -#include "bits.h" -#include "ptrwrap.h" +#include "../ds_core/ds_core.h" #include diff --git a/src/backend/backend.h b/src/snmalloc/backend/backend.h similarity index 96% rename from src/backend/backend.h rename to src/snmalloc/backend/backend.h index 6d33827fc..e791fe264 100644 --- a/src/backend/backend.h +++ b/src/snmalloc/backend/backend.h @@ -1,19 +1,5 @@ #pragma once -#include "../mem/allocconfig.h" -#include "../mem/metadata.h" -#include "../pal/pal.h" -#include "commitrange.h" -#include "commonconfig.h" -#include "empty_range.h" -#include "globalrange.h" -#include "largebuddyrange.h" -#include "pagemap.h" -#include "pagemapregisterrange.h" -#include "palrange.h" -#include "range_helpers.h" -#include "smallbuddyrange.h" -#include "statsrange.h" -#include "subrange.h" +#include "../backend_helpers/backend_helpers.h" #if defined(SNMALLOC_CHECK_CLIENT) && !defined(OPEN_ENCLAVE) /** diff --git a/src/backend/fixedglobalconfig.h b/src/snmalloc/backend/fixedglobalconfig.h similarity index 95% rename from src/backend/fixedglobalconfig.h rename to src/snmalloc/backend/fixedglobalconfig.h index 5b9cbf5d2..f0e8a570a 100644 --- a/src/backend/fixedglobalconfig.h +++ b/src/snmalloc/backend/fixedglobalconfig.h @@ -1,9 +1,6 @@ #pragma once #include "../backend/backend.h" -#include "../mem/corealloc.h" -#include "../mem/pool.h" -#include "commonconfig.h" namespace snmalloc { diff --git a/src/backend/globalconfig.h b/src/snmalloc/backend/globalconfig.h similarity index 79% rename from src/backend/globalconfig.h rename to src/snmalloc/backend/globalconfig.h index 31176621c..6e88f1751 100644 --- a/src/backend/globalconfig.h +++ b/src/snmalloc/backend/globalconfig.h @@ -1,19 +1,17 @@ #pragma once +// If you define SNMALLOC_PROVIDE_OWN_CONFIG then you must provide your own +// definition of `snmalloc::Alloc` before including any files that include +// `snmalloc.h` or consume the global allocation APIs. +#ifndef SNMALLOC_PROVIDE_OWN_CONFIG -#include "../backend/backend.h" -#include "../mem/corealloc.h" -#include "../mem/pool.h" -#include "commonconfig.h" +# include "../backend/backend.h" -#ifdef SNMALLOC_TRACING -# include -#endif namespace snmalloc { // Forward reference to thread local cleanup. void register_clean_up(); -#ifdef USE_SNMALLOC_STATS +# ifdef USE_SNMALLOC_STATS inline static void print_stats() { printf("No Stats yet!"); @@ -21,7 +19,7 @@ namespace snmalloc // current_alloc_pool()->aggregate_stats(s); // s.print(std::cout); } -#endif +# endif /** * The default configuration for a global snmalloc. This allocates memory @@ -59,9 +57,9 @@ namespace snmalloc static void ensure_init() { FlagLock lock{initialisation_lock}; -#ifdef SNMALLOC_TRACING +# ifdef SNMALLOC_TRACING message<1024>("Run init_impl"); -#endif +# endif if (initialised) return; @@ -74,9 +72,9 @@ namespace snmalloc // Need to initialise pagemap. Backend::init(); -#ifdef USE_SNMALLOC_STATS +# ifdef USE_SNMALLOC_STATS atexit(snmalloc::print_stats); -#endif +# endif initialised = true; } @@ -96,3 +94,10 @@ namespace snmalloc } }; } // namespace snmalloc + +// The default configuration for snmalloc +namespace snmalloc +{ + using Alloc = snmalloc::LocalAllocator; +} // namespace snmalloc +#endif diff --git a/src/snmalloc/backend_helpers/backend_helpers.h b/src/snmalloc/backend_helpers/backend_helpers.h new file mode 100644 index 000000000..06f0c9206 --- /dev/null +++ b/src/snmalloc/backend_helpers/backend_helpers.h @@ -0,0 +1,14 @@ +#include "../mem/mem.h" +#include "buddy.h" +#include "commitrange.h" +#include "commonconfig.h" +#include "empty_range.h" +#include "globalrange.h" +#include "largebuddyrange.h" +#include "pagemap.h" +#include "pagemapregisterrange.h" +#include "palrange.h" +#include "range_helpers.h" +#include "smallbuddyrange.h" +#include "statsrange.h" +#include "subrange.h" diff --git a/src/backend/buddy.h b/src/snmalloc/backend_helpers/buddy.h similarity index 96% rename from src/backend/buddy.h rename to src/snmalloc/backend_helpers/buddy.h index 7e94c845a..1c9b8e4af 100644 --- a/src/backend/buddy.h +++ b/src/snmalloc/backend_helpers/buddy.h @@ -1,8 +1,6 @@ #pragma once -#include "../ds/address.h" -#include "../ds/bits.h" -#include "../ds/redblacktree.h" +#include "../ds/ds.h" namespace snmalloc { @@ -119,4 +117,4 @@ namespace snmalloc return bigger; } }; -} // namespace snmalloc \ No newline at end of file +} // namespace snmalloc diff --git a/src/backend/commitrange.h b/src/snmalloc/backend_helpers/commitrange.h similarity index 94% rename from src/backend/commitrange.h rename to src/snmalloc/backend_helpers/commitrange.h index a5b24d92a..c9cdb39b1 100644 --- a/src/backend/commitrange.h +++ b/src/snmalloc/backend_helpers/commitrange.h @@ -1,7 +1,6 @@ #pragma once -#include "../ds/ptrwrap.h" -#include "../pal/pal_consts.h" +#include "../pal/pal.h" namespace snmalloc { diff --git a/src/backend/commonconfig.h b/src/snmalloc/backend_helpers/commonconfig.h similarity index 66% rename from src/backend/commonconfig.h rename to src/snmalloc/backend_helpers/commonconfig.h index 66314dbf1..c8e55e1a0 100644 --- a/src/backend/commonconfig.h +++ b/src/snmalloc/backend_helpers/commonconfig.h @@ -1,8 +1,6 @@ #pragma once -#include "../backend/backend_concept.h" -#include "../ds/defines.h" -#include "../mem/remoteallocator.h" +#include "../mem/mem.h" namespace snmalloc { @@ -108,75 +106,5 @@ namespace snmalloc inline static RemoteAllocator unused_remote; }; - /** - * SFINAE helper. Matched only if `T` implements `is_initialised`. Calls - * it if it exists. - */ - template - SNMALLOC_FAST_PATH auto call_is_initialised(T*, int) - -> decltype(T::is_initialised()) - { - return T::is_initialised(); - } - - /** - * SFINAE helper. Matched only if `T` does not implement `is_initialised`. - * Unconditionally returns true if invoked. - */ - template - SNMALLOC_FAST_PATH auto call_is_initialised(T*, long) - { - return true; - } - - namespace detail - { - /** - * SFINAE helper, calls capptr_domesticate in the backend if it exists. - */ - template< - SNMALLOC_CONCEPT(ConceptBackendDomestication) Backend, - typename T, - SNMALLOC_CONCEPT(capptr::ConceptBound) B> - SNMALLOC_FAST_PATH_INLINE auto - capptr_domesticate(typename Backend::LocalState* ls, CapPtr p, int) - -> decltype(Backend::capptr_domesticate(ls, p)) - { - return Backend::capptr_domesticate(ls, p); - } - - /** - * SFINAE helper. If the back end does not provide special handling for - * domestication then assume all wild pointers can be domesticated. - */ - template< - SNMALLOC_CONCEPT(ConceptBackendGlobals) Backend, - typename T, - SNMALLOC_CONCEPT(capptr::ConceptBound) B> - SNMALLOC_FAST_PATH_INLINE auto - capptr_domesticate(typename Backend::LocalState*, CapPtr p, long) - { - return CapPtr< - T, - typename B::template with_wildness>( - p.unsafe_ptr()); - } - } // namespace detail - - /** - * Wrapper that calls `Backend::capptr_domesticate` if and only if it is - * implemented. If it is not implemented then this assumes that any wild - * pointer can be domesticated. - */ - template< - SNMALLOC_CONCEPT(ConceptBackendGlobals) Backend, - typename T, - SNMALLOC_CONCEPT(capptr::ConceptBound) B> - SNMALLOC_FAST_PATH_INLINE auto - capptr_domesticate(typename Backend::LocalState* ls, CapPtr p) - { - return detail::capptr_domesticate(ls, p, 0); - } - } // namespace snmalloc #include "../mem/remotecache.h" diff --git a/src/backend/empty_range.h b/src/snmalloc/backend_helpers/empty_range.h similarity index 93% rename from src/backend/empty_range.h rename to src/snmalloc/backend_helpers/empty_range.h index 5287cd65d..b5102a1e5 100644 --- a/src/backend/empty_range.h +++ b/src/snmalloc/backend_helpers/empty_range.h @@ -1,5 +1,5 @@ #pragma once -#include "../ds/ptrwrap.h" +#include "../ds_core/ds_core.h" namespace snmalloc { diff --git a/src/backend/globalrange.h b/src/snmalloc/backend_helpers/globalrange.h similarity index 91% rename from src/backend/globalrange.h rename to src/snmalloc/backend_helpers/globalrange.h index d8ae667c2..016701eba 100644 --- a/src/backend/globalrange.h +++ b/src/snmalloc/backend_helpers/globalrange.h @@ -1,8 +1,6 @@ #pragma once -#include "../ds/defines.h" -#include "../ds/helpers.h" -#include "../ds/ptrwrap.h" +#include "../ds/ds.h" namespace snmalloc { @@ -53,4 +51,4 @@ namespace snmalloc parent->dealloc_range(base, size); } }; -} // namespace snmalloc \ No newline at end of file +} // namespace snmalloc diff --git a/src/backend/largebuddyrange.h b/src/snmalloc/backend_helpers/largebuddyrange.h similarity index 98% rename from src/backend/largebuddyrange.h rename to src/snmalloc/backend_helpers/largebuddyrange.h index a57ed6973..e3b5eccca 100644 --- a/src/backend/largebuddyrange.h +++ b/src/snmalloc/backend_helpers/largebuddyrange.h @@ -1,10 +1,7 @@ #pragma once -#include "../ds/address.h" -#include "../ds/bits.h" -#include "../mem/allocconfig.h" -#include "../pal/pal.h" -#include "backend_concept.h" +#include "../ds/ds.h" +#include "../mem/mem.h" #include "buddy.h" #include "range_helpers.h" diff --git a/src/backend/pagemap.h b/src/snmalloc/backend_helpers/pagemap.h similarity index 98% rename from src/backend/pagemap.h rename to src/snmalloc/backend_helpers/pagemap.h index 6bb259d6e..ec7bbdfdb 100644 --- a/src/backend/pagemap.h +++ b/src/snmalloc/backend_helpers/pagemap.h @@ -1,9 +1,7 @@ #pragma once -#include "../ds/bits.h" -#include "../ds/helpers.h" -#include "../mem/entropy.h" -#include "../pal/pal.h" +#include "../ds/ds.h" +#include "../mem/mem.h" #include #include diff --git a/src/backend/pagemapregisterrange.h b/src/snmalloc/backend_helpers/pagemapregisterrange.h similarity index 94% rename from src/backend/pagemapregisterrange.h rename to src/snmalloc/backend_helpers/pagemapregisterrange.h index 4084eed86..059cc157f 100644 --- a/src/backend/pagemapregisterrange.h +++ b/src/snmalloc/backend_helpers/pagemapregisterrange.h @@ -1,6 +1,5 @@ #pragma once -#include "../ds/address.h" #include "../pal/pal.h" namespace snmalloc @@ -42,4 +41,4 @@ namespace snmalloc return base; } }; -} // namespace snmalloc \ No newline at end of file +} // namespace snmalloc diff --git a/src/backend/palrange.h b/src/snmalloc/backend_helpers/palrange.h similarity index 96% rename from src/backend/palrange.h rename to src/snmalloc/backend_helpers/palrange.h index 5fa3fe6c8..f4473254b 100644 --- a/src/backend/palrange.h +++ b/src/snmalloc/backend_helpers/palrange.h @@ -1,5 +1,4 @@ #pragma once -#include "../ds/address.h" #include "../pal/pal.h" namespace snmalloc @@ -61,4 +60,4 @@ namespace snmalloc } } }; -} // namespace snmalloc \ No newline at end of file +} // namespace snmalloc diff --git a/src/backend/range_helpers.h b/src/snmalloc/backend_helpers/range_helpers.h similarity index 95% rename from src/backend/range_helpers.h rename to src/snmalloc/backend_helpers/range_helpers.h index 1dd6ea4d4..a9dc43c86 100644 --- a/src/backend/range_helpers.h +++ b/src/snmalloc/backend_helpers/range_helpers.h @@ -1,6 +1,6 @@ #pragma once -#include "../ds/ptrwrap.h" +#include "../ds_core/ds_core.h" namespace snmalloc { @@ -34,4 +34,4 @@ namespace snmalloc length -= align; } } -} // namespace snmalloc \ No newline at end of file +} // namespace snmalloc diff --git a/src/backend/smallbuddyrange.h b/src/snmalloc/backend_helpers/smallbuddyrange.h similarity index 99% rename from src/backend/smallbuddyrange.h rename to src/snmalloc/backend_helpers/smallbuddyrange.h index 2500e1f2c..b3ccc4544 100644 --- a/src/backend/smallbuddyrange.h +++ b/src/snmalloc/backend_helpers/smallbuddyrange.h @@ -1,7 +1,5 @@ #pragma once -#include "../ds/address.h" -#include "../mem/allocconfig.h" #include "../pal/pal.h" #include "range_helpers.h" diff --git a/src/backend/statsrange.h b/src/snmalloc/backend_helpers/statsrange.h similarity index 100% rename from src/backend/statsrange.h rename to src/snmalloc/backend_helpers/statsrange.h diff --git a/src/backend/subrange.h b/src/snmalloc/backend_helpers/subrange.h similarity index 96% rename from src/backend/subrange.h rename to src/snmalloc/backend_helpers/subrange.h index dd4824339..44c1db9d5 100644 --- a/src/backend/subrange.h +++ b/src/snmalloc/backend_helpers/subrange.h @@ -1,5 +1,5 @@ #pragma once -#include "../mem/entropy.h" +#include "../mem/mem.h" namespace snmalloc { @@ -54,4 +54,4 @@ namespace snmalloc return pointer_offset(overblock, offset); } }; -} // namespace snmalloc \ No newline at end of file +} // namespace snmalloc diff --git a/src/ds/aba.h b/src/snmalloc/ds/aba.h similarity index 99% rename from src/ds/aba.h rename to src/snmalloc/ds/aba.h index 64b0f9955..51c447035 100644 --- a/src/ds/aba.h +++ b/src/snmalloc/ds/aba.h @@ -1,7 +1,6 @@ #pragma once -#include "bits.h" -#include "ptrwrap.h" +#include "../aal/aal.h" /** * This file contains an abstraction of ABA protection. This API should be diff --git a/src/mem/allocconfig.h b/src/snmalloc/ds/allocconfig.h similarity index 98% rename from src/mem/allocconfig.h rename to src/snmalloc/ds/allocconfig.h index a83d2a8d1..2186b78f9 100644 --- a/src/mem/allocconfig.h +++ b/src/snmalloc/ds/allocconfig.h @@ -1,8 +1,5 @@ #pragma once -#include "../ds/bits.h" -#include "../pal/pal.h" - namespace snmalloc { // 0 intermediate bits results in power of 2 small allocs. 1 intermediate diff --git a/src/snmalloc/ds/ds.h b/src/snmalloc/ds/ds.h new file mode 100644 index 000000000..432277dcb --- /dev/null +++ b/src/snmalloc/ds/ds.h @@ -0,0 +1,11 @@ +/** + * Data structures used by snmalloc. + * + */ +#pragma once +#include "../pal/pal.h" +#include "aba.h" +#include "allocconfig.h" +#include "flaglock.h" +#include "mpmcstack.h" +#include "singleton.h" diff --git a/src/ds/flaglock.h b/src/snmalloc/ds/flaglock.h similarity index 98% rename from src/ds/flaglock.h rename to src/snmalloc/ds/flaglock.h index c25f1eed6..5fbbf0a0a 100644 --- a/src/ds/flaglock.h +++ b/src/snmalloc/ds/flaglock.h @@ -1,8 +1,9 @@ #pragma once -#include "bits.h" +#include "../aal/aal.h" #include +#include namespace snmalloc { diff --git a/src/ds/mpmcstack.h b/src/snmalloc/ds/mpmcstack.h similarity index 97% rename from src/ds/mpmcstack.h rename to src/snmalloc/ds/mpmcstack.h index 48af89354..cd005e9bf 100644 --- a/src/ds/mpmcstack.h +++ b/src/snmalloc/ds/mpmcstack.h @@ -1,7 +1,8 @@ #pragma once +#include "../ds_core/ds_core.h" #include "aba.h" -#include "ptrwrap.h" +#include "allocconfig.h" #if defined(__has_feature) # if __has_feature(thread_sanitizer) diff --git a/src/snmalloc/ds/singleton.h b/src/snmalloc/ds/singleton.h new file mode 100644 index 000000000..c85635d39 --- /dev/null +++ b/src/snmalloc/ds/singleton.h @@ -0,0 +1,51 @@ +#pragma once + +#include "../ds_core/ds_core.h" +#include "flaglock.h" + +#include +#include +#include +#include + +namespace snmalloc +{ + /* + * In some use cases we need to run before any of the C++ runtime has been + * initialised. This singleton class is designed to not depend on the + * runtime. + */ + template + class Singleton + { + inline static FlagWord flag; + inline static std::atomic initialised{false}; + inline static Object obj; + + public: + /** + * If argument is non-null, then it is assigned the value + * true, if this is the first call to get. + * At most one call will be first. + */ + inline SNMALLOC_SLOW_PATH static Object& get(bool* first = nullptr) + { + // If defined should be initially false; + SNMALLOC_ASSERT(first == nullptr || *first == false); + + if (SNMALLOC_UNLIKELY(!initialised.load(std::memory_order_acquire))) + { + FlagLock lock(flag); + if (!initialised) + { + init(&obj); + initialised.store(true, std::memory_order_release); + if (first != nullptr) + *first = true; + } + } + return obj; + } + }; + +} // namespace snmalloc diff --git a/src/ds/bits.h b/src/snmalloc/ds_core/bits.h similarity index 87% rename from src/ds/bits.h rename to src/snmalloc/ds_core/bits.h index 455ec4ab7..f1dc4ffd6 100644 --- a/src/ds/bits.h +++ b/src/snmalloc/ds_core/bits.h @@ -5,13 +5,16 @@ // #define USE_LZCNT -#include "../aal/aal.h" -#include "../pal/pal_consts.h" #include "defines.h" #include +#include #include #include +#if defined(_MSC_VER) +# include +#endif + #ifdef pause # undef pause #endif @@ -30,7 +33,16 @@ namespace snmalloc namespace bits { - static constexpr size_t BITS = Aal::bits; + /** + * The number of bits in a `size_t`. Many of the functions in the + * `snmalloc::bits` namespace are defined to operate over `size_t`, mapping + * to the correct compiler builtins irrespective of the size. This size + * does *not* imply the number of bits of virtual address space that are + * actually allowed to be set. `Aal::bits` provides an + * architecture-specific definition of the number of bits of address space + * that exist. + */ + static constexpr size_t BITS = sizeof(size_t) * CHAR_BIT; /** * Returns a value of type T that has a single bit set, @@ -52,7 +64,7 @@ namespace snmalloc SNMALLOC_ASSERT(x != 0); // Calling with 0 is UB on some implementations #if defined(_MSC_VER) # ifdef USE_LZCNT -# ifdef SNMALLOC_VA_BITS_64 +# ifdef _WIN64 return __lzcnt64(x); # else return __lzcnt((uint32_t)x); @@ -60,10 +72,10 @@ namespace snmalloc # else unsigned long index; -# ifdef SNMALLOC_VA_BITS_64 - _BitScanReverse64(&index, x); +# ifdef _WIN64 + _BitScanReverse64(&index, static_cast(x)); # else - _BitScanReverse(&index, (unsigned long)x); + _BitScanReverse(&index, static_cast(x)); # endif return BITS - index - 1; @@ -101,10 +113,10 @@ namespace snmalloc inline size_t rotr(size_t x, size_t n) { #if defined(_MSC_VER) -# ifdef SNMALLOC_VA_BITS_64 - return _rotr64(x, (int)n); +# ifdef _WIN64 + return _rotr64(static_cast(x), static_cast(n)); # else - return _rotr((uint32_t)x, (int)n); + return _rotr(static_cast(x), static_cast(n)); # endif #else return rotr_const(x, n); @@ -114,10 +126,10 @@ namespace snmalloc inline size_t rotl(size_t x, size_t n) { #if defined(_MSC_VER) -# ifdef SNMALLOC_VA_BITS_64 - return _rotl64(x, (int)n); +# ifdef _WIN64 + return _rotl64(static_cast(x), static_cast(n)); # else - return _rotl((uint32_t)x, (int)n); + return _rotl(static_cast(x), static_cast(n)); # endif #else return rotl_const(x, n); @@ -145,11 +157,11 @@ namespace snmalloc { SNMALLOC_ASSERT(x != 0); // Calling with 0 is UB on some implementations -#if defined(_MSC_VER) -# ifdef SNMALLOC_VA_BITS_64 - return _tzcnt_u64(x); +#if defined(_MSC_VER) && !defined(__clang__) +# ifdef _WIN64 + return _tzcnt_u64(static_cast(x)); # else - return _tzcnt_u32((uint32_t)x); + return _tzcnt_u32(static_cast(x)); # endif #else if constexpr (std::is_same_v) @@ -191,14 +203,14 @@ namespace snmalloc overflow = __builtin_mul_overflow(x, y, &prod); return prod; #elif defined(_MSC_VER) -# if defined(SNMALLOC_VA_BITS_64) +# ifdef _WIN64 size_t high_prod; size_t prod = _umul128(x, y, &high_prod); overflow = high_prod != 0; return prod; # else - size_t prod; - overflow = S_OK != UIntMult(x, y, &prod); + UINT prod; + overflow = S_OK != UIntMult(UINT(x), UINT(y), &prod); return prod; # endif #else diff --git a/src/ds/concept.h b/src/snmalloc/ds_core/concept.h similarity index 100% rename from src/ds/concept.h rename to src/snmalloc/ds_core/concept.h diff --git a/src/ds/defines.h b/src/snmalloc/ds_core/defines.h similarity index 100% rename from src/ds/defines.h rename to src/snmalloc/ds_core/defines.h diff --git a/src/snmalloc/ds_core/ds_core.h b/src/snmalloc/ds_core/ds_core.h new file mode 100644 index 000000000..672f7d1b0 --- /dev/null +++ b/src/snmalloc/ds_core/ds_core.h @@ -0,0 +1,16 @@ +#pragma once +/** + * The core definitions for snmalloc. These provide some basic helpers that do + * not depend on anything except for a working C++ implementation. + * + * Files in this directory may not include anything from any other directory in + * snmalloc. + */ + +#include "bits.h" +#include "concept.h" +#include "defines.h" +#include "helpers.h" +#include "ptrwrap.h" +#include "redblacktree.h" +#include "seqset.h" diff --git a/src/ds/helpers.h b/src/snmalloc/ds_core/helpers.h similarity index 90% rename from src/ds/helpers.h rename to src/snmalloc/ds_core/helpers.h index f599aacb8..693b734e3 100644 --- a/src/ds/helpers.h +++ b/src/snmalloc/ds_core/helpers.h @@ -1,53 +1,15 @@ #pragma once #include "bits.h" -#include "flaglock.h" #include #include +#include #include #include namespace snmalloc { - /* - * In some use cases we need to run before any of the C++ runtime has been - * initialised. This singleton class is designed to not depend on the - * runtime. - */ - template - class Singleton - { - inline static FlagWord flag; - inline static std::atomic initialised{false}; - inline static Object obj; - - public: - /** - * If argument is non-null, then it is assigned the value - * true, if this is the first call to get. - * At most one call will be first. - */ - inline SNMALLOC_SLOW_PATH static Object& get(bool* first = nullptr) - { - // If defined should be initially false; - SNMALLOC_ASSERT(first == nullptr || *first == false); - - if (SNMALLOC_UNLIKELY(!initialised.load(std::memory_order_acquire))) - { - FlagLock lock(flag); - if (!initialised) - { - init(&obj); - initialised.store(true, std::memory_order_release); - if (first != nullptr) - *first = true; - } - } - return obj; - } - }; - /** * Wrapper for wrapping values. * @@ -477,4 +439,5 @@ namespace snmalloc */ struct Empty {}; + } // namespace snmalloc diff --git a/src/ds/ptrwrap.h b/src/snmalloc/ds_core/ptrwrap.h similarity index 99% rename from src/ds/ptrwrap.h rename to src/snmalloc/ds_core/ptrwrap.h index b94c90f4b..e7aa85c9a 100644 --- a/src/ds/ptrwrap.h +++ b/src/snmalloc/ds_core/ptrwrap.h @@ -1,7 +1,7 @@ #pragma once -#include "../ds/concept.h" -#include "../ds/defines.h" +#include "concept.h" +#include "defines.h" #include diff --git a/src/ds/redblacktree.h b/src/snmalloc/ds_core/redblacktree.h similarity index 99% rename from src/ds/redblacktree.h rename to src/snmalloc/ds_core/redblacktree.h index 3d87f9e3f..0d684698d 100644 --- a/src/ds/redblacktree.h +++ b/src/snmalloc/ds_core/redblacktree.h @@ -1,7 +1,4 @@ #pragma once -#include "../pal/pal.h" -#include "concept.h" -#include "defines.h" #include #include diff --git a/src/ds/seqset.h b/src/snmalloc/ds_core/seqset.h similarity index 98% rename from src/ds/seqset.h rename to src/snmalloc/ds_core/seqset.h index 510961c32..22a0fcd07 100644 --- a/src/ds/seqset.h +++ b/src/snmalloc/ds_core/seqset.h @@ -1,8 +1,6 @@ #pragma once -#include "address.h" -#include "defines.h" -#include "ptrwrap.h" +#include "../ds_core/ds_core.h" #include #include diff --git a/src/mem/bounds_checks.h b/src/snmalloc/global/bounds_checks.h similarity index 98% rename from src/mem/bounds_checks.h rename to src/snmalloc/global/bounds_checks.h index 2843530bc..66b67ec43 100644 --- a/src/mem/bounds_checks.h +++ b/src/snmalloc/global/bounds_checks.h @@ -1,5 +1,5 @@ #pragma once -#include "../snmalloc.h" +#include "threadalloc.h" namespace snmalloc { @@ -107,4 +107,4 @@ namespace snmalloc } } -} +} // namespace snmalloc diff --git a/src/snmalloc/global/global.h b/src/snmalloc/global/global.h new file mode 100644 index 000000000..a2f1159a1 --- /dev/null +++ b/src/snmalloc/global/global.h @@ -0,0 +1,4 @@ +#include "bounds_checks.h" +#include "memcpy.h" +#include "scopedalloc.h" +#include "threadalloc.h" diff --git a/src/mem/memcpy.h b/src/snmalloc/global/memcpy.h similarity index 99% rename from src/mem/memcpy.h rename to src/snmalloc/global/memcpy.h index 30eb7fbf6..beba37cd2 100644 --- a/src/mem/memcpy.h +++ b/src/snmalloc/global/memcpy.h @@ -1,5 +1,6 @@ #pragma once -#include "../mem/bounds_checks.h" +#include "../backend/globalconfig.h" +#include "bounds_checks.h" namespace snmalloc { @@ -328,4 +329,4 @@ namespace snmalloc Arch::copy(dst, src, len); return orig_dst; } -} // namespace +} // namespace snmalloc diff --git a/src/mem/scopedalloc.h b/src/snmalloc/global/scopedalloc.h similarity index 98% rename from src/mem/scopedalloc.h rename to src/snmalloc/global/scopedalloc.h index 345635a70..cb9f0fc8b 100644 --- a/src/mem/scopedalloc.h +++ b/src/snmalloc/global/scopedalloc.h @@ -1,4 +1,5 @@ #pragma once +#include "../backend/globalconfig.h" /** * This header requires that Alloc has been defined. diff --git a/src/mem/threadalloc.h b/src/snmalloc/global/threadalloc.h similarity index 99% rename from src/mem/threadalloc.h rename to src/snmalloc/global/threadalloc.h index bbda32d7f..d900fb272 100644 --- a/src/mem/threadalloc.h +++ b/src/snmalloc/global/threadalloc.h @@ -1,7 +1,6 @@ #pragma once -#include "../ds/helpers.h" -#include "localalloc.h" +#include "../backend/globalconfig.h" #if defined(SNMALLOC_EXTERNAL_THREAD_ALLOC) # define SNMALLOC_THREAD_TEARDOWN_DEFINED diff --git a/src/backend/backend_concept.h b/src/snmalloc/mem/backend_concept.h similarity index 97% rename from src/backend/backend_concept.h rename to src/snmalloc/mem/backend_concept.h index 27aa8df39..e29a2df8b 100644 --- a/src/backend/backend_concept.h +++ b/src/snmalloc/mem/backend_concept.h @@ -1,9 +1,7 @@ #pragma once #ifdef __cpp_concepts -# include "../ds/address.h" -# include "../ds/concept.h" -# include "../pal/pal_concept.h" +# include "../ds/ds.h" # include namespace snmalloc diff --git a/src/snmalloc/mem/backend_wrappers.h b/src/snmalloc/mem/backend_wrappers.h new file mode 100644 index 000000000..36657dfe5 --- /dev/null +++ b/src/snmalloc/mem/backend_wrappers.h @@ -0,0 +1,85 @@ +#pragma once +/** + * Several of the functions provided by the back end are optional. This file + * contains helpers that are templated on a back end and either call the + * corresponding function or do nothing. This allows the rest of the front end + * to assume that these functions always exist and avoid the need for `if + * constexpr` clauses everywhere. The no-op versions are always inlined and so + * will be optimised away. + */ + +#include "../ds_core/ds_core.h" + +namespace snmalloc +{ + /** + * SFINAE helper. Matched only if `T` implements `is_initialised`. Calls + * it if it exists. + */ + template + SNMALLOC_FAST_PATH auto call_is_initialised(T*, int) + -> decltype(T::is_initialised()) + { + return T::is_initialised(); + } + + /** + * SFINAE helper. Matched only if `T` does not implement `is_initialised`. + * Unconditionally returns true if invoked. + */ + template + SNMALLOC_FAST_PATH auto call_is_initialised(T*, long) + { + return true; + } + + namespace detail + { + /** + * SFINAE helper, calls capptr_domesticate in the backend if it exists. + */ + template< + SNMALLOC_CONCEPT(ConceptBackendDomestication) Backend, + typename T, + SNMALLOC_CONCEPT(capptr::ConceptBound) B> + SNMALLOC_FAST_PATH_INLINE auto + capptr_domesticate(typename Backend::LocalState* ls, CapPtr p, int) + -> decltype(Backend::capptr_domesticate(ls, p)) + { + return Backend::capptr_domesticate(ls, p); + } + + /** + * SFINAE helper. If the back end does not provide special handling for + * domestication then assume all wild pointers can be domesticated. + */ + template< + SNMALLOC_CONCEPT(ConceptBackendGlobals) Backend, + typename T, + SNMALLOC_CONCEPT(capptr::ConceptBound) B> + SNMALLOC_FAST_PATH_INLINE auto + capptr_domesticate(typename Backend::LocalState*, CapPtr p, long) + { + return CapPtr< + T, + typename B::template with_wildness>( + p.unsafe_ptr()); + } + } // namespace detail + + /** + * Wrapper that calls `Backend::capptr_domesticate` if and only if it is + * implemented. If it is not implemented then this assumes that any wild + * pointer can be domesticated. + */ + template< + SNMALLOC_CONCEPT(ConceptBackendGlobals) Backend, + typename T, + SNMALLOC_CONCEPT(capptr::ConceptBound) B> + SNMALLOC_FAST_PATH_INLINE auto + capptr_domesticate(typename Backend::LocalState* ls, CapPtr p) + { + return detail::capptr_domesticate(ls, p, 0); + } + +} // namespace snmalloc diff --git a/src/mem/corealloc.h b/src/snmalloc/mem/corealloc.h similarity index 99% rename from src/mem/corealloc.h rename to src/snmalloc/mem/corealloc.h index a1fc5c5e4..4baab6f50 100644 --- a/src/mem/corealloc.h +++ b/src/snmalloc/mem/corealloc.h @@ -1,7 +1,6 @@ #pragma once -#include "../ds/defines.h" -#include "allocconfig.h" +#include "../ds/ds.h" #include "localcache.h" #include "metadata.h" #include "pool.h" diff --git a/src/mem/entropy.h b/src/snmalloc/mem/entropy.h similarity index 99% rename from src/mem/entropy.h rename to src/snmalloc/mem/entropy.h index 7f1491950..1b590942f 100644 --- a/src/mem/entropy.h +++ b/src/snmalloc/mem/entropy.h @@ -1,6 +1,5 @@ #pragma once -#include "../ds/address.h" #include "../pal/pal.h" #include diff --git a/src/mem/external_alloc.h b/src/snmalloc/mem/external_alloc.h similarity index 100% rename from src/mem/external_alloc.h rename to src/snmalloc/mem/external_alloc.h diff --git a/src/mem/freelist.h b/src/snmalloc/mem/freelist.h similarity index 99% rename from src/mem/freelist.h rename to src/snmalloc/mem/freelist.h index 9850b6126..c310153a5 100644 --- a/src/mem/freelist.h +++ b/src/snmalloc/mem/freelist.h @@ -33,8 +33,7 @@ * and randomly deciding which list to add an element to. */ -#include "../ds/address.h" -#include "allocconfig.h" +#include "../ds/ds.h" #include "entropy.h" #include diff --git a/src/mem/globalalloc.h b/src/snmalloc/mem/globalalloc.h similarity index 72% rename from src/mem/globalalloc.h rename to src/snmalloc/mem/globalalloc.h index 0152f6cb2..b898eed9f 100644 --- a/src/mem/globalalloc.h +++ b/src/snmalloc/mem/globalalloc.h @@ -1,53 +1,10 @@ #pragma once -#include "../ds/helpers.h" +#include "../ds_core/ds_core.h" #include "localalloc.h" namespace snmalloc { - template - inline static void aggregate_stats(Stats& stats) - { - static_assert( - SharedStateHandle::Options.CoreAllocIsPoolAllocated, - "Global statistics are available only for pool-allocated configurations"); - auto* alloc = AllocPool::iterate(); - - while (alloc != nullptr) - { - auto a = alloc->attached_stats(); - if (a != nullptr) - stats.add(*a); - stats.add(alloc->stats()); - alloc = AllocPool::iterate(alloc); - } - } - -#ifdef USE_SNMALLOC_STATS - template - inline static void print_all_stats(std::ostream& o, uint64_t dumpid = 0) - { - static_assert( - SharedStateHandle::Options.CoreAllocIsPoolAllocated, - "Global statistics are available only for pool-allocated configurations"); - auto alloc = AllocPool::iterate(); - - while (alloc != nullptr) - { - auto stats = alloc->stats(); - if (stats != nullptr) - stats->template print(o, dumpid, alloc->id()); - alloc = AllocPool::iterate(alloc); - } - } -#else - template - inline static void print_all_stats(void*& o, uint64_t dumpid = 0) - { - UNUSED(o, dumpid); - } -#endif - template inline static void cleanup_unused() { diff --git a/src/mem/localalloc.h b/src/snmalloc/mem/localalloc.h similarity index 99% rename from src/mem/localalloc.h rename to src/snmalloc/mem/localalloc.h index bbe0781db..086fd8a01 100644 --- a/src/mem/localalloc.h +++ b/src/snmalloc/mem/localalloc.h @@ -6,7 +6,7 @@ # define ALLOCATOR #endif -#include "../ds/ptrwrap.h" +#include "../ds/ds.h" #include "corealloc.h" #include "freelist.h" #include "localcache.h" @@ -18,9 +18,6 @@ # include "external_alloc.h" #endif -#ifdef SNMALLOC_TRACING -# include -#endif #include #include namespace snmalloc diff --git a/src/mem/localcache.h b/src/snmalloc/mem/localcache.h similarity index 90% rename from src/mem/localcache.h rename to src/snmalloc/mem/localcache.h index 9b3f174d4..2ae8ffd68 100644 --- a/src/mem/localcache.h +++ b/src/snmalloc/mem/localcache.h @@ -1,7 +1,6 @@ #pragma once -#include "../ds/ptrwrap.h" -#include "allocstats.h" +#include "../ds/ds.h" #include "freelist.h" #include "remotecache.h" #include "sizeclasstable.h" @@ -10,8 +9,6 @@ namespace snmalloc { - using Stats = AllocStats; - inline static SNMALLOC_FAST_PATH capptr::Alloc finish_alloc_no_zero(freelist::HeadPtr p, smallsizeclass_t sizeclass) { @@ -51,10 +48,6 @@ namespace snmalloc // This is the entropy for a particular thread. LocalEntropy entropy; - // TODO: Minimal stats object for just the stats on this datastructure. - // This will be a zero-size structure if stats are not enabled. - Stats stats; - // Pointer to the remote allocator message_queue, used to check // if a deallocation is local. RemoteAllocator* remote_allocator; @@ -111,8 +104,6 @@ namespace snmalloc { auto& key = entropy.get_free_list_key(); smallsizeclass_t sizeclass = size_to_sizeclass(size); - stats.alloc_request(size); - stats.sizeclass_alloc(sizeclass); auto& fl = small_fast_free_lists[sizeclass]; if (SNMALLOC_LIKELY(!fl.empty())) { diff --git a/src/snmalloc/mem/mem.h b/src/snmalloc/mem/mem.h new file mode 100644 index 000000000..9fb29a985 --- /dev/null +++ b/src/snmalloc/mem/mem.h @@ -0,0 +1,16 @@ +#include "backend_concept.h" +#include "backend_wrappers.h" +#include "corealloc.h" +#include "entropy.h" +#include "external_alloc.h" +#include "freelist.h" +#include "globalalloc.h" +#include "localalloc.h" +#include "localcache.h" +#include "metadata.h" +#include "pool.h" +#include "pooled.h" +#include "remoteallocator.h" +#include "remotecache.h" +#include "sizeclasstable.h" +#include "ticker.h" diff --git a/src/mem/metadata.h b/src/snmalloc/mem/metadata.h similarity index 99% rename from src/mem/metadata.h rename to src/snmalloc/mem/metadata.h index 765e2989f..ac56f0ab4 100644 --- a/src/mem/metadata.h +++ b/src/snmalloc/mem/metadata.h @@ -1,8 +1,6 @@ #pragma once -#include "../backend/backend_concept.h" -#include "../ds/helpers.h" -#include "../ds/seqset.h" +#include "../ds/ds.h" #include "freelist.h" #include "sizeclasstable.h" diff --git a/src/mem/pool.h b/src/snmalloc/mem/pool.h similarity index 98% rename from src/mem/pool.h rename to src/snmalloc/mem/pool.h index 141a14c51..119777a7c 100644 --- a/src/mem/pool.h +++ b/src/snmalloc/mem/pool.h @@ -1,8 +1,6 @@ #pragma once -#include "../ds/flaglock.h" -#include "../ds/mpmcstack.h" -#include "../pal/pal_concept.h" +#include "../ds/ds.h" #include "pooled.h" #include diff --git a/src/mem/pooled.h b/src/snmalloc/mem/pooled.h similarity index 90% rename from src/mem/pooled.h rename to src/snmalloc/mem/pooled.h index 7b499fd81..ac1af5d1c 100644 --- a/src/mem/pooled.h +++ b/src/snmalloc/mem/pooled.h @@ -1,6 +1,7 @@ #pragma once -#include "../ds/bits.h" +#include "../ds/ds.h" +#include "backend_concept.h" namespace snmalloc { diff --git a/src/mem/remoteallocator.h b/src/snmalloc/mem/remoteallocator.h similarity index 98% rename from src/mem/remoteallocator.h rename to src/snmalloc/mem/remoteallocator.h index e851a0af9..2b92e9f6e 100644 --- a/src/mem/remoteallocator.h +++ b/src/snmalloc/mem/remoteallocator.h @@ -1,9 +1,9 @@ #pragma once -#include "../mem/allocconfig.h" -#include "../mem/freelist.h" -#include "../mem/metadata.h" -#include "../mem/sizeclasstable.h" +#include "../ds/ds.h" +#include "freelist.h" +#include "metadata.h" +#include "sizeclasstable.h" #include #include diff --git a/src/mem/remotecache.h b/src/snmalloc/mem/remotecache.h similarity index 99% rename from src/mem/remotecache.h rename to src/snmalloc/mem/remotecache.h index 116665687..01a275257 100644 --- a/src/mem/remotecache.h +++ b/src/snmalloc/mem/remotecache.h @@ -1,6 +1,7 @@ #pragma once -#include "allocconfig.h" +#include "../ds/ds.h" +#include "backend_wrappers.h" #include "freelist.h" #include "metadata.h" #include "remoteallocator.h" diff --git a/src/mem/sizeclasstable.h b/src/snmalloc/mem/sizeclasstable.h similarity index 99% rename from src/mem/sizeclasstable.h rename to src/snmalloc/mem/sizeclasstable.h index ae5554585..b257d0775 100644 --- a/src/mem/sizeclasstable.h +++ b/src/snmalloc/mem/sizeclasstable.h @@ -1,9 +1,6 @@ #pragma once -#include "../ds/bits.h" -#include "../ds/defines.h" -#include "../ds/helpers.h" -#include "allocconfig.h" +#include "../ds/ds.h" /** * This file contains all the code for transforming transforming sizes to diff --git a/src/mem/ticker.h b/src/snmalloc/mem/ticker.h similarity index 98% rename from src/mem/ticker.h rename to src/snmalloc/mem/ticker.h index 4dee55eb6..2bce041e0 100644 --- a/src/mem/ticker.h +++ b/src/snmalloc/mem/ticker.h @@ -1,6 +1,6 @@ #pragma once -#include "../ds/defines.h" +#include "../ds_core/ds_core.h" #include diff --git a/src/override/jemalloc_compat.cc b/src/snmalloc/override/jemalloc_compat.cc similarity index 100% rename from src/override/jemalloc_compat.cc rename to src/snmalloc/override/jemalloc_compat.cc diff --git a/src/override/malloc-extensions.cc b/src/snmalloc/override/malloc-extensions.cc similarity index 100% rename from src/override/malloc-extensions.cc rename to src/snmalloc/override/malloc-extensions.cc diff --git a/src/override/malloc-extensions.h b/src/snmalloc/override/malloc-extensions.h similarity index 100% rename from src/override/malloc-extensions.h rename to src/snmalloc/override/malloc-extensions.h diff --git a/src/override/malloc.cc b/src/snmalloc/override/malloc.cc similarity index 100% rename from src/override/malloc.cc rename to src/snmalloc/override/malloc.cc diff --git a/src/override/memcpy.cc b/src/snmalloc/override/memcpy.cc similarity index 90% rename from src/override/memcpy.cc rename to src/snmalloc/override/memcpy.cc index 5de742e1d..c2283ec1e 100644 --- a/src/override/memcpy.cc +++ b/src/snmalloc/override/memcpy.cc @@ -1,5 +1,3 @@ -#include "mem/memcpy.h" - #include "override.h" extern "C" diff --git a/src/override/new.cc b/src/snmalloc/override/new.cc similarity index 100% rename from src/override/new.cc rename to src/snmalloc/override/new.cc diff --git a/src/snmalloc/override/override.h b/src/snmalloc/override/override.h new file mode 100644 index 000000000..0ca70bc11 --- /dev/null +++ b/src/snmalloc/override/override.h @@ -0,0 +1,15 @@ +#pragma once + +#include "../global/global.h" + +#ifndef SNMALLOC_EXPORT +# define SNMALLOC_EXPORT +#endif +#ifdef SNMALLOC_STATIC_LIBRARY_PREFIX +# define __SN_CONCAT(a, b) a##b +# define __SN_EVALUATE(a, b) __SN_CONCAT(a, b) +# define SNMALLOC_NAME_MANGLE(a) \ + __SN_EVALUATE(SNMALLOC_STATIC_LIBRARY_PREFIX, a) +#elif !defined(SNMALLOC_NAME_MANGLE) +# define SNMALLOC_NAME_MANGLE(a) a +#endif diff --git a/src/override/rust.cc b/src/snmalloc/override/rust.cc similarity index 100% rename from src/override/rust.cc rename to src/snmalloc/override/rust.cc diff --git a/src/pal/pal.h b/src/snmalloc/pal/pal.h similarity index 89% rename from src/pal/pal.h rename to src/snmalloc/pal/pal.h index 8a0260fe3..697b22cb9 100644 --- a/src/pal/pal.h +++ b/src/snmalloc/pal/pal.h @@ -1,10 +1,22 @@ +/** + * The platform abstraction layer. This defines an abstraction that exposes + * services from the operating system or any equivalent environment. + * + * It is possible to have multiple PALs in a single snmalloc instance. For + * example, one may provide the base operating system functionality, another + * may hide some of this or provide its own abstractions for sandboxing or + * isolating heaps. + * + * Files in this directory may depend on the architecture abstraction and core + * layers (`aal` and `ds_core`, respectively) but nothing else in snmalloc. + */ #pragma once -#include "../ds/concept.h" +#include "../aal/aal.h" #include "pal_concept.h" #include "pal_consts.h" -// If simultating OE, then we need the underlying platform +// If simulating OE, then we need the underlying platform #if defined(OPEN_ENCLAVE) # include "pal_open_enclave.h" #endif diff --git a/src/pal/pal_apple.h b/src/snmalloc/pal/pal_apple.h similarity index 100% rename from src/pal/pal_apple.h rename to src/snmalloc/pal/pal_apple.h diff --git a/src/pal/pal_bsd.h b/src/snmalloc/pal/pal_bsd.h similarity index 100% rename from src/pal/pal_bsd.h rename to src/snmalloc/pal/pal_bsd.h diff --git a/src/pal/pal_bsd_aligned.h b/src/snmalloc/pal/pal_bsd_aligned.h similarity index 100% rename from src/pal/pal_bsd_aligned.h rename to src/snmalloc/pal/pal_bsd_aligned.h diff --git a/src/pal/pal_concept.h b/src/snmalloc/pal/pal_concept.h similarity index 58% rename from src/pal/pal_concept.h rename to src/snmalloc/pal/pal_concept.h index a476cce9e..78270f27c 100644 --- a/src/pal/pal_concept.h +++ b/src/snmalloc/pal/pal_concept.h @@ -1,14 +1,13 @@ #pragma once #ifdef __cpp_concepts -# include "../ds/concept.h" # include "pal_consts.h" # include "pal_ds.h" + # include namespace snmalloc { - /* * These concepts enforce that these are indeed constants that fit in the * desired types. (This is subtly different from saying that they are the @@ -41,7 +40,10 @@ namespace snmalloc template concept ConceptPAL_error = requires(const char* const str) { - { PAL::error(str) } -> ConceptSame; + { + PAL::error(str) + } + ->ConceptSame; }; /** @@ -50,25 +52,40 @@ namespace snmalloc template concept ConceptPAL_memops = requires(void* vp, std::size_t sz) { - { PAL::notify_not_using(vp, sz) } noexcept -> ConceptSame; + { + PAL::notify_not_using(vp, sz) + } + noexcept->ConceptSame; - { PAL::template notify_using(vp, sz) } noexcept - -> ConceptSame; - { PAL::template notify_using(vp, sz) } noexcept - -> ConceptSame; + { + PAL::template notify_using(vp, sz) + } + noexcept->ConceptSame; + { + PAL::template notify_using(vp, sz) + } + noexcept->ConceptSame; - { PAL::template zero(vp, sz) } noexcept -> ConceptSame; - { PAL::template zero(vp, sz) } noexcept -> ConceptSame; + { + PAL::template zero(vp, sz) + } + noexcept->ConceptSame; + { + PAL::template zero(vp, sz) + } + noexcept->ConceptSame; }; /** * Absent any feature flags, the PAL must support a crude primitive allocator */ template - concept ConceptPAL_reserve = - requires(PAL p, std::size_t sz) + concept ConceptPAL_reserve = requires(PAL p, std::size_t sz) { - { PAL::reserve(sz) } noexcept -> ConceptSame; + { + PAL::reserve(sz) + } + noexcept->ConceptSame; }; /** @@ -77,9 +94,14 @@ namespace snmalloc template concept ConceptPAL_reserve_aligned = requires(std::size_t sz) { - { PAL::template reserve_aligned(sz) } noexcept -> ConceptSame; - { PAL::template reserve_aligned(sz) } noexcept - -> ConceptSame; + { + PAL::template reserve_aligned(sz) + } + noexcept->ConceptSame; + { + PAL::template reserve_aligned(sz) + } + noexcept->ConceptSame; }; /** @@ -88,14 +110,23 @@ namespace snmalloc template concept ConceptPAL_mem_low_notify = requires(PalNotificationObject* pno) { - { PAL::expensive_low_memory_check() } -> ConceptSame; - { PAL::register_for_low_memory_callback(pno) } -> ConceptSame; + { + PAL::expensive_low_memory_check() + } + ->ConceptSame; + { + PAL::register_for_low_memory_callback(pno) + } + ->ConceptSame; }; template concept ConceptPAL_get_entropy64 = requires() { - { PAL::get_entropy64() } -> ConceptSame; + { + PAL::get_entropy64() + } + ->ConceptSame; }; /** @@ -105,21 +136,17 @@ namespace snmalloc * are, naturally, not bound by the corresponding concept. */ template - concept ConceptPAL = - ConceptPAL_static_features && - ConceptPAL_static_sizes && - ConceptPAL_error && - ConceptPAL_memops && + concept ConceptPAL = ConceptPAL_static_features&& + ConceptPAL_static_sizes&& ConceptPAL_error&& + ConceptPAL_memops && (!pal_supports || - ConceptPAL_get_entropy64) && - (!pal_supports || - ConceptPAL_mem_low_notify) && - (pal_supports || - ( - (!pal_supports || - ConceptPAL_reserve_aligned) && - ConceptPAL_reserve) - ); + ConceptPAL_get_entropy64< + PAL>)&&(!pal_supports || + ConceptPAL_mem_low_notify< + PAL>)&&(pal_supports || + ((!pal_supports || + ConceptPAL_reserve_aligned< + PAL>)&&ConceptPAL_reserve)); } // namespace snmalloc #endif diff --git a/src/pal/pal_consts.h b/src/snmalloc/pal/pal_consts.h similarity index 98% rename from src/pal/pal_consts.h rename to src/snmalloc/pal/pal_consts.h index 31df5470b..8de4cb09c 100644 --- a/src/pal/pal_consts.h +++ b/src/snmalloc/pal/pal_consts.h @@ -1,6 +1,6 @@ #pragma once -#include "../ds/defines.h" +#include "../ds_core/ds_core.h" #include #include diff --git a/src/pal/pal_dragonfly.h b/src/snmalloc/pal/pal_dragonfly.h similarity index 100% rename from src/pal/pal_dragonfly.h rename to src/snmalloc/pal/pal_dragonfly.h diff --git a/src/pal/pal_ds.h b/src/snmalloc/pal/pal_ds.h similarity index 98% rename from src/pal/pal_ds.h rename to src/snmalloc/pal/pal_ds.h index d6197d1e4..3da37cf46 100644 --- a/src/pal/pal_ds.h +++ b/src/snmalloc/pal/pal_ds.h @@ -1,7 +1,6 @@ #pragma once -#include "../ds/defines.h" -#include "../ds/helpers.h" +#include "../ds_core/ds_core.h" #include #include diff --git a/src/pal/pal_freebsd.h b/src/snmalloc/pal/pal_freebsd.h similarity index 100% rename from src/pal/pal_freebsd.h rename to src/snmalloc/pal/pal_freebsd.h diff --git a/src/pal/pal_freebsd_kernel.h b/src/snmalloc/pal/pal_freebsd_kernel.h similarity index 98% rename from src/pal/pal_freebsd_kernel.h rename to src/snmalloc/pal/pal_freebsd_kernel.h index 2e25d915b..11b951ce5 100644 --- a/src/pal/pal_freebsd_kernel.h +++ b/src/snmalloc/pal/pal_freebsd_kernel.h @@ -1,6 +1,6 @@ #pragma once -#include "../ds/bits.h" +#include "../ds_core/ds_core.h" #if defined(FreeBSD_KERNEL) extern "C" diff --git a/src/pal/pal_haiku.h b/src/snmalloc/pal/pal_haiku.h similarity index 100% rename from src/pal/pal_haiku.h rename to src/snmalloc/pal/pal_haiku.h diff --git a/src/pal/pal_linux.h b/src/snmalloc/pal/pal_linux.h similarity index 99% rename from src/pal/pal_linux.h rename to src/snmalloc/pal/pal_linux.h index aa28deda2..66240fab2 100644 --- a/src/pal/pal_linux.h +++ b/src/snmalloc/pal/pal_linux.h @@ -1,7 +1,7 @@ #pragma once #if defined(__linux__) -# include "../ds/bits.h" +# include "../ds_core/ds_core.h" # include "pal_posix.h" # include diff --git a/src/pal/pal_netbsd.h b/src/snmalloc/pal/pal_netbsd.h similarity index 100% rename from src/pal/pal_netbsd.h rename to src/snmalloc/pal/pal_netbsd.h diff --git a/src/pal/pal_noalloc.h b/src/snmalloc/pal/pal_noalloc.h similarity index 100% rename from src/pal/pal_noalloc.h rename to src/snmalloc/pal/pal_noalloc.h diff --git a/src/pal/pal_open_enclave.h b/src/snmalloc/pal/pal_open_enclave.h similarity index 100% rename from src/pal/pal_open_enclave.h rename to src/snmalloc/pal/pal_open_enclave.h diff --git a/src/pal/pal_openbsd.h b/src/snmalloc/pal/pal_openbsd.h similarity index 100% rename from src/pal/pal_openbsd.h rename to src/snmalloc/pal/pal_openbsd.h diff --git a/src/pal/pal_plain.h b/src/snmalloc/pal/pal_plain.h similarity index 95% rename from src/pal/pal_plain.h rename to src/snmalloc/pal/pal_plain.h index 27230f2b7..005ef98fa 100644 --- a/src/pal/pal_plain.h +++ b/src/snmalloc/pal/pal_plain.h @@ -1,6 +1,6 @@ #pragma once -#include "../ds/bits.h" +#include "../ds_core/ds_core.h" #include "pal_timer_default.h" namespace snmalloc diff --git a/src/pal/pal_posix.h b/src/snmalloc/pal/pal_posix.h similarity index 99% rename from src/pal/pal_posix.h rename to src/snmalloc/pal/pal_posix.h index add7712ef..f71755d20 100644 --- a/src/pal/pal_posix.h +++ b/src/snmalloc/pal/pal_posix.h @@ -1,9 +1,6 @@ #pragma once -#ifdef SNMALLOC_TRACING -# include -#endif -#include "../ds/address.h" +#include "../aal/aal.h" #include "pal_timer_default.h" #if defined(SNMALLOC_BACKTRACE_HEADER) # include SNMALLOC_BACKTRACE_HEADER diff --git a/src/pal/pal_solaris.h b/src/snmalloc/pal/pal_solaris.h similarity index 100% rename from src/pal/pal_solaris.h rename to src/snmalloc/pal/pal_solaris.h diff --git a/src/pal/pal_timer_default.h b/src/snmalloc/pal/pal_timer_default.h similarity index 100% rename from src/pal/pal_timer_default.h rename to src/snmalloc/pal/pal_timer_default.h diff --git a/src/pal/pal_windows.h b/src/snmalloc/pal/pal_windows.h similarity index 99% rename from src/pal/pal_windows.h rename to src/snmalloc/pal/pal_windows.h index 6d64a25c8..2d6f82283 100644 --- a/src/pal/pal_windows.h +++ b/src/snmalloc/pal/pal_windows.h @@ -1,7 +1,6 @@ #pragma once -#include "../ds/address.h" -#include "../ds/bits.h" +#include "../aal/aal.h" #include "pal_timer_default.h" #ifdef _WIN32 diff --git a/src/snmalloc/snmalloc.h b/src/snmalloc/snmalloc.h new file mode 100644 index 000000000..47bd6e78a --- /dev/null +++ b/src/snmalloc/snmalloc.h @@ -0,0 +1,10 @@ +#pragma once + +// Core implementation of snmalloc independent of the configuration mode +#include "snmalloc_core.h" + +// If the user has defined SNMALLOC_PROVIDE_OWN_CONFIG, this include does +// nothing. Otherwise, it provide a default configuration of snmalloc::Alloc. +#include "backend/globalconfig.h" +// User facing API surface, needs to know what `Alloc` is. +#include "snmalloc_front.h" diff --git a/src/snmalloc/snmalloc_core.h b/src/snmalloc/snmalloc_core.h new file mode 100644 index 000000000..633b7abea --- /dev/null +++ b/src/snmalloc/snmalloc_core.h @@ -0,0 +1,3 @@ +#pragma once + +#include "backend_helpers/backend_helpers.h" diff --git a/src/snmalloc/snmalloc_front.h b/src/snmalloc/snmalloc_front.h new file mode 100644 index 000000000..4c5aa60ca --- /dev/null +++ b/src/snmalloc/snmalloc_front.h @@ -0,0 +1 @@ +#include "global/global.h" diff --git a/src/snmalloc_core.h b/src/snmalloc_core.h deleted file mode 100644 index 5cd224f4d..000000000 --- a/src/snmalloc_core.h +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once - -#include "backend/commitrange.h" -#include "backend/commonconfig.h" -#include "backend/empty_range.h" -#include "backend/globalrange.h" -#include "backend/largebuddyrange.h" -#include "backend/pagemap.h" -#include "backend/pagemapregisterrange.h" -#include "backend/palrange.h" -#include "backend/range_helpers.h" -#include "backend/smallbuddyrange.h" -#include "backend/statsrange.h" -#include "backend/subrange.h" -#include "mem/globalalloc.h" diff --git a/src/snmalloc_front.h b/src/snmalloc_front.h deleted file mode 100644 index 00167ddb2..000000000 --- a/src/snmalloc_front.h +++ /dev/null @@ -1,2 +0,0 @@ -#include "mem/scopedalloc.h" -#include "mem/threadalloc.h" \ No newline at end of file diff --git a/src/test/func/bits/bits.cc b/src/test/func/bits/bits.cc index 874c5fbd6..0046516a3 100644 --- a/src/test/func/bits/bits.cc +++ b/src/test/func/bits/bits.cc @@ -3,7 +3,7 @@ */ #include -#include +#include #include void test_ctz() @@ -41,4 +41,4 @@ int main(int argc, char** argv) test_clz(); test_ctz(); -} \ No newline at end of file +} diff --git a/src/test/func/domestication/domestication.cc b/src/test/func/domestication/domestication.cc index 64c74dd14..26c033af0 100644 --- a/src/test/func/domestication/domestication.cc +++ b/src/test/func/domestication/domestication.cc @@ -10,8 +10,8 @@ int main() // # define SNMALLOC_TRACING -# include -# include +# include +# include // Specify type of allocator # define SNMALLOC_PROVIDE_OWN_CONFIG @@ -98,7 +98,7 @@ namespace snmalloc } # define SNMALLOC_NAME_MANGLE(a) test_##a -# include "../../../override/malloc.cc" +# include int main() { diff --git a/src/test/func/external_pagemap/external_pagemap.cc b/src/test/func/external_pagemap/external_pagemap.cc index 3d139a4b3..5e79c6a1a 100644 --- a/src/test/func/external_pagemap/external_pagemap.cc +++ b/src/test/func/external_pagemap/external_pagemap.cc @@ -9,7 +9,7 @@ int main() } #else # define SNMALLOC_EXPOSE_PAGEMAP 1 -# include +# include using ExternalChunkmap = ExternalGlobalPagemapTemplate; diff --git a/src/test/func/first_operation/first_operation.cc b/src/test/func/first_operation/first_operation.cc index aa22635b8..629027fc9 100644 --- a/src/test/func/first_operation/first_operation.cc +++ b/src/test/func/first_operation/first_operation.cc @@ -8,7 +8,7 @@ #include "test/setup.h" #include -#include +#include #include void alloc1(size_t size) diff --git a/src/test/func/fixed_region/fixed_region.cc b/src/test/func/fixed_region/fixed_region.cc index 527377221..0a9962731 100644 --- a/src/test/func/fixed_region/fixed_region.cc +++ b/src/test/func/fixed_region/fixed_region.cc @@ -1,8 +1,8 @@ -#include "backend/fixedglobalconfig.h" #include "test/setup.h" #include -#include +#include +#include #ifdef assert # undef assert diff --git a/src/test/func/jemalloc/jemalloc.cc b/src/test/func/jemalloc/jemalloc.cc index 1d0271e29..5baddd76e 100644 --- a/src/test/func/jemalloc/jemalloc.cc +++ b/src/test/func/jemalloc/jemalloc.cc @@ -9,8 +9,8 @@ #define SNMALLOC_BOOTSTRAP_ALLOCATOR #define SNMALLOC_JEMALLOC3_EXPERIMENTAL #define SNMALLOC_JEMALLOC_NONSTANDARD -#include "../../../override/jemalloc_compat.cc" -#include "../../../override/malloc.cc" +#include +#include #if __has_include() # include diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index 4d7ae282f..04b883973 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -6,7 +6,7 @@ #undef SNMALLOC_NO_REALLOCARRAY #undef SNMALLOC_NO_REALLOCARR #define SNMALLOC_BOOTSTRAP_ALLOCATOR -#include "../../../override/malloc.cc" +#include using namespace snmalloc; diff --git a/src/test/func/memcpy/func-memcpy.cc b/src/test/func/memcpy/func-memcpy.cc index ad8ea17d3..ff1856fac 100644 --- a/src/test/func/memcpy/func-memcpy.cc +++ b/src/test/func/memcpy/func-memcpy.cc @@ -19,14 +19,13 @@ int main() # endif # define SNMALLOC_FAIL_FAST false # define SNMALLOC_STATIC_LIBRARY_PREFIX my_ -# include "ds/defines.h" # ifndef SNMALLOC_PASS_THROUGH -# include "override/malloc.cc" +# include "snmalloc/override/malloc.cc" # else # define my_malloc(x) malloc(x) # define my_free(x) free(x) # endif -# include "override/memcpy.cc" +# include "snmalloc/override/memcpy.cc" # include "test/helpers.h" # include diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index 8db19e307..b010f2a8b 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include diff --git a/src/test/func/memory_usage/memory_usage.cc b/src/test/func/memory_usage/memory_usage.cc index 98e35484a..6d7dc40f6 100644 --- a/src/test/func/memory_usage/memory_usage.cc +++ b/src/test/func/memory_usage/memory_usage.cc @@ -7,8 +7,8 @@ #include #define SNMALLOC_NAME_MANGLE(a) our_##a -#include "../../../override/malloc-extensions.cc" -#include "../../../override/malloc.cc" +#include "../../../snmalloc/override/malloc-extensions.cc" +#include "../../../snmalloc/override/malloc.cc" using namespace snmalloc; diff --git a/src/test/func/pagemap/pagemap.cc b/src/test/func/pagemap/pagemap.cc index 74f14c45f..1f00ff8af 100644 --- a/src/test/func/pagemap/pagemap.cc +++ b/src/test/func/pagemap/pagemap.cc @@ -6,10 +6,8 @@ * but no examples were using multiple levels of pagemap. */ -#include -#include #include -#include +#include #include using namespace snmalloc; diff --git a/src/test/func/pool/pool.cc b/src/test/func/pool/pool.cc index 7d63e9e2a..215c39355 100644 --- a/src/test/func/pool/pool.cc +++ b/src/test/func/pool/pool.cc @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/test/func/redblack/redblack.cc b/src/test/func/redblack/redblack.cc index 208acea28..f13c72ebb 100644 --- a/src/test/func/redblack/redblack.cc +++ b/src/test/func/redblack/redblack.cc @@ -8,10 +8,11 @@ #include #include -#define SNMALLOC_TRACING +#ifndef SNMALLOC_TRACING +# define SNMALLOC_TRACING +#endif // Redblack tree needs some libraries with trace enabled. -#include "ds/redblacktree.h" -#include "snmalloc.h" +#include "snmalloc/snmalloc.h" struct NodeRef { diff --git a/src/test/func/release-rounding/rounding.cc b/src/test/func/release-rounding/rounding.cc index 7b1a2d765..f9541331c 100644 --- a/src/test/func/release-rounding/rounding.cc +++ b/src/test/func/release-rounding/rounding.cc @@ -1,5 +1,5 @@ #include -#include +#include #include using namespace snmalloc; diff --git a/src/test/func/sizeclass/sizeclass.cc b/src/test/func/sizeclass/sizeclass.cc index 5f3c4aa86..d42794e44 100644 --- a/src/test/func/sizeclass/sizeclass.cc +++ b/src/test/func/sizeclass/sizeclass.cc @@ -1,5 +1,5 @@ #include -#include +#include #include NOINLINE @@ -118,4 +118,4 @@ int main(int, char**) abort(); test_align_size(); -} \ No newline at end of file +} diff --git a/src/test/func/statistics/stats.cc b/src/test/func/statistics/stats.cc index 156612c27..d83dd3305 100644 --- a/src/test/func/statistics/stats.cc +++ b/src/test/func/statistics/stats.cc @@ -1,4 +1,4 @@ -#include +#include int main() { diff --git a/src/test/func/teardown/teardown.cc b/src/test/func/teardown/teardown.cc index b3d26457a..f68ed4d03 100644 --- a/src/test/func/teardown/teardown.cc +++ b/src/test/func/teardown/teardown.cc @@ -8,7 +8,7 @@ #include "test/setup.h" #include -#include +#include #include void trigger_teardown() diff --git a/src/test/func/thread_alloc_external/thread_alloc_external.cc b/src/test/func/thread_alloc_external/thread_alloc_external.cc index bc0952e2e..b8b1b2313 100644 --- a/src/test/func/thread_alloc_external/thread_alloc_external.cc +++ b/src/test/func/thread_alloc_external/thread_alloc_external.cc @@ -2,13 +2,13 @@ # undef SNMALLOC_USE_PTHREAD_DESTRUCTORS #endif -#include +#include #include // Specify using own #define SNMALLOC_EXTERNAL_THREAD_ALLOC -#include "backend/globalconfig.h" +#include namespace snmalloc { @@ -32,7 +32,7 @@ class ThreadAllocExternal } }; -#include +#include void allocator_thread_init(void) { diff --git a/src/test/func/two_alloc_types/alloc1.cc b/src/test/func/two_alloc_types/alloc1.cc index 8a21c8004..8bfe41345 100644 --- a/src/test/func/two_alloc_types/alloc1.cc +++ b/src/test/func/two_alloc_types/alloc1.cc @@ -1,10 +1,12 @@ -#define SNMALLOC_TRACING +#ifndef SNMALLOC_TRACING +# define SNMALLOC_TRACING +#endif // Redefine the namespace, so we can have two versions. #define snmalloc snmalloc_enclave -#include -#include +#include +#include // Specify type of allocator #define SNMALLOC_PROVIDE_OWN_CONFIG @@ -15,7 +17,7 @@ namespace snmalloc } #define SNMALLOC_NAME_MANGLE(a) enclave_##a -#include "../../../override/malloc.cc" +#include extern "C" void oe_allocator_init(void* base, void* end) { diff --git a/src/test/func/two_alloc_types/alloc2.cc b/src/test/func/two_alloc_types/alloc2.cc index 9bdfbf8d0..f25b5dcf0 100644 --- a/src/test/func/two_alloc_types/alloc2.cc +++ b/src/test/func/two_alloc_types/alloc2.cc @@ -1,6 +1,8 @@ -#define SNMALLOC_TRACING +#ifndef SNMALLOC_TRACING +# define SNMALLOC_TRACING +#endif #define SNMALLOC_NAME_MANGLE(a) host_##a // Redefine the namespace, so we can have two versions. #define snmalloc snmalloc_host -#include "../../../override/malloc.cc" +#include diff --git a/src/test/func/two_alloc_types/main.cc b/src/test/func/two_alloc_types/main.cc index 3cea54cb3..b7f6ded9e 100644 --- a/src/test/func/two_alloc_types/main.cc +++ b/src/test/func/two_alloc_types/main.cc @@ -1,6 +1,5 @@ -#include "../../../snmalloc.h" - #include +#include #include #include #include diff --git a/src/test/perf/contention/contention.cc b/src/test/perf/contention/contention.cc index 74f9cd200..6f6bd3959 100644 --- a/src/test/perf/contention/contention.cc +++ b/src/test/perf/contention/contention.cc @@ -5,7 +5,7 @@ #include #include -#include +#include #include #include diff --git a/src/test/perf/external_pointer/externalpointer.cc b/src/test/perf/external_pointer/externalpointer.cc index b589124cb..b2509a65e 100644 --- a/src/test/perf/external_pointer/externalpointer.cc +++ b/src/test/perf/external_pointer/externalpointer.cc @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/test/perf/low_memory/low-memory.cc b/src/test/perf/low_memory/low-memory.cc index aa024d451..fa2997fdf 100644 --- a/src/test/perf/low_memory/low-memory.cc +++ b/src/test/perf/low_memory/low-memory.cc @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include @@ -164,4 +164,4 @@ int main(int argc, char** argv) // std::cout << "Release test only." << std::endl; // #endif return 0; -} \ No newline at end of file +} diff --git a/src/test/perf/memcpy/memcpy.cc b/src/test/perf/memcpy/memcpy.cc index 62f5da35f..7642d60cc 100644 --- a/src/test/perf/memcpy/memcpy.cc +++ b/src/test/perf/memcpy/memcpy.cc @@ -1,4 +1,4 @@ -#include "mem/memcpy.h" +#include "snmalloc/global/memcpy.h" #include #include diff --git a/src/test/perf/singlethread/singlethread.cc b/src/test/perf/singlethread/singlethread.cc index cd15a12b8..29595230d 100644 --- a/src/test/perf/singlethread/singlethread.cc +++ b/src/test/perf/singlethread/singlethread.cc @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/test/setup.h b/src/test/setup.h index 2b440a623..642720ddf 100644 --- a/src/test/setup.h +++ b/src/test/setup.h @@ -1,10 +1,9 @@ #if defined(SNMALLOC_CI_BUILD) -# include +# include # if defined(WIN32) -# include # include -# include # include +# include # include // Has to come after the PAL. # include From d4226a1ea2d536e0ad3b6dfeb6a69ea85e1800ae Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Mon, 11 Apr 2022 13:28:03 +0100 Subject: [PATCH 245/302] Remove indirection of state in ranges. (#505) This doesn't give any extra flexibility: the range itself can be either a stateless class, a class with no per-instance state that stores all of static fields, or a class with stateful instances. It did add a requirement that every range implementation added an indirection layer. --- src/snmalloc/backend/backend.h | 36 +++++++++---------- src/snmalloc/backend_helpers/commitrange.h | 19 ++-------- src/snmalloc/backend_helpers/empty_range.h | 12 ------- src/snmalloc/backend_helpers/globalrange.h | 21 +++-------- .../backend_helpers/largebuddyrange.h | 27 ++++---------- .../backend_helpers/pagemapregisterrange.h | 17 ++------- src/snmalloc/backend_helpers/palrange.h | 14 -------- .../backend_helpers/smallbuddyrange.h | 23 +++--------- src/snmalloc/backend_helpers/statsrange.h | 21 +++-------- src/snmalloc/backend_helpers/subrange.h | 17 ++------- 10 files changed, 45 insertions(+), 162 deletions(-) diff --git a/src/snmalloc/backend/backend.h b/src/snmalloc/backend/backend.h index e791fe264..5bf16a6b9 100644 --- a/src/snmalloc/backend/backend.h +++ b/src/snmalloc/backend/backend.h @@ -202,17 +202,17 @@ namespace snmalloc struct LocalState { - typename ObjectRange::State object_range; + ObjectRange object_range; #ifdef SNMALLOC_META_PROTECTED - typename MetaRange::State meta_range; + MetaRange meta_range; - typename MetaRange::State& get_meta_range() + MetaRange& get_meta_range() { return meta_range; } #else - typename ObjectRange::State& get_meta_range() + ObjectRange& get_meta_range() { return object_range; } @@ -243,8 +243,8 @@ namespace snmalloc capptr::Chunk(heap_base), heap_length, [&](capptr::Chunk p, size_t sz, bool) { - typename GlobalR::State g; - g->dealloc_range(p, sz); + GlobalR g; + g.dealloc_range(p, sz); }); } @@ -266,15 +266,15 @@ namespace snmalloc capptr::Chunk p; if (local_state != nullptr) { - p = local_state->get_meta_range()->alloc_range_with_leftover(size); + p = local_state->get_meta_range().alloc_range_with_leftover(size); } else { static_assert( GlobalMetaRange::ConcurrencySafe, "Global meta data range needs to be concurrency safe."); - typename GlobalMetaRange::State global_state; - p = global_state->alloc_range(bits::next_pow2(size)); + GlobalMetaRange global_state; + p = global_state.alloc_range(bits::next_pow2(size)); } if (p == nullptr) @@ -299,7 +299,7 @@ namespace snmalloc SNMALLOC_ASSERT(size >= MIN_CHUNK_SIZE); auto meta_cap = - local_state.get_meta_range()->alloc_range(sizeof(SlabMetadata)); + local_state.get_meta_range().alloc_range(sizeof(SlabMetadata)); auto meta = meta_cap.template as_reinterpret().unsafe_ptr(); @@ -309,14 +309,14 @@ namespace snmalloc return {nullptr, nullptr}; } - auto p = local_state.object_range->alloc_range(size); + auto p = local_state.object_range.alloc_range(size); #ifdef SNMALLOC_TRACING message<1024>("Alloc chunk: {} ({})", p.unsafe_ptr(), size); #endif if (p == nullptr) { - local_state.get_meta_range()->dealloc_range( + local_state.get_meta_range().dealloc_range( meta_cap, sizeof(SlabMetadata)); errno = ENOMEM; #ifdef SNMALLOC_TRACING @@ -356,26 +356,26 @@ namespace snmalloc Pagemap::get_metaentry(address_cast(alloc)).get_slab_metadata()); Pagemap::set_metaentry(address_cast(alloc), size, t); - local_state.get_meta_range()->dealloc_range( + local_state.get_meta_range().dealloc_range( capptr::Chunk(&slab_metadata), sizeof(SlabMetadata)); // On non-CHERI platforms, we don't need to re-derive to get a pointer to // the chunk. On CHERI platforms this will need to be stored in the // SlabMetadata or similar. capptr::Chunk chunk{alloc.unsafe_ptr()}; - local_state.object_range->dealloc_range(chunk, size); + local_state.object_range.dealloc_range(chunk, size); } static size_t get_current_usage() { - typename StatsR::State stats_state; - return stats_state->get_current_usage(); + StatsR stats_state; + return stats_state.get_current_usage(); } static size_t get_peak_usage() { - typename StatsR::State stats_state; - return stats_state->get_peak_usage(); + StatsR stats_state; + return stats_state.get_peak_usage(); } }; } // namespace snmalloc diff --git a/src/snmalloc/backend_helpers/commitrange.h b/src/snmalloc/backend_helpers/commitrange.h index c9cdb39b1..d2e6d54ff 100644 --- a/src/snmalloc/backend_helpers/commitrange.h +++ b/src/snmalloc/backend_helpers/commitrange.h @@ -7,22 +7,9 @@ namespace snmalloc template class CommitRange { - typename ParentRange::State parent{}; + ParentRange parent{}; public: - class State - { - CommitRange commit_range{}; - - public: - constexpr State() = default; - - CommitRange* operator->() - { - return &commit_range; - } - }; - static constexpr bool Aligned = ParentRange::Aligned; static constexpr bool ConcurrencySafe = ParentRange::ConcurrencySafe; @@ -31,7 +18,7 @@ namespace snmalloc capptr::Chunk alloc_range(size_t size) { - auto range = parent->alloc_range(size); + auto range = parent.alloc_range(size); if (range != nullptr) PAL::template notify_using(range.unsafe_ptr(), size); return range; @@ -40,7 +27,7 @@ namespace snmalloc void dealloc_range(capptr::Chunk base, size_t size) { PAL::notify_not_using(base.unsafe_ptr(), size); - parent->dealloc_range(base, size); + parent.dealloc_range(base, size); } }; } // namespace snmalloc diff --git a/src/snmalloc/backend_helpers/empty_range.h b/src/snmalloc/backend_helpers/empty_range.h index b5102a1e5..6507a01e3 100644 --- a/src/snmalloc/backend_helpers/empty_range.h +++ b/src/snmalloc/backend_helpers/empty_range.h @@ -6,18 +6,6 @@ namespace snmalloc class EmptyRange { public: - class State - { - public: - EmptyRange* operator->() - { - static EmptyRange range{}; - return ⦥ - } - - constexpr State() = default; - }; - static constexpr bool Aligned = true; static constexpr bool ConcurrencySafe = true; diff --git a/src/snmalloc/backend_helpers/globalrange.h b/src/snmalloc/backend_helpers/globalrange.h index 016701eba..b21d4d08b 100644 --- a/src/snmalloc/backend_helpers/globalrange.h +++ b/src/snmalloc/backend_helpers/globalrange.h @@ -11,28 +11,15 @@ namespace snmalloc template class GlobalRange { - typename ParentRange::State parent{}; + SNMALLOC_REQUIRE_CONSTINIT static inline ParentRange parent{}; /** * This is infrequently used code, a spin lock simplifies the code * considerably, and should never be on the fast path. */ - FlagWord spin_lock{}; + SNMALLOC_REQUIRE_CONSTINIT static inline FlagWord spin_lock{}; public: - class State - { - SNMALLOC_REQUIRE_CONSTINIT static inline GlobalRange global_range{}; - - public: - constexpr GlobalRange* operator->() - { - return &global_range; - } - - constexpr State() = default; - }; - static constexpr bool Aligned = ParentRange::Aligned; static constexpr bool ConcurrencySafe = true; @@ -42,13 +29,13 @@ namespace snmalloc capptr::Chunk alloc_range(size_t size) { FlagLock lock(spin_lock); - return parent->alloc_range(size); + return parent.alloc_range(size); } void dealloc_range(capptr::Chunk base, size_t size) { FlagLock lock(spin_lock); - parent->dealloc_range(base, size); + parent.dealloc_range(base, size); } }; } // namespace snmalloc diff --git a/src/snmalloc/backend_helpers/largebuddyrange.h b/src/snmalloc/backend_helpers/largebuddyrange.h index e3b5eccca..3248df139 100644 --- a/src/snmalloc/backend_helpers/largebuddyrange.h +++ b/src/snmalloc/backend_helpers/largebuddyrange.h @@ -173,7 +173,7 @@ namespace snmalloc bool Consolidate = true> class LargeBuddyRange { - typename ParentRange::State parent{}; + ParentRange parent{}; static constexpr size_t REFILL_SIZE = bits::one_at_bit(REFILL_SIZE_BITS); @@ -192,7 +192,7 @@ namespace snmalloc { static_assert( MAX_SIZE_BITS != (bits::BITS - 1), "Don't set SFINAE parameter"); - parent->dealloc_range(base, size); + parent.dealloc_range(base, size); } void dealloc_overflow(capptr::Chunk overflow) @@ -201,7 +201,7 @@ namespace snmalloc { if (overflow != nullptr) { - parent->dealloc_range(overflow, bits::one_at_bit(MAX_SIZE_BITS)); + parent.dealloc_range(overflow, bits::one_at_bit(MAX_SIZE_BITS)); } } else @@ -250,10 +250,10 @@ namespace snmalloc // TODO have to add consolidation blocker for these cases. if (size >= REFILL_SIZE) { - return parent->alloc_range(size); + return parent.alloc_range(size); } - auto refill_range = parent->alloc_range(REFILL_SIZE); + auto refill_range = parent.alloc_range(REFILL_SIZE); if (refill_range != nullptr) add_range(pointer_offset(refill_range, size), REFILL_SIZE - size); return refill_range; @@ -270,7 +270,7 @@ namespace snmalloc auto refill_size = bits::max(needed_size, REFILL_SIZE); while (needed_size <= refill_size) { - auto refill = parent->alloc_range(refill_size); + auto refill = parent.alloc_range(refill_size); if (refill != nullptr) { @@ -292,19 +292,6 @@ namespace snmalloc } public: - class State - { - LargeBuddyRange buddy_range; - - public: - LargeBuddyRange* operator->() - { - return &buddy_range; - } - - constexpr State() = default; - }; - static constexpr bool Aligned = true; static constexpr bool ConcurrencySafe = false; @@ -319,7 +306,7 @@ namespace snmalloc if (size >= (bits::one_at_bit(MAX_SIZE_BITS) - 1)) { if (ParentRange::Aligned) - return parent->alloc_range(size); + return parent.alloc_range(size); return nullptr; } diff --git a/src/snmalloc/backend_helpers/pagemapregisterrange.h b/src/snmalloc/backend_helpers/pagemapregisterrange.h index 059cc157f..17ba6e192 100644 --- a/src/snmalloc/backend_helpers/pagemapregisterrange.h +++ b/src/snmalloc/backend_helpers/pagemapregisterrange.h @@ -9,22 +9,9 @@ namespace snmalloc typename ParentRange> class PagemapRegisterRange { - typename ParentRange::State state{}; + ParentRange state{}; public: - class State - { - PagemapRegisterRange range; - - public: - PagemapRegisterRange* operator->() - { - return ⦥ - } - - constexpr State() = default; - }; - constexpr PagemapRegisterRange() = default; static constexpr bool Aligned = ParentRange::Aligned; @@ -33,7 +20,7 @@ namespace snmalloc capptr::Chunk alloc_range(size_t size) { - auto base = state->alloc_range(size); + auto base = state.alloc_range(size); if (base != nullptr) Pagemap::register_range(address_cast(base), size); diff --git a/src/snmalloc/backend_helpers/palrange.h b/src/snmalloc/backend_helpers/palrange.h index f4473254b..0962e00bf 100644 --- a/src/snmalloc/backend_helpers/palrange.h +++ b/src/snmalloc/backend_helpers/palrange.h @@ -7,20 +7,6 @@ namespace snmalloc class PalRange { public: - class State - { - public: - PalRange* operator->() - { - // There is no state required for the PalRange - // using a global just to satisfy the typing. - static PalRange range{}; - return ⦥ - } - - constexpr State() = default; - }; - static constexpr bool Aligned = pal_supports; // Note we have always assumed the Pals to provide a concurrency safe diff --git a/src/snmalloc/backend_helpers/smallbuddyrange.h b/src/snmalloc/backend_helpers/smallbuddyrange.h index b3ccc4544..b212323b5 100644 --- a/src/snmalloc/backend_helpers/smallbuddyrange.h +++ b/src/snmalloc/backend_helpers/smallbuddyrange.h @@ -145,7 +145,7 @@ namespace snmalloc template class SmallBuddyRange { - typename ParentRange::State parent{}; + ParentRange parent{}; static constexpr size_t MIN_BITS = bits::next_pow2_bits_const(sizeof(FreeChunk)); @@ -164,13 +164,13 @@ namespace snmalloc buddy_small.add_block(base.as_reinterpret(), align) .template as_reinterpret(); if (overflow != nullptr) - parent->dealloc_range(overflow, bits::one_at_bit(MIN_CHUNK_BITS)); + parent.dealloc_range(overflow, bits::one_at_bit(MIN_CHUNK_BITS)); }); } capptr::Chunk refill(size_t size) { - auto refill = parent->alloc_range(MIN_CHUNK_SIZE); + auto refill = parent.alloc_range(MIN_CHUNK_SIZE); if (refill != nullptr) add_range(pointer_offset(refill, size), MIN_CHUNK_SIZE - size); @@ -179,19 +179,6 @@ namespace snmalloc } public: - class State - { - SmallBuddyRange buddy_range; - - public: - SmallBuddyRange* operator->() - { - return &buddy_range; - } - - constexpr State() = default; - }; - static constexpr bool Aligned = true; static_assert(ParentRange::Aligned, "ParentRange must be aligned"); @@ -203,7 +190,7 @@ namespace snmalloc { if (size >= MIN_CHUNK_SIZE) { - return parent->alloc_range(size); + return parent.alloc_range(size); } auto result = buddy_small.remove_block(size); @@ -238,7 +225,7 @@ namespace snmalloc { if (size >= MIN_CHUNK_SIZE) { - parent->dealloc_range(base, size); + parent.dealloc_range(base, size); return; } diff --git a/src/snmalloc/backend_helpers/statsrange.h b/src/snmalloc/backend_helpers/statsrange.h index 069398375..da874b237 100644 --- a/src/snmalloc/backend_helpers/statsrange.h +++ b/src/snmalloc/backend_helpers/statsrange.h @@ -10,25 +10,12 @@ namespace snmalloc template class StatsRange { - typename ParentRange::State parent{}; + ParentRange parent{}; static inline std::atomic current_usage{}; static inline std::atomic peak_usage{}; public: - class State - { - StatsRange stats_range{}; - - public: - constexpr StatsRange* operator->() - { - return &stats_range; - } - - constexpr State() = default; - }; - static constexpr bool Aligned = ParentRange::Aligned; static constexpr bool ConcurrencySafe = ParentRange::ConcurrencySafe; @@ -37,7 +24,7 @@ namespace snmalloc capptr::Chunk alloc_range(size_t size) { - auto result = parent->alloc_range(size); + auto result = parent.alloc_range(size); if (result != nullptr) { auto prev = current_usage.fetch_add(size); @@ -54,7 +41,7 @@ namespace snmalloc void dealloc_range(capptr::Chunk base, size_t size) { current_usage -= size; - parent->dealloc_range(base, size); + parent.dealloc_range(base, size); } size_t get_current_usage() @@ -67,4 +54,4 @@ namespace snmalloc return peak_usage.load(); } }; -} // namespace snmalloc \ No newline at end of file +} // namespace snmalloc diff --git a/src/snmalloc/backend_helpers/subrange.h b/src/snmalloc/backend_helpers/subrange.h index 44c1db9d5..3c6617d7c 100644 --- a/src/snmalloc/backend_helpers/subrange.h +++ b/src/snmalloc/backend_helpers/subrange.h @@ -11,22 +11,9 @@ namespace snmalloc template class SubRange { - typename ParentRange::State parent{}; + ParentRange parent{}; public: - class State - { - SubRange sub_range{}; - - public: - constexpr State() = default; - - SubRange* operator->() - { - return &sub_range; - } - }; - constexpr SubRange() = default; static constexpr bool Aligned = ParentRange::Aligned; @@ -38,7 +25,7 @@ namespace snmalloc SNMALLOC_ASSERT(bits::is_pow2(sub_size)); auto full_size = sub_size << RATIO_BITS; - auto overblock = parent->alloc_range(full_size); + auto overblock = parent.alloc_range(full_size); if (overblock == nullptr) return nullptr; From f0361f8c0153a11fb91e86ede3b0f76d13e65f4f Mon Sep 17 00:00:00 2001 From: Robert Norton Date: Mon, 11 Apr 2022 13:29:06 +0100 Subject: [PATCH 246/302] Pagemap move (#504) * Move PageMap interface into pagemap.h and rename to BasicPagemap. Refactoring suggested by David. This allows custom backends to reuse or extend the BasicPagemap. It has template parameters for the PAL, concrete page map and page map entry types as well as the Backend (so that it can be friends). BackendAllocator provides an exmple page map entry type. --- src/snmalloc/backend/backend.h | 153 ++++++++----------------- src/snmalloc/backend_helpers/pagemap.h | 101 ++++++++++++++++ 2 files changed, 146 insertions(+), 108 deletions(-) diff --git a/src/snmalloc/backend/backend.h b/src/snmalloc/backend/backend.h index 5bf16a6b9..cbff62b04 100644 --- a/src/snmalloc/backend/backend.h +++ b/src/snmalloc/backend/backend.h @@ -24,135 +24,72 @@ namespace snmalloc class BackendAllocator : public CommonConfig { public: + class PageMapEntry; using Pal = PAL; - using SlabMetadata = FrontendSlabMetadata; - class Pagemap - { - friend class BackendAllocator; + private: + using ConcretePagemap = + FlatPagemap; - public: + public: + /** + * Example of type stored in the pagemap. + * The following class could be replaced by: + * + * ``` + * using PageMapEntry = FrontendMetaEntry; + * ``` + * + * The full form here provides an example of how to extend the pagemap + * entries. It also guarantees that the front end never directly + * constructs meta entries, it only ever reads them or modifies them in + * place. + */ + class PageMapEntry : public FrontendMetaEntry + { /** - * Export the type stored in the pagemap. - * The following class could be replaced by: - * - * ``` - * using Entry = FrontendMetaEntry; - * ``` - * - * The full form here provides an example of how to extend the pagemap - * entries. It also guarantees that the front end never directly - * constructs meta entries, it only ever reads them or modifies them in - * place. + * The private initialising constructor is usable only by this back end. */ - class Entry : public FrontendMetaEntry - { - /** - * The private initialising constructor is usable only by this back end. - */ - friend class BackendAllocator; - - /** - * The private default constructor is usable only by the pagemap. - */ - friend class FlatPagemap; - - /** - * The only constructor that creates newly initialised meta entries. - * This is callable only by the back end. The front end may copy, - * query, and update these entries, but it may not create them - * directly. This contract allows the back end to store any arbitrary - * metadata in meta entries when they are first constructed. - */ - SNMALLOC_FAST_PATH - Entry(SlabMetadata* meta, uintptr_t ras) - : FrontendMetaEntry(meta, ras) - {} - - /** - * Default constructor. This must be callable from the pagemap. - */ - SNMALLOC_FAST_PATH Entry() = default; - - /** - * Copy assignment is used only by the pagemap. - */ - Entry& operator=(const Entry& other) - { - FrontendMetaEntry::operator=(other); - return *this; - } - }; - - private: - SNMALLOC_REQUIRE_CONSTINIT - static inline FlatPagemap - concretePagemap; + friend class BackendAllocator; /** - * Set the metadata associated with a chunk. + * The private default constructor is usable only by the pagemap. */ - SNMALLOC_FAST_PATH - static void set_metaentry(address_t p, size_t size, const Entry& t) - { - for (address_t a = p; a < p + size; a += MIN_CHUNK_SIZE) - { - concretePagemap.set(a, t); - } - } + friend ConcretePagemap; - public: /** - * Get the metadata associated with a chunk. - * - * Set template parameter to true if it not an error - * to access a location that is not backed by a chunk. + * The only constructor that creates newly initialised meta entries. + * This is callable only by the back end. The front end may copy, + * query, and update these entries, but it may not create them + * directly. This contract allows the back end to store any arbitrary + * metadata in meta entries when they are first constructed. */ - template - SNMALLOC_FAST_PATH static const auto& get_metaentry(address_t p) - { - return concretePagemap.template get(p); - } + SNMALLOC_FAST_PATH + PageMapEntry(SlabMetadata* meta, uintptr_t ras) + : FrontendMetaEntry(meta, ras) + {} /** - * Get the metadata associated with a chunk. - * - * Set template parameter to true if it not an error - * to access a location that is not backed by a chunk. + * Copy assignment is used only by the pagemap. */ - template - SNMALLOC_FAST_PATH static auto& get_metaentry_mut(address_t p) + PageMapEntry& operator=(const PageMapEntry& other) { - return concretePagemap.template get_mut(p); - } - - static void register_range(address_t p, size_t sz) - { - concretePagemap.register_range(p, sz); + FrontendMetaEntry::operator=(other); + return *this; } /** - * Return the bounds of the memory this back-end manages as a pair of - * addresses (start then end). This is available iff this is a - * fixed-range Backend. + * Default constructor. This must be callable from the pagemap. */ - template - static SNMALLOC_FAST_PATH - std::enable_if_t> - get_bounds() - { - static_assert( - fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); - - return concretePagemap.get_bounds(); - } - - static bool is_initialised() - { - return concretePagemap.is_initialised(); - } + SNMALLOC_FAST_PATH PageMapEntry() = default; }; + using Pagemap = BasicPagemap< + BackendAllocator, + PAL, + ConcretePagemap, + PageMapEntry, + fixed_range>; #if defined(_WIN32) || defined(__CHERI_PURE_CAPABILITY__) static constexpr bool CONSOLIDATE_PAL_ALLOCS = false; diff --git a/src/snmalloc/backend_helpers/pagemap.h b/src/snmalloc/backend_helpers/pagemap.h index ec7bbdfdb..15ff51cfc 100644 --- a/src/snmalloc/backend_helpers/pagemap.h +++ b/src/snmalloc/backend_helpers/pagemap.h @@ -325,4 +325,105 @@ namespace snmalloc body[p >> SHIFT] = t; } }; + + /** + * This is a generic implementation of the backend's interface to the page + * map. It takes a concrete page map implementation (probably FlatPageMap + * above) and entry type. It is friends with the backend passed in as a + * template parameter so that the backend can initialise the concrete page map + * and use set_metaentry which no one else should use. + */ + template< + typename Backend, + typename PAL, + typename ConcreteMap, + typename PageMapEntry, + bool fixed_range> + class BasicPagemap + { + public: + /** + * Export the type stored in the pagemap. + */ + using Entry = PageMapEntry; + + private: + friend Backend; + + /** + * Instance of the concrete pagemap, accessible to the backend so that + * it can call the init method whose type dependent on fixed_range. + */ + SNMALLOC_REQUIRE_CONSTINIT + static inline ConcreteMap concretePagemap; + + /** + * Set the metadata associated with a chunk. + */ + SNMALLOC_FAST_PATH + static void set_metaentry(address_t p, size_t size, const Entry& t) + { + for (address_t a = p; a < p + size; a += MIN_CHUNK_SIZE) + { + concretePagemap.set(a, t); + } + } + + public: + /** + * Get the metadata associated with a chunk. + * + * Set template parameter to true if it not an error + * to access a location that is not backed by a chunk. + */ + template + SNMALLOC_FAST_PATH static const auto& get_metaentry(address_t p) + { + return concretePagemap.template get(p); + } + + /** + * Get the metadata associated with a chunk. + * + * Set template parameter to true if it not an error + * to access a location that is not backed by a chunk. + */ + template + SNMALLOC_FAST_PATH static auto& get_metaentry_mut(address_t p) + { + return concretePagemap.template get_mut(p); + } + + /** + * Register a range in the pagemap as in-use, requiring it to allow writing + * to the underlying memory. + */ + static void register_range(address_t p, size_t sz) + { + concretePagemap.register_range(p, sz); + } + + /** + * Return the bounds of the memory this back-end manages as a pair of + * addresses (start then end). This is available iff this is a + * fixed-range Backend. + */ + template + static SNMALLOC_FAST_PATH + std::enable_if_t> + get_bounds() + { + static_assert(fixed_range_ == fixed_range, "Don't set SFINAE parameter!"); + + return concretePagemap.get_bounds(); + } + + /** + * Return whether the pagemap is initialised, ready for access. + */ + static bool is_initialised() + { + return concretePagemap.is_initialised(); + } + }; } // namespace snmalloc From 43f5f33913d03de4d9602b7e07b0b0666f564e98 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 11 Apr 2022 11:38:43 +0100 Subject: [PATCH 247/302] Fix missing consolidation marker During creation of large allocation the code was not setting the consolidation bit. This meant that Windows would crash for certain patterns for large allocations. --- src/snmalloc/backend/backend.h | 14 +++++------ .../backend_helpers/largebuddyrange.h | 23 ++----------------- .../backend_helpers/pagemapregisterrange.h | 10 +++++++- src/snmalloc/mem/metadata.h | 3 ++- 4 files changed, 19 insertions(+), 31 deletions(-) diff --git a/src/snmalloc/backend/backend.h b/src/snmalloc/backend/backend.h index cbff62b04..d9a2d2340 100644 --- a/src/snmalloc/backend/backend.h +++ b/src/snmalloc/backend/backend.h @@ -108,15 +108,13 @@ namespace snmalloc #else // Set up source of memory using P = PalRange; - using Base = std:: - conditional_t>; + using Base = std::conditional_t< + fixed_range, + EmptyRange, + PagemapRegisterRange>; // Global range of memory - using StatsR = StatsRange>; + using StatsR = + StatsRange>; using GlobalR = GlobalRange; # ifdef SNMALLOC_META_PROTECTED diff --git a/src/snmalloc/backend_helpers/largebuddyrange.h b/src/snmalloc/backend_helpers/largebuddyrange.h index 3248df139..6a7bd2444 100644 --- a/src/snmalloc/backend_helpers/largebuddyrange.h +++ b/src/snmalloc/backend_helpers/largebuddyrange.h @@ -169,8 +169,7 @@ namespace snmalloc typename ParentRange, size_t REFILL_SIZE_BITS, size_t MAX_SIZE_BITS, - SNMALLOC_CONCEPT(ConceptBuddyRangeMeta) Pagemap, - bool Consolidate = true> + SNMALLOC_CONCEPT(ConceptBuddyRangeMeta) Pagemap> class LargeBuddyRange { ParentRange parent{}; @@ -218,24 +217,7 @@ namespace snmalloc void add_range(capptr::Chunk base, size_t length) { range_to_pow_2_blocks( - base, - length, - [this](capptr::Chunk base, size_t align, bool first) { - if constexpr (!Consolidate) - { - // Tag first entry so we don't consolidate it. - if (first) - { - auto& entry = Pagemap::get_metaentry_mut(address_cast(base)); - entry.claim_for_backend(); - entry.set_boundary(); - } - } - else - { - UNUSED(first); - } - + base, length, [this](capptr::Chunk base, size_t align, bool) { auto overflow = capptr::Chunk(reinterpret_cast( buddy_large.add_block(base.unsafe_uintptr(), align))); @@ -247,7 +229,6 @@ namespace snmalloc { if (ParentRange::Aligned) { - // TODO have to add consolidation blocker for these cases. if (size >= REFILL_SIZE) { return parent.alloc_range(size); diff --git a/src/snmalloc/backend_helpers/pagemapregisterrange.h b/src/snmalloc/backend_helpers/pagemapregisterrange.h index 17ba6e192..de60dd8da 100644 --- a/src/snmalloc/backend_helpers/pagemapregisterrange.h +++ b/src/snmalloc/backend_helpers/pagemapregisterrange.h @@ -6,7 +6,8 @@ namespace snmalloc { template< SNMALLOC_CONCEPT(ConceptBackendMetaRange) Pagemap, - typename ParentRange> + typename ParentRange, + bool CanConsolidate = true> class PagemapRegisterRange { ParentRange state{}; @@ -25,6 +26,13 @@ namespace snmalloc if (base != nullptr) Pagemap::register_range(address_cast(base), size); + if (!CanConsolidate) + { + // Mark start of allocation in pagemap. + auto& entry = Pagemap::get_metaentry_mut(address_cast(base)); + entry.set_boundary(); + } + return base; } }; diff --git a/src/snmalloc/mem/metadata.h b/src/snmalloc/mem/metadata.h index ac56f0ab4..ca90ec1af 100644 --- a/src/snmalloc/mem/metadata.h +++ b/src/snmalloc/mem/metadata.h @@ -129,7 +129,8 @@ namespace snmalloc */ [[nodiscard]] bool is_unowned() const { - return (meta == 0) && (remote_and_sizeclass == 0); + return ((meta == 0) || (meta == META_BOUNDARY_BIT)) && + (remote_and_sizeclass == 0); } /** From 943bae1b34736b584a0a86b3db134c8421adfbf5 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 11 Apr 2022 11:39:29 +0100 Subject: [PATCH 248/302] Minimal example of #506 --- src/test/func/memory/memory.cc | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index b010f2a8b..57cafc235 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -4,6 +4,7 @@ #include #include #include +#include #if ((defined(__linux__) && !defined(__ANDROID__)) || defined(__sun)) && \ !defined(SNMALLOC_QEMU_WORKAROUND) /* @@ -492,6 +493,28 @@ void test_remaining_bytes() } } +void test_consolidaton_bug() +{ + /** + * Check for consolidation of various sizes, but allocating and deallocating, + * then requesting larger sizes. See issue #506 + */ + auto& alloc = ThreadAlloc::get(); + + for (size_t i = 0; i < 27; i++) + { + std::vector allocs; + for (size_t j = 0; j < 4; j++) + { + allocs.push_back(alloc.alloc(bits::one_at_bit(i))); + } + for (auto a : allocs) + { + alloc.dealloc(a); + } + } +} + int main(int argc, char** argv) { setup(); @@ -531,5 +554,6 @@ int main(int argc, char** argv) test_alloc_16M(); test_calloc_16M(); #endif + test_consolidaton_bug(); return 0; } From 848a7b1499e357d1eab784082f3a201b5ff64dd8 Mon Sep 17 00:00:00 2001 From: David Chisnall Date: Thu, 14 Apr 2022 09:29:27 +0100 Subject: [PATCH 249/302] Check things in release builds with check-client. Clang 15 doesn't build the release builds with CHECK_CLIENT enabled because they are using `SNMALLOC_ASSERT` and so the values that we're collecting to check are never actually checked. This is probably a bug - if we're turning on the checks, I imagine it's because we want them. --- src/snmalloc/mem/corealloc.h | 4 ++-- src/snmalloc/mem/freelist.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/snmalloc/mem/corealloc.h b/src/snmalloc/mem/corealloc.h index 4baab6f50..8aae67d45 100644 --- a/src/snmalloc/mem/corealloc.h +++ b/src/snmalloc/mem/corealloc.h @@ -315,7 +315,7 @@ namespace snmalloc count++; } // Check the list contains all the elements - SNMALLOC_ASSERT( + SNMALLOC_CHECK( (count + more) == snmalloc::sizeclass_to_slab_object_count(sizeclass)); if (more > 0) @@ -330,7 +330,7 @@ namespace snmalloc count++; } } - SNMALLOC_ASSERT( + SNMALLOC_CHECK( count == snmalloc::sizeclass_to_slab_object_count(sizeclass)); #endif // TODO: This is a capability amplification as we are saying we diff --git a/src/snmalloc/mem/freelist.h b/src/snmalloc/mem/freelist.h index c310153a5..3c70307f4 100644 --- a/src/snmalloc/mem/freelist.h +++ b/src/snmalloc/mem/freelist.h @@ -730,7 +730,7 @@ namespace snmalloc { if (&head[i] == end[i]) { - SNMALLOC_ASSERT(length[i] == 0); + SNMALLOC_CHECK(length[i] == 0); continue; } @@ -747,7 +747,7 @@ namespace snmalloc prev = signed_prev(address_cast(curr), address_cast(next), key); curr = next; } - SNMALLOC_ASSERT(count == length[i]); + SNMALLOC_CHECK(count == length[i]); } #else UNUSED(key); From bf54eeb7bed7c0b9716c02efb5209e95b675b4fc Mon Sep 17 00:00:00 2001 From: David Carlier Date: Sun, 3 Apr 2022 07:54:57 +0100 Subject: [PATCH 250/302] New option to name reserved pages. --- CMakeLists.txt | 3 +++ src/snmalloc/pal/pal_linux.h | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index db1e5c545..b9b07b4e9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,6 +24,7 @@ cmake_dependent_option(SNMALLOC_STATIC_LIBRARY "Build static libraries" ON "NOT cmake_dependent_option(SNMALLOC_MEMCPY_BOUNDS "Perform bounds checks on memcpy with heap objects" OFF "NOT SNMALLOC_HEADER_ONLY_LIBRARY" OFF) cmake_dependent_option(SNMALLOC_CHECK_LOADS "Perform bounds checks on the source argument to memcpy with heap objects" OFF "SNMALLOC_MEMCPY_BOUNDS" OFF) cmake_dependent_option(SNMALLOC_OPTIMISE_FOR_CURRENT_MACHINE "Compile for current machine architecture" Off "NOT SNMALLOC_HEADER_ONLY_LIBRARY" OFF) +cmake_dependent_option(SNMALLOC_PAGEID "Set an id to memory regions" OFF "NOT SNMALLOC_PAGEID" OFF) if (NOT SNMALLOC_HEADER_ONLY_LIBRARY) # Pick a sensible default for the thread cleanup mechanism if (${CMAKE_SYSTEM_NAME} STREQUAL FreeBSD) @@ -323,6 +324,8 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) target_compile_definitions(${name} PRIVATE SNMALLOC_CHECK_LOADS=$,true,false>) + target_compile_definitions(${name} PRIVATE + SNMALLOC_PAGEID=$,true,false>) install(TARGETS ${name} EXPORT snmallocConfig) diff --git a/src/snmalloc/pal/pal_linux.h b/src/snmalloc/pal/pal_linux.h index 66240fab2..3a06bf8e8 100644 --- a/src/snmalloc/pal/pal_linux.h +++ b/src/snmalloc/pal/pal_linux.h @@ -6,6 +6,7 @@ # include # include +# include # include extern "C" int puts(const char* str); @@ -38,7 +39,32 @@ namespace snmalloc { void* p = PALPOSIX::reserve(size); if (p) + { madvise(p, size, MADV_DONTDUMP); +# ifdef SNMALLOC_PAGEID +# ifndef PR_SET_VMA +# define PR_SET_VMA 0x53564d41 +# define PR_SET_VMA_ANON_NAME 0 +# endif + + /** + * + * If the kernel is set with CONFIG_ANON_VMA_NAME + * the reserved pages would appear as follow + * + * 7fa5f0ceb000-7fa5f0e00000 rw-p 00000000 00:00 0 [anon:snmalloc] + * 7fa5f0e00000-7fa5f1800000 rw-p 00000000 00:00 0 [anon:snmalloc] + * + */ + + prctl( + PR_SET_VMA, + PR_SET_VMA_ANON_NAME, + (unsigned long)p, + size, + (unsigned long)"snmalloc"); +# endif + } return p; } From f277cf2f0094f58e2b2de2f56bc7735f0b4c20f3 Mon Sep 17 00:00:00 2001 From: Robert Norton Date: Wed, 27 Apr 2022 11:56:16 +0100 Subject: [PATCH 251/302] Refactor capptr_domesticate SFINAE to make more statically safe. This refactoring was provided by David. Previously if a backend provided a capptr_domesticate function with the wrong type it would be silently ignored. This change requires backends to explicitly opt in to domestication via a new Backend::Option and ensures the compiler will loudly complain if there is a mismatch. --- src/snmalloc/backend/fixedglobalconfig.h | 21 ++++++- src/snmalloc/backend_helpers/commonconfig.h | 7 +++ src/snmalloc/mem/backend_wrappers.h | 62 ++++++++++++++------ src/test/func/domestication/domestication.cc | 1 + 4 files changed, 72 insertions(+), 19 deletions(-) diff --git a/src/snmalloc/backend/fixedglobalconfig.h b/src/snmalloc/backend/fixedglobalconfig.h index f0e8a570a..41b502688 100644 --- a/src/snmalloc/backend/fixedglobalconfig.h +++ b/src/snmalloc/backend/fixedglobalconfig.h @@ -24,7 +24,26 @@ namespace snmalloc return alloc_pool; } - static constexpr Flags Options{}; + /* + * The obvious + * `static constexpr Flags Options{.HasDomesticate = true};` fails on + * Ubuntu 18.04 with an error "sorry, unimplemented: non-trivial + * designated initializers not supported". + * The following was copied from domestication.cc test with the following + * comment: + * C++, even as late as C++20, has some really quite strict limitations on + * designated initializers. However, as of C++17, we can have constexpr + * lambdas and so can use more of the power of the statement fragment of + * C++, and not just its initializer fragment, to initialize a non-prefix + * subset of the flags (in any order, at that). + */ + static constexpr Flags Options = []() constexpr + { + Flags opts = {}; + opts.HasDomesticate = true; + return opts; + } + (); // This needs to be a forward reference as the // thread local state will need to know about this. diff --git a/src/snmalloc/backend_helpers/commonconfig.h b/src/snmalloc/backend_helpers/commonconfig.h index c8e55e1a0..aca6103f7 100644 --- a/src/snmalloc/backend_helpers/commonconfig.h +++ b/src/snmalloc/backend_helpers/commonconfig.h @@ -86,6 +86,13 @@ namespace snmalloc * the queue nodes themselves (which are always considered Wild)? */ bool QueueHeadsAreTame = true; + + /** + * Does the backend provide a capptr_domesticate function to sanity check + * pointers? If so it will be called when untrusted pointers are consumed + * (on dealloc and in freelists) otherwise a no-op version is provided. + */ + bool HasDomesticate = false; }; /** diff --git a/src/snmalloc/mem/backend_wrappers.h b/src/snmalloc/mem/backend_wrappers.h index 36657dfe5..a16b7e8e3 100644 --- a/src/snmalloc/mem/backend_wrappers.h +++ b/src/snmalloc/mem/backend_wrappers.h @@ -36,41 +36,46 @@ namespace snmalloc namespace detail { /** - * SFINAE helper, calls capptr_domesticate in the backend if it exists. + * SFINAE helper to detect the presence of capptr_domesticate function in + * backend. Returns true if there is a function with correct name and type. */ template< SNMALLOC_CONCEPT(ConceptBackendDomestication) Backend, typename T, SNMALLOC_CONCEPT(capptr::ConceptBound) B> - SNMALLOC_FAST_PATH_INLINE auto - capptr_domesticate(typename Backend::LocalState* ls, CapPtr p, int) - -> decltype(Backend::capptr_domesticate(ls, p)) + constexpr SNMALLOC_FAST_PATH_INLINE auto has_domesticate(int) + -> std::enable_if_t< + std::is_same_v< + decltype(Backend::capptr_domesticate( + std::declval(), + std::declval>())), + CapPtr< + T, + typename B::template with_wildness< + capptr::dimension::Wildness::Tame>>>, + bool> { - return Backend::capptr_domesticate(ls, p); + return true; } /** - * SFINAE helper. If the back end does not provide special handling for - * domestication then assume all wild pointers can be domesticated. + * SFINAE helper to detect the presence of capptr_domesticate function in + * backend. Returns false in case where above template does not match. */ template< SNMALLOC_CONCEPT(ConceptBackendGlobals) Backend, typename T, SNMALLOC_CONCEPT(capptr::ConceptBound) B> - SNMALLOC_FAST_PATH_INLINE auto - capptr_domesticate(typename Backend::LocalState*, CapPtr p, long) + constexpr SNMALLOC_FAST_PATH_INLINE bool has_domesticate(long) { - return CapPtr< - T, - typename B::template with_wildness>( - p.unsafe_ptr()); + return false; } } // namespace detail /** - * Wrapper that calls `Backend::capptr_domesticate` if and only if it is - * implemented. If it is not implemented then this assumes that any wild - * pointer can be domesticated. + * Wrapper that calls `Backend::capptr_domesticate` if and only if + * Backend::Options.HasDomesticate is true. If it is not implemented then + * this assumes that any wild pointer can be domesticated. */ template< SNMALLOC_CONCEPT(ConceptBackendGlobals) Backend, @@ -79,7 +84,28 @@ namespace snmalloc SNMALLOC_FAST_PATH_INLINE auto capptr_domesticate(typename Backend::LocalState* ls, CapPtr p) { - return detail::capptr_domesticate(ls, p, 0); - } + static_assert( + !detail::has_domesticate(0) || + (detail::has_domesticate(0) && + Backend::Options.HasDomesticate), + "Back end provides domesticate function but opts out of using it "); + static_assert( + detail::has_domesticate(0) || + !(detail::has_domesticate(0) && + Backend::Options.HasDomesticate), + "Back end does not provide capptr_domesticate and requests its use"); + if constexpr (Backend::Options.HasDomesticate) + { + return Backend::capptr_domesticate(ls, p); + } + else + { + UNUSED(ls); + return CapPtr< + T, + typename B::template with_wildness>( + p.unsafe_ptr()); + } + } } // namespace snmalloc diff --git a/src/test/func/domestication/domestication.cc b/src/test/func/domestication/domestication.cc index 26c033af0..726a2f394 100644 --- a/src/test/func/domestication/domestication.cc +++ b/src/test/func/domestication/domestication.cc @@ -40,6 +40,7 @@ namespace snmalloc { Flags opts = {}; opts.QueueHeadsAreTame = false; + opts.HasDomesticate = true; return opts; } (); From 70eba1e70b8fb3ba63e1a1e1c81ab36e2da14fb5 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 28 Apr 2022 17:08:27 +0100 Subject: [PATCH 252/302] Refill heuristic (#511) * Adding a refilling heuristic The large buddy allocator requests memory from its parent range. The request size was a fixed large request. This was sufficiently large, so that contention was not a problem. This change makes it initially smaller, and gradually growing so that contention is still not a problem, but for small work loads it requests less memory. * Remove special case for OE as no longer required. --- src/snmalloc/backend/backend.h | 39 ++++++----- .../backend_helpers/largebuddyrange.h | 68 +++++++++++++++++-- 2 files changed, 82 insertions(+), 25 deletions(-) diff --git a/src/snmalloc/backend/backend.h b/src/snmalloc/backend/backend.h index d9a2d2340..04a0fd02f 100644 --- a/src/snmalloc/backend/backend.h +++ b/src/snmalloc/backend/backend.h @@ -97,42 +97,45 @@ namespace snmalloc static constexpr bool CONSOLIDATE_PAL_ALLOCS = true; #endif -#if defined(OPEN_ENCLAVE) - // Single global buddy allocator is used on open enclave due to - // the limited address space. - using StatsR = StatsRange>>; - using GlobalR = GlobalRange; - using ObjectRange = GlobalR; - using GlobalMetaRange = ObjectRange; -#else // Set up source of memory - using P = PalRange; + using P = PalRange; using Base = std::conditional_t< fixed_range, EmptyRange, PagemapRegisterRange>; + + static constexpr size_t MinBaseSizeBits() + { + if constexpr (pal_supports) + { + return bits::next_pow2_bits_const(PAL::minimum_alloc_size); + } + else + { + return MIN_CHUNK_BITS; + } + } + // Global range of memory - using StatsR = - StatsRange>; + using StatsR = StatsRange< + LargeBuddyRange>; using GlobalR = GlobalRange; -# ifdef SNMALLOC_META_PROTECTED +#ifdef SNMALLOC_META_PROTECTED // Source for object allocations using ObjectRange = - LargeBuddyRange, 21, 21, Pagemap>; + LargeBuddyRange, 21, 21, Pagemap>; // Set up protected range for metadata - using SubR = CommitRange, DefaultPal>; + using SubR = CommitRange, PAL>; using MetaRange = SmallBuddyRange>; using GlobalMetaRange = GlobalRange; -# else +#else // Source for object allocations and metadata // No separation between the two using ObjectRange = SmallBuddyRange< - LargeBuddyRange, 21, 21, Pagemap>>; + LargeBuddyRange, 21, 21, Pagemap>>; using GlobalMetaRange = GlobalRange; -# endif #endif struct LocalState diff --git a/src/snmalloc/backend_helpers/largebuddyrange.h b/src/snmalloc/backend_helpers/largebuddyrange.h index 6a7bd2444..b81e392ef 100644 --- a/src/snmalloc/backend_helpers/largebuddyrange.h +++ b/src/snmalloc/backend_helpers/largebuddyrange.h @@ -165,19 +165,53 @@ namespace snmalloc } }; + /** + * Used to represent a consolidating range of memory. Uses a buddy allocator + * to consolidate adjacent blocks. + * + * ParentRange - Represents the range to get memory from to fill this range. + * + * REFILL_SIZE_BITS - Maximum size of a refill, may ask for less during warm + * up phase. + * + * MAX_SIZE_BITS - Maximum size that this range will store. + * + * Pagemap - How to access the pagemap, which is used to store the red black + * tree nodes for the buddy allocators. + * + * MIN_REFILL_SIZE_BITS - The minimum size that the ParentRange can be asked + * for + */ template< typename ParentRange, size_t REFILL_SIZE_BITS, size_t MAX_SIZE_BITS, - SNMALLOC_CONCEPT(ConceptBuddyRangeMeta) Pagemap> + SNMALLOC_CONCEPT(ConceptBuddyRangeMeta) Pagemap, + size_t MIN_REFILL_SIZE_BITS = 0> class LargeBuddyRange { ParentRange parent{}; + /** + * Maximum size of a refill + */ static constexpr size_t REFILL_SIZE = bits::one_at_bit(REFILL_SIZE_BITS); /** + * Minimum size of a refill + */ + static constexpr size_t MIN_REFILL_SIZE = + bits::one_at_bit(MIN_REFILL_SIZE_BITS); + + /** + * The size of memory requested so far. * + * This is used to determine the refill size. + */ + size_t requested_total = 0; + + /** + * Buddy allocator used to represent this range of memory. */ Buddy, MIN_CHUNK_BITS, MAX_SIZE_BITS> buddy_large; @@ -229,17 +263,36 @@ namespace snmalloc { if (ParentRange::Aligned) { - if (size >= REFILL_SIZE) + // Use amount currently requested to determine refill size. + // This will gradually increase the usage of the parent range. + // So small examples can grow local caches slowly, and larger + // examples will grow them by the refill size. + // + // The heuristic is designed to allocate the following sequence for + // 16KiB requests 16KiB, 16KiB, 32Kib, 64KiB, ..., REFILL_SIZE/2, + // REFILL_SIZE, REFILL_SIZE, ... Hence if this if they are coming from a + // contiguous aligned range, then they could be consolidated. This + // depends on the ParentRange behaviour. + size_t refill_size = bits::min(REFILL_SIZE, requested_total); + refill_size = bits::max(refill_size, MIN_REFILL_SIZE); + refill_size = bits::max(refill_size, size); + refill_size = bits::next_pow2(refill_size); + + auto refill_range = parent.alloc_range(refill_size); + if (refill_range != nullptr) { - return parent.alloc_range(size); + requested_total += refill_size; + add_range(pointer_offset(refill_range, size), refill_size - size); } - - auto refill_range = parent.alloc_range(REFILL_SIZE); - if (refill_range != nullptr) - add_range(pointer_offset(refill_range, size), REFILL_SIZE - size); return refill_range; } + // Note the unaligned parent path does not use + // requested_total in the heuristic for the initial size + // this is because the request needs to introduce alignment. + // Currently the unaligned variant is not used as a local cache. + // So the gradual growing of refill_size is not needed. + // Need to overallocate to get the alignment right. bool overflow = false; size_t needed_size = bits::umul(size, 2, overflow); @@ -255,6 +308,7 @@ namespace snmalloc if (refill != nullptr) { + requested_total += refill_size; add_range(refill, refill_size); SNMALLOC_ASSERT(refill_size < bits::one_at_bit(MAX_SIZE_BITS)); From 3e08caaf327dc41928ef04440e3027e2ddfcefb5 Mon Sep 17 00:00:00 2001 From: Robert Norton Date: Fri, 29 Apr 2022 11:33:14 +0100 Subject: [PATCH 253/302] Simplify asserts introduced in #512. (#513) These are functionally equivalent but have the form A -> B (i.e. !A || B in C++). --- src/snmalloc/mem/backend_wrappers.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/snmalloc/mem/backend_wrappers.h b/src/snmalloc/mem/backend_wrappers.h index a16b7e8e3..e26cc22ae 100644 --- a/src/snmalloc/mem/backend_wrappers.h +++ b/src/snmalloc/mem/backend_wrappers.h @@ -86,14 +86,12 @@ namespace snmalloc { static_assert( !detail::has_domesticate(0) || - (detail::has_domesticate(0) && - Backend::Options.HasDomesticate), + Backend::Options.HasDomesticate, "Back end provides domesticate function but opts out of using it "); static_assert( detail::has_domesticate(0) || - !(detail::has_domesticate(0) && - Backend::Options.HasDomesticate), + !Backend::Options.HasDomesticate, "Back end does not provide capptr_domesticate and requests its use"); if constexpr (Backend::Options.HasDomesticate) { From 56ccb5c794dbc4d07755fcdb8a457ec229b75224 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 3 May 2022 14:26:25 +0100 Subject: [PATCH 254/302] Enable memcpy checks for check shim (#515) Build three levels of checking - None - Checks memcpy only - Checks (full) Currently you can build checks without enabling the memcpy protection. This PR fixes that. --- CMakeLists.txt | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b9b07b4e9..9dc5e9319 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,8 +21,7 @@ option(SNMALLOC_NO_REALLOCARR "Build without reallocarr exported" ON) # Options that apply only if we're not building the header-only library cmake_dependent_option(SNMALLOC_RUST_SUPPORT "Build static library for rust" OFF "NOT SNMALLOC_HEADER_ONLY_LIBRARY" OFF) cmake_dependent_option(SNMALLOC_STATIC_LIBRARY "Build static libraries" ON "NOT SNMALLOC_HEADER_ONLY_LIBRARY" OFF) -cmake_dependent_option(SNMALLOC_MEMCPY_BOUNDS "Perform bounds checks on memcpy with heap objects" OFF "NOT SNMALLOC_HEADER_ONLY_LIBRARY" OFF) -cmake_dependent_option(SNMALLOC_CHECK_LOADS "Perform bounds checks on the source argument to memcpy with heap objects" OFF "SNMALLOC_MEMCPY_BOUNDS" OFF) +cmake_dependent_option(SNMALLOC_CHECK_LOADS "Perform bounds checks on the source argument to memcpy with heap objects" OFF "NOT SNMALLOC_HEADER_ONLY_LIBRARY" OFF) cmake_dependent_option(SNMALLOC_OPTIMISE_FOR_CURRENT_MACHINE "Compile for current machine architecture" Off "NOT SNMALLOC_HEADER_ONLY_LIBRARY" OFF) cmake_dependent_option(SNMALLOC_PAGEID "Set an id to memory regions" OFF "NOT SNMALLOC_PAGEID" OFF) if (NOT SNMALLOC_HEADER_ONLY_LIBRARY) @@ -332,9 +331,7 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) endfunction() set(SHIM_FILES src/snmalloc/override/new.cc) - if (SNMALLOC_MEMCPY_BOUNDS) - list(APPEND SHIM_FILES src/snmalloc/override/memcpy.cc) - endif () + set(SHIM_FILES_MEMCPY src/snmalloc/override/memcpy.cc) if (SNMALLOC_STATIC_LIBRARY) add_shim(snmallocshim-static STATIC ${SHIM_FILES}) @@ -344,7 +341,8 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) if(NOT WIN32) add_shim(snmallocshim SHARED ${SHIM_FILES}) - add_shim(snmallocshim-checks SHARED ${SHIM_FILES}) + add_shim(snmallocshim-checks-memcpy-only SHARED ${SHIM_FILES} ${SHIM_FILES_MEMCPY}) + add_shim(snmallocshim-checks SHARED ${SHIM_FILES} ${SHIM_FILES_MEMCPY}) target_compile_definitions(snmallocshim-checks PRIVATE SNMALLOC_CHECK_CLIENT) endif() From d927a9a179f315c79f2f57e5f2b9307c95730186 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 29 Apr 2022 16:04:31 +0100 Subject: [PATCH 255/302] Modified Metadata range to have separate pool The Metadata range should not be shared with the object range. This change ensures that their are separate requests to the Pal for meta-data and object data ranges. The requests are never combined, and thus memory cannot flow from being used in malloc to later be used in meta- data. --- src/snmalloc/backend/backend.h | 49 +++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/src/snmalloc/backend/backend.h b/src/snmalloc/backend/backend.h index 04a0fd02f..be498f456 100644 --- a/src/snmalloc/backend/backend.h +++ b/src/snmalloc/backend/backend.h @@ -117,24 +117,39 @@ namespace snmalloc } // Global range of memory - using StatsR = StatsRange< + using GlobalR = GlobalRange< LargeBuddyRange>; - using GlobalR = GlobalRange; #ifdef SNMALLOC_META_PROTECTED + // Introduce two global ranges, so we don't mix Object and Meta + using CentralObectRange = GlobalRange< + LargeBuddyRange>; + using CentralMetaRange = GlobalRange, // Use SubRange to introduce guard pages. + 24, + bits::BITS - 1, + Pagemap, + MinBaseSizeBits()>>; + // Source for object allocations - using ObjectRange = - LargeBuddyRange, 21, 21, Pagemap>; - // Set up protected range for metadata - using SubR = CommitRange, PAL>; - using MetaRange = - SmallBuddyRange>; - using GlobalMetaRange = GlobalRange; + using StatsObject = StatsRange>; + using ObjectRange = LargeBuddyRange; + using StatsR = StatsObject; + + using StatsRMeta = StatsRange>; + + using MetaRange = SmallBuddyRange< + LargeBuddyRange>; + // Create global range that can service small meta-data requests. + // Don't want to add this to the CentralMetaRange to move Commit outside the + // lock on the common case. + using GlobalMetaRange = GlobalRange>; #else // Source for object allocations and metadata // No separation between the two + using StatsR = StatsRange; using ObjectRange = SmallBuddyRange< - LargeBuddyRange, 21, 21, Pagemap>>; + LargeBuddyRange, 21, 21, Pagemap>>; using GlobalMetaRange = GlobalRange; #endif @@ -307,13 +322,23 @@ namespace snmalloc static size_t get_current_usage() { StatsR stats_state; - return stats_state.get_current_usage(); + auto result = stats_state.get_current_usage(); +#ifdef SNMALLOC_PROTECT_METADATA + StatsRMeta stats_state_meta; + result += stats_state_meta.get_current_usage(); +#endif + return result; } static size_t get_peak_usage() { StatsR stats_state; - return stats_state.get_peak_usage(); + auto result = stats_state.get_peak_usage(); +#ifdef SNMALLOC_PROTECT_METADATA + StatsRMeta stats_state_meta; + result += stats_state_meta.get_peak_usage(); +#endif + return result; } }; } // namespace snmalloc From 563d5a5ceec3b21c55803971a4c55781301cd7fc Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 3 May 2022 13:49:05 +0100 Subject: [PATCH 256/302] Make clang on Windows use /Debug --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9dc5e9319..9a5bb261e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -244,7 +244,7 @@ function(add_warning_flags name) $<$:-Wsign-conversion -Wconversion>) target_link_options(${name} PRIVATE $<$:-Wl,--no-undefined> - $<$:$<${ci_or_debug}:/DEBUG>>) + $<$:$<${ci_or_debug}:/DEBUG>>) endfunction() From d47c44783da551f020b0b6db3a46c670aa3be81f Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 5 May 2022 11:09:14 +0100 Subject: [PATCH 257/302] Remove redundant params. --- src/snmalloc/ds/allocconfig.h | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/src/snmalloc/ds/allocconfig.h b/src/snmalloc/ds/allocconfig.h index 2186b78f9..b3e789e58 100644 --- a/src/snmalloc/ds/allocconfig.h +++ b/src/snmalloc/ds/allocconfig.h @@ -13,33 +13,6 @@ namespace snmalloc #endif ; - enum DecommitStrategy - { - /** - * Never decommit memory. - */ - DecommitNone, - /** - * Decommit superslabs when they are entirely empty. - */ - DecommitSuper, - /** - * Decommit superslabs only when we are informed of memory pressure by the - * OS, do not decommit anything in normal operation. - */ - DecommitSuperLazy - }; - - static constexpr DecommitStrategy decommit_strategy = -#ifdef USE_DECOMMIT_STRATEGY - USE_DECOMMIT_STRATEGY -#elif defined(_WIN32) && !defined(OPEN_ENCLAVE) - DecommitSuperLazy -#else - DecommitSuper -#endif - ; - // The remaining values are derived, not configurable. static constexpr size_t POINTER_BITS = bits::next_pow2_bits_const(sizeof(uintptr_t)); From 14b7b40a82003689e2ac197995b8731584ac8168 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 5 May 2022 11:09:37 +0100 Subject: [PATCH 258/302] Add a Stats combiner to make code cleaner. --- src/snmalloc/backend/backend.h | 30 ++++++++--------------- src/snmalloc/backend_helpers/statsrange.h | 18 ++++++++++++++ 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/src/snmalloc/backend/backend.h b/src/snmalloc/backend/backend.h index be498f456..db380fc5a 100644 --- a/src/snmalloc/backend/backend.h +++ b/src/snmalloc/backend/backend.h @@ -134,22 +134,22 @@ namespace snmalloc // Source for object allocations using StatsObject = StatsRange>; using ObjectRange = LargeBuddyRange; - using StatsR = StatsObject; - using StatsRMeta = StatsRange>; + using StatsMeta = StatsRange>; using MetaRange = SmallBuddyRange< - LargeBuddyRange>; + LargeBuddyRange>; // Create global range that can service small meta-data requests. // Don't want to add this to the CentralMetaRange to move Commit outside the // lock on the common case. - using GlobalMetaRange = GlobalRange>; + using GlobalMetaRange = GlobalRange>; + using Stats = StatsCombiner; #else // Source for object allocations and metadata // No separation between the two - using StatsR = StatsRange; + using Stats = StatsRange; using ObjectRange = SmallBuddyRange< - LargeBuddyRange, 21, 21, Pagemap>>; + LargeBuddyRange, 21, 21, Pagemap>>; using GlobalMetaRange = GlobalRange; #endif @@ -321,24 +321,14 @@ namespace snmalloc static size_t get_current_usage() { - StatsR stats_state; - auto result = stats_state.get_current_usage(); -#ifdef SNMALLOC_PROTECT_METADATA - StatsRMeta stats_state_meta; - result += stats_state_meta.get_current_usage(); -#endif - return result; + Stats stats_state; + return stats_state.get_current_usage(); } static size_t get_peak_usage() { - StatsR stats_state; - auto result = stats_state.get_peak_usage(); -#ifdef SNMALLOC_PROTECT_METADATA - StatsRMeta stats_state_meta; - result += stats_state_meta.get_peak_usage(); -#endif - return result; + Stats stats_state; + return stats_state.get_peak_usage(); } }; } // namespace snmalloc diff --git a/src/snmalloc/backend_helpers/statsrange.h b/src/snmalloc/backend_helpers/statsrange.h index da874b237..98a06aec3 100644 --- a/src/snmalloc/backend_helpers/statsrange.h +++ b/src/snmalloc/backend_helpers/statsrange.h @@ -54,4 +54,22 @@ namespace snmalloc return peak_usage.load(); } }; + + template + class StatsCombiner + { + StatsR1 r1{}; + StatsR2 r2{}; + + public: + size_t get_current_usage() + { + return r1.get_current_usage() + r2.get_current_usage(); + } + + size_t get_peak_usage() + { + return r1.get_peak_usage() + r2.get_peak_usage(); + } + }; } // namespace snmalloc From 9f9964239e21d2e70df5ba1bf75fbdcc3c507cf0 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Sun, 8 May 2022 20:32:52 +0100 Subject: [PATCH 259/302] Ensure logging doesn't affect errno --- src/snmalloc/pal/pal_posix.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/snmalloc/pal/pal_posix.h b/src/snmalloc/pal/pal_posix.h index f71755d20..58b3c1f3b 100644 --- a/src/snmalloc/pal/pal_posix.h +++ b/src/snmalloc/pal/pal_posix.h @@ -163,6 +163,9 @@ namespace snmalloc */ static void message(const char* const str) noexcept { + // We don't want logging to affect the errno behaviour of the program. + auto hold = KeepErrno(); + void* nl = const_cast("\n"); struct iovec iov[] = {{const_cast(str), strlen(str)}, {nl, 1}}; UNUSED(writev(STDERR_FILENO, iov, sizeof(iov) / sizeof(struct iovec))); From 325c013e85aa0640b04fa2072993c6ba06678dff Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Sun, 8 May 2022 20:33:15 +0100 Subject: [PATCH 260/302] Add some tracing to the backend --- src/snmalloc/backend/backend.h | 34 +++++++----- .../backend_helpers/backend_helpers.h | 1 + src/snmalloc/backend_helpers/logrange.h | 52 +++++++++++++++++++ 3 files changed, 75 insertions(+), 12 deletions(-) create mode 100644 src/snmalloc/backend_helpers/logrange.h diff --git a/src/snmalloc/backend/backend.h b/src/snmalloc/backend/backend.h index db380fc5a..dda05abd8 100644 --- a/src/snmalloc/backend/backend.h +++ b/src/snmalloc/backend/backend.h @@ -117,23 +117,33 @@ namespace snmalloc } // Global range of memory - using GlobalR = GlobalRange< - LargeBuddyRange>; + using GlobalR = GlobalRange>>; #ifdef SNMALLOC_META_PROTECTED // Introduce two global ranges, so we don't mix Object and Meta - using CentralObectRange = GlobalRange< - LargeBuddyRange>; - using CentralMetaRange = GlobalRange, // Use SubRange to introduce guard pages. - 24, - bits::BITS - 1, - Pagemap, - MinBaseSizeBits()>>; + using CentralObjectRange = GlobalRange>>; + using CentralMetaRange = GlobalRange, // Use SubRange to introduce guard pages. + 24, + bits::BITS - 1, + Pagemap, + MinBaseSizeBits()>>>; // Source for object allocations - using StatsObject = StatsRange>; - using ObjectRange = LargeBuddyRange; + using StatsObject = StatsRange>; + using ObjectRange = + LogRange<5, LargeBuddyRange>; using StatsMeta = StatsRange>; diff --git a/src/snmalloc/backend_helpers/backend_helpers.h b/src/snmalloc/backend_helpers/backend_helpers.h index 06f0c9206..94dfec2f2 100644 --- a/src/snmalloc/backend_helpers/backend_helpers.h +++ b/src/snmalloc/backend_helpers/backend_helpers.h @@ -5,6 +5,7 @@ #include "empty_range.h" #include "globalrange.h" #include "largebuddyrange.h" +#include "logrange.h" #include "pagemap.h" #include "pagemapregisterrange.h" #include "palrange.h" diff --git a/src/snmalloc/backend_helpers/logrange.h b/src/snmalloc/backend_helpers/logrange.h new file mode 100644 index 000000000..432e8772c --- /dev/null +++ b/src/snmalloc/backend_helpers/logrange.h @@ -0,0 +1,52 @@ +#pragma once + +namespace snmalloc +{ + /** + * RangeName is an integer to specify which range is being logged. Strings can + * be used as template parameters. + * + * ParentRange is what the range is logging calls to. + */ + template + class LogRange + { + ParentRange parent{}; + + public: + static constexpr bool Aligned = ParentRange::Aligned; + + static constexpr bool ConcurrencySafe = ParentRange::ConcurrencySafe; + + constexpr LogRange() = default; + + capptr::Chunk alloc_range(size_t size) + { +#ifdef SNMALLOC_TRACING + message<1024>("Call alloc_range({}) on {}", size, RangeName); +#endif + auto range = parent.alloc_range(size); +#ifdef SNMALLOC_TRACING + message<1024>( + "{} = alloc_range({}) in {}", range.unsafe_ptr(), size, RangeName); +#endif + return range; + } + + void dealloc_range(capptr::Chunk base, size_t size) + { +#ifdef SNMALLOC_TRACING + message<1024>( + "dealloc_range({}, {}}) on {}", base.unsafe_ptr(), size, RangeName); +#endif + parent.dealloc_range(base, size); +#ifdef SNMALLOC_TRACING + message<1024>( + "Done dealloc_range({}, {}})! on {}", + base.unsafe_ptr(), + size, + RangeName); +#endif + } + }; +} // namespace snmalloc \ No newline at end of file From 2d44ae9db45825c5d98ec5131c7c75d4a574bd3c Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Sun, 8 May 2022 20:33:39 +0100 Subject: [PATCH 261/302] Check for allocation failure. --- src/test/perf/contention/contention.cc | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/test/perf/contention/contention.cc b/src/test/perf/contention/contention.cc index 6f6bd3959..94a45c5ad 100644 --- a/src/test/perf/contention/contention.cc +++ b/src/test/perf/contention/contention.cc @@ -83,7 +83,16 @@ void test_tasks_f(size_t id) size_t size = 16 + (r.next() % 1024); size_t* res = (size_t*)(use_malloc ? malloc(size) : a.alloc(size)); - *res = size; + if (res != nullptr) + { + *res = size; + } + else + { + std::cout << "Failed to allocate " << size << " bytes" << std::endl; + abort(); + } + size_t* out = contention[n % swapsize].exchange(res, std::memory_order_acq_rel); @@ -100,6 +109,8 @@ void test_tasks_f(size_t id) void test_tasks(size_t num_tasks, size_t count, size_t size) { + std::cout << "Sequential setup" << std::endl; + auto& a = ThreadAlloc::get(); contention = new std::atomic[size]; @@ -120,6 +131,7 @@ void test_tasks(size_t num_tasks, size_t count, size_t size) Stats s0; current_alloc_pool()->aggregate_stats(s0); #endif + std::cout << "Begin parallel test:" << std::endl; { ParallelTest test(num_tasks); From 5906b1458601129876864d08274418eb2ee62e9e Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 9 May 2022 09:46:24 +0100 Subject: [PATCH 262/302] Out-of-memory can fail silently If this test fails to allocate memory, that should not cause the test to fail. The 'abort' was added previously to confirm a infrequent failure was caused by out-of-memory causing the test to assign to nullptr. This was confirmed in a CI run, and now the test can be made to ignore allocation failure. --- src/test/perf/contention/contention.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/perf/contention/contention.cc b/src/test/perf/contention/contention.cc index 94a45c5ad..bcea629d2 100644 --- a/src/test/perf/contention/contention.cc +++ b/src/test/perf/contention/contention.cc @@ -90,7 +90,7 @@ void test_tasks_f(size_t id) else { std::cout << "Failed to allocate " << size << " bytes" << std::endl; - abort(); + // Continue as this is not an important failure. } size_t* out = From d5c732f3c14969d119178cbc8382d592800c5421 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 9 May 2022 13:38:12 +0100 Subject: [PATCH 263/302] Preparation for 0.6.0 (#517) Co-authored-by: David Chisnall Co-authored-by: Robert Norton <1412774+rmn30@users.noreply.github.com> Co-authored-by: Nathaniel Wesley Filardo Co-authored-by: Istvan Haller <31476121+ihaller@users.noreply.github.com> --- README.md | 19 +- difference.md | 42 - docs/security/FreelistProtection.md | 130 ++ docs/security/GuardedMemcpy.md | 151 +++ docs/security/README.md | 39 + docs/security/Randomisation.md | 69 + docs/security/VariableSizedChunks.md | 97 ++ docs/security/data/ChunkMap.png | Bin 0 -> 146696 bytes docs/security/data/benchres.csv | 1281 ++++++++++++++++++ docs/security/data/doublefreeprotection.gif | Bin 0 -> 1552092 bytes docs/security/data/memcpy_perf.png | Bin 0 -> 300561 bytes docs/security/data/perfgraph-memcpy-only.png | Bin 0 -> 69872 bytes docs/security/data/perfgraph.png | Bin 0 -> 151184 bytes docs/security/data/res_je.csv | 160 +++ docs/security/data/res_mi.csv | 160 +++ docs/security/data/res_scudo.csv | 160 +++ docs/security/data/res_smi.csv | 160 +++ docs/security/data/res_sn-0.5.3.csv | 160 +++ docs/security/data/res_sn-0.6.0-checks.csv | 160 +++ docs/security/data/res_sn-0.6.0-memcpy.csv | 160 +++ docs/security/data/res_sn-0.6.0.csv | 160 +++ 21 files changed, 3062 insertions(+), 46 deletions(-) delete mode 100644 difference.md create mode 100644 docs/security/FreelistProtection.md create mode 100644 docs/security/GuardedMemcpy.md create mode 100644 docs/security/README.md create mode 100644 docs/security/Randomisation.md create mode 100644 docs/security/VariableSizedChunks.md create mode 100644 docs/security/data/ChunkMap.png create mode 100644 docs/security/data/benchres.csv create mode 100644 docs/security/data/doublefreeprotection.gif create mode 100644 docs/security/data/memcpy_perf.png create mode 100644 docs/security/data/perfgraph-memcpy-only.png create mode 100644 docs/security/data/perfgraph.png create mode 100644 docs/security/data/res_je.csv create mode 100644 docs/security/data/res_mi.csv create mode 100644 docs/security/data/res_scudo.csv create mode 100644 docs/security/data/res_smi.csv create mode 100644 docs/security/data/res_sn-0.5.3.csv create mode 100644 docs/security/data/res_sn-0.6.0-checks.csv create mode 100644 docs/security/data/res_sn-0.6.0-memcpy.csv create mode 100644 docs/security/data/res_sn-0.6.0.csv diff --git a/README.md b/README.md index 96c6311d7..c57b2a9af 100644 --- a/README.md +++ b/README.md @@ -29,13 +29,24 @@ scenarios that can be problematic for other allocators: Both of these can cause massive reductions in performance of other allocators, but do not for snmalloc. -Comprehensive details about snmalloc's design can be found in the -[accompanying paper](snmalloc.pdf), and differences between the paper and the -current implementation are [described here](difference.md). -Since writing the paper, the performance of snmalloc has improved considerably. +The implementation of snmalloc has evolved significantly since the [initial paper](snmalloc.pdf). +The mechanism for returning memory to remote threads has remained, but most of the meta-data layout has changed. +We recommend you read [docs/security](./docs/security/README.md) to find out about the current design, and +if you want to dive into the code (./docs/AddressSpace.md) provides a good overview of the allocation and deallocation paths. [![snmalloc CI](https://github.com/microsoft/snmalloc/actions/workflows/main.yml/badge.svg?branch=master)](https://github.com/microsoft/snmalloc/actions/workflows/main.yml) +# Hardening + +There is a hardened version of snmalloc, it contains + +* Randomisation of the allocations' relative locations, +* Most meta-data is stored separately from allocations, and is protected with guard pages, +* All in-band meta-data is protected with a novel encoding that can detect corruption, and +* Provides a `memcpy` that automatically checks the bounds relative to the underlying malloc. + +A more comprehensive write up is in [docs/security](./docs/security/README.md). + # Further documentation - [Instructions for building snmalloc](docs/BUILDING.md) diff --git a/difference.md b/difference.md deleted file mode 100644 index 273a74ca6..000000000 --- a/difference.md +++ /dev/null @@ -1,42 +0,0 @@ -# Difference from published paper - -This document outlines the changes that have diverged from -[the published paper](snmalloc.pdf) on `snmalloc`. - - 1. Link no longer terminates the bump-free list. The paper describes a - complex invariant for how the final element of the bump-free list can - also be the link node. - - We now have a much simpler invariant. The link is either 1, signifying - the block is completely full. Or not 1, signifying it has at least one - free element at the offset contained in link, and that contains the DLL - node for this sizeclass. - - The bump-free list contains additional free elements, and the remaining - bump allocated space. - - The value 1, is never a valid bump allocation value, as we initially - allocate the first entry as the link, so we can use 1 as the no more bump - space value. - - 2. Separate Bump/Free list. We have separate bump ptr and free list. This - is required to have a "fast free list" in each allocator for each - sizeclass. We bump allocate a whole os page (4KiB) worth of allocations - in one go, so that the CPU predicts the free list path for the fast - path. - - 3. Per allocator per sizeclass fast free list. Each allocator has an array - for each small size class that contains a free list of some elements for - that sizeclass. This enables a very compressed path for the common - allocation case. - - 4. We now store a direct pointer to the next element in each slabs free list - rather than a relative offset into the slab. This enables list - calculation on the fast path. - - 5. There is a single bump-ptr per size class that is part of the - allocator structure. The per size class slab list now only contains slabs - with free list, and not if it only has a bump ptr. - -[2-4] Are changes that are directly inspired by -(mimalloc)[http://github.com/microsoft/mimalloc]. \ No newline at end of file diff --git a/docs/security/FreelistProtection.md b/docs/security/FreelistProtection.md new file mode 100644 index 000000000..9263100b3 --- /dev/null +++ b/docs/security/FreelistProtection.md @@ -0,0 +1,130 @@ +# Protecting meta-data + +Corrupting an allocator's meta-data is a common pattern for increasing the power of a use-after-free or out-of-bounds write vulnerabilities. +If you can corrupt the allocator's meta-data, then you can take a control gadget in one part of a system, and use it to affect other parts of the system. +There are various approaches to protecting allocator meta-data, the most common are: + +* make the allocator meta-data hard to find through randomisation +* use completely separate ranges of memory for meta-data and allocations +* surround meta-data with guard pages +* add some level of encryption/checksuming + +With the refactoring of the page table ([described earlier](./VariableSizedChunks.md)), we can put all the slab meta-data in completely separate regions of memory to the allocations. +We maintain this separation over time, and never allow memory that has been used for allocations to become meta-data and vice versa. +Within the meta-data regions, we add randomisation to make the data hard to find, and add large guard regions around the meta-data. +By using completely separate regions of memory for allocations and meta-data we ensure that no dangling allocation can refer to current meta-data. +This is particularly important for CHERI as it means a UAF can be used to corrupt allocator meta-data. + +But there is one super important bit that still remains: free lists. + +## What are free lists? + +Many allocators chain together unused allocations into a linked list. +This is remarkably space efficient, as it doesn't require meta-data proportional to the number of allocations on a slab. +The disused objects can be used in either a linked stack or queue. +However, the key problem is neither randomisation or guard pages can be used to protect this _in-band_ meta-data. + +In snmalloc, we have introduced a novel technique for protecting this data. + +## Protecting a free queue. + +The idea is remarkably simple: a doubly linked list is far harder to corrupt than a single linked list, because you can check its invariant: +``` + x.next.prev == x +``` +In every kind of free list in snmalloc, we encode both the forward and backward pointers in our lists. +For the forward direction, we use an [involution](https://en.wikipedia.org/wiki/Involution_(mathematics)), `f`, such as XORing a randomly choosen value: +``` + f(a) = a XOR k0 +``` +For the backward direction, we use a more complex, two-argument function +``` + g(a, b) = (a XOR k1) * (b XOR k2) +``` +where `k1` and `k2` are two randomly chosen 64 bit values. +The encoded back pointer of the node after `x` in the list is `g(x, f(x.next))`, which gives a value that is hard to forge and still encodes the back edge relationship. + +As we build the list, we add this value to the disused object, and when we consume the free list later, we check the value is correct. +Importantly, the order of construction and consumption have to be the same, which means we can only use queues, and not stacks. + +The checks give us a way to detect that the list has not been corrupted. +In particular, use-after-free or out-of-bounds writes to either the `next` or `prev` value are highly likely to be detected later. + +## Double free protection + +This encoding also provides a great double free protection. +If you free twice, it will corrupt the `prev` pointer, and thus when we come to reallocate that object later, we will detect the double free. +The following animation shows the effect of a double free: + +![Double free protection example](./data/doublefreeprotection.gif) + +This is a weak protection as it is lazy, in that only when the object is reused will snmalloc raise an error, so a `malloc` can fail due to double free, but we are only aiming to make exploits harder; this is not a bug finding tool. + + +## Where do we use this? + +Everywhere we link disused objects, so (1) per-slab free queues and (2) per-allocator message queues for returning freed allocations to other threads. +Originally, snmalloc used queues for returning memory to other threads. +We had to refactor the per slab free lists to be queues rather than stacks, but that is fairly straightforward. +The code for the free lists can be found here: + +[Code](https://github.com/microsoft/snmalloc/blob/main/src/snmalloc/mem/freelist.h) + +The idea could easily be applied to other allocators, and we're happy to discuss this. + +## Finished assembly + +So let's look at what costs we incur from this. +There are bits that are added to both creating the queues, and taking elements from the queues. +Here we show the assembly for taking from a per-slab free list, which is integrated into the fast path of allocation: +```x86asm +: + lea rax,[rdi-0x1] # Check for small size class + cmp rax,0xdfff # | zero is considered a large size + ja SLOW_SIZE # | to remove from fast path. + shr rax,0x4 # Lookup size class in table + lea rcx,[size_table] # | + movzx edx,BYTE PTR [rax+rcx*1] # | + mov rdi,rdx #+Caclulate index into free lists + shl rdi,0x4 #+| (without checks this is a shift by + # | 0x3, and can be fused into an lea) + mov r8,QWORD PTR [rip+0xab9b] # Find thread local allocator state + mov rcx,QWORD PTR fs:0x0 # | + add rcx,r8 # | + add rcx,rdi # Load head of free list for size class + mov rax,QWORD PTR fs:[r8+rdi*1] # | + test rax,rax # Check if free list is empty + je SLOW_PATH_REFILL # | + mov rsi,QWORD PTR fs:0x0 # Calculate location of free list structure + add rsi,r8 # | rsi = fs:[r8] + mov rdx,QWORD PTR fs:[r8+0x2e8] #+Load next pointer key + xor rdx,QWORD PTR [rax] # Load next pointer + prefetcht0 BYTE PTR [rdx] # Prefetch next object + mov QWORD PTR [rcx],rdx # Update head of free list + mov rcx,QWORD PTR [rax+0x8] #+Check signed_prev value is correct + cmp rcx,QWORD PTR fs:[r8+rdi*1+0x8] #+| + jne CORRUPTION_ERROR #+| + lea rcx,[rdi+rsi*1] #+Calculate signed_prev location + add rcx,0x8 #+| rcx = fs:[r8+rdi*1+0x8] + mov rsi,QWORD PTR fs:[r8+0x2d8] #+Calculate next signed_prev value + add rsi,rax #+| + add rdx,QWORD PTR fs:[r8+0x2e0] #+| + imul rdx,rsi #+| + mov QWORD PTR [rcx],rdx #+Store signed_prev for next entry. + ret +``` +The extra instructions specific to handling the checks are marked with `+`. +As you can see the fast path is about twice the length of the fast path without protection, but only adds a single branch to the fast path, one multiplication, five additional loads, and one store. +The loads only involve one additional cache line for key material. +Overall, the cost is surprisingly low. + +Note: the free list header now contains the value that `prev` should contain, which leads to slightly worse x86 codegen. +For instance the checks introduce `shl rdi,0x4`, which was previously fused with an `lea` instruction without the checks. + +## Conclusion + +This approach provides a strong defense against corruption of the free lists used in snmalloc. +This means all inline meta-data has corruption detection. +The check is remarkably simple for building double free detection, and has far lower memory overhead compared to using an allocation bitmap. + +[Next we show how to randomise the layout of memory in snmalloc, and thus make it harder to guess relative address of a pair of allocations.](./Randomisation.md) diff --git a/docs/security/GuardedMemcpy.md b/docs/security/GuardedMemcpy.md new file mode 100644 index 000000000..a871042b8 --- /dev/null +++ b/docs/security/GuardedMemcpy.md @@ -0,0 +1,151 @@ +# Providing a guarded memcpy + +Out of bounds errors are a serious problem for systems. +We did some analysis of the Microsoft Security Response Center data to look at the out-of-bounds heap corruption, and found a common culprit: `memcpy`. +Of the OOB writes that were categorised as leading to remote code execution (RCE), 1/3 of them had a block copy operation like memcpy as the initial source of corruption. +This makes any mitigation to `memcpy` extremely high-value. + +Now, if a `memcpy` crosses a boundary of a `malloc` allocation, then we have a well-defined error in the semantics of the program. +No sensible program should do this. +So let's see how we detect this with snmalloc. + + +## What is `memcpy`? + +So `memcpy(src, dst, len)` copies `len` bytes from `src` to `dst`. +For this to be valid, we can check: +``` + if (src is managed by snmalloc) + check(remaining_bytes(src) >= len) + if (dst is managed by snmalloc) + check(remaining_bytes(dst) >= len) +``` +Now, the first `if` is checking for reading beyond the end of the object, and the second is checking for writing beyond the end of the destination object. +By default, for release checks we only check the `dst` is big enough. + + +## How can we implement `remaining_bytes`? + +In the previous [page](./VariableSizedChunks.md), we discussed how we enable variable sized slabs. +Let's consider how that representation enables us to quickly find the start/end of any object. + +All slab sizes are powers of two, and a given slab's lowest address will be naturally aligned for the slab's size. +(For brevity, slabs are sometimes said to be "naturally aligned (at) powers of two".) +That is if `x` is the start of a slab of size `2^n`, then `x % (2^n) == 0`. +This means that a single mask can be used to find the offset into a slab. +As the objects are layed out continguously, we can also get the offset in the object with a modulus operations, that is, `remaining_bytes(p)` is effectively: +``` + object_size - ((p % slab_size) % object_size) +``` + +Well, as anyone will tell you, division/modulus on a fast path is a non-starter. +The first modulus is easy to deal with, we can replace `% slab_size` with a bit-wise mask. +However, as `object_size` can be non-power-of-two values, we need to work a little harder. + +## Reciprocal division to the rescue + +When you have a finite domain, you can lower divisions into a multiply and shift. +By pre-calculating `c = (((2^n) - 1)/size) + 1`, the division `x / size` can instead be computed by +``` + (x * c) >> n +``` +The choice of `n` has to be done carefully for the possible values of `x`, but with a large enough `n` we can make this work for all slab offsets and sizes. + +Now from division, we can calculate the modulus, by multiplying the result of the division +by the size, and then subtracting the result from the original value: +``` + x - (((x * c) >> n) * size) +``` +and thus `remaining_bytes(x)` is: +``` + (((x * c) >> n) * size) + size - x +``` + +There is a great article that explains this in more detail by [Daniel Lemire](https://lemire.me/blog/2019/02/20/more-fun-with-fast-remainders-when-the-divisor-is-a-constant/). + +Making sure you have everything correct is tricky, but thankfully computers are fast enough to check all possilities. +In snmalloc, we have a test program that verifies, for all possible slab offsets and all object sizes, that our optimised result is equivalent to the original modulus. + +We build the set of constants per sizeclass using `constexpr`, which enables us to determine the end of an object in a handful of instructions. + +## Non-snmalloc memory. + +The `memcpy` function is not just called on memory that is received from `malloc`. +This means we need our lookup to work on all memory, and in the case where it is not managed by `snmalloc` to assume it is correct. +We ensure that the `0` value in the chunk map is interpreted as an object covering the whole of the address space. +This works for compatibility. + +To achieve this nicely, we map 0 to a slab that covers the whole of address space, and consider there to be single object in this space. +This works by setting the reciprocal constant to 0, and then the division term is always zero. + +There is a second complication: `memcpy` can be called before `snmalloc` has been initialised. +So we need a check for this case. + +## Finished Assembly + +The finished assembly for checking the destination length in `memcpy` is: + +```x86asm +: + mov rax,QWORD PTR [rip+0xbfa] # Load Chunk map base + test rax,rax # Check if chunk map is initialised + je DONE # | + mov rcx,rdi # Get chunk map entry + shr rcx,0xa # | + and rcx,0xfffffffffffffff0 # | + mov rax,QWORD PTR [rax+rcx*1+0x8] # Load sizeclass + and eax,0x7f # | + shl rax,0x5 # | + lea r8,[sizeclass_meta_data] # | + mov rcx,QWORD PTR [rax+r8*1] # Load object size + mov r9,QWORD PTR [rax+r8*1+0x8] # Load slab mask + and r9,rdi # Offset within slab + mov rax,QWORD PTR [rax+r8*1+0x10] # Load modulus constant + imul rax,r9 # Perform recripocal modulus + shr rax,0x36 # | + imul rax,rcx # | + sub rcx,r9 # Find distance to end of object. + add rcx,rax # | + cmp rax,rdx # Compare to length of memcpy. + jb ERROR # | +DONE: + jmp +ERROR: + ud2 # Trap +``` + +## Performance + +We measured the overhead of adding checks to various sizes of `memcpy`s. +We did a batch of 1000 `memcpy`s, and measured the time with and without checks. +The benchmark code can be found here: [Benchmark Code](../../src/test/perf/memcpy/) + +![Performance graphs](./data/memcpy_perf.png) + +As you can see, the overhead for small copies can be significant (60% on a single byte `memcpy`), but the overhead rapidly drops and is mostly in the noise once you hit 128 bytes. + +When we actually apply this to more realistic examples, we can see a small overhead, which for many examples is not significant. +We compared snmalloc (`libsnmallocshim.so`) to snmalloc with just the checks enabled for bounds of the destination of the `memcpy` (`libsnmallocshim-checks-memcpy-only`) on the applications contained in mimalloc-bench. +The results of this comparison are in the following graph: + +![Performance Graphs](./data/perfgraph-memcpy-only.png) + +The worst regression is for `redis` with a 2-3% regression relative to snmalloc running without memcpy checks. +However, given that we this benchmark runs 20% faster than jemalloc, we believe the feature is able to be switched on for production workloads. + +## Conclusion + +We have an efficient check we can add to any block memory operation to prevent corruption. +The cost on small allocations will be higher due to the number of arithmetic instructions, but as the objects grow the overhead diminishes. +The memory overhead for adding checks is almost zero as all the dynamic meta-data was already required by snmalloc to understand the memory layout, and the small cost for lookup tables in the binary is negligible. + +The idea can easily be applied to other block operations in libc, we have just done `memcpy` as a proof of concept. +If the feature is tightly coupled with libc, then an initialisation check could also be removed improving the performance. + +[Next, we look at how to defend the internal structures of snmalloc against corruption due to memory safety violations.](./FreelistProtection.md) + + +# Thanks + +The research behind this has involved a lot of discussions with a lot of people. +We are particularly grateful to Andrew Paverd, Joe Bialek, Matt Miller, Mike Macelletti, Rohit Mothe, Saar Amar and Swamy Nagaraju for countless discussions on guarded memcpy, its possible implementations and applications. diff --git a/docs/security/README.md b/docs/security/README.md new file mode 100644 index 000000000..8789009e5 --- /dev/null +++ b/docs/security/README.md @@ -0,0 +1,39 @@ +# Hardening snmalloc + +The key focus of the 0.6.0 release of snmalloc is security. +This was inspired by a few different things coming together. + +First, we had been discussing with the Microsoft Security Response various research on allocator hardening. +Saar Amar had been categorising exploits and what features an allocator should have. +As part of this, we both realised the existing structures of snmalloc made certain things hard to harden, but more interesting we had some ideas for stuff that could advance the state of the art. + +Secondly, we had been experimenting with adding support to snmalloc for [CHERI](http://www.chericpu.org). +This support illustrated many places where snmalloc (like most allocators) does pointer tricks that go against the grain of CHERI. +There were refactorings that would make CHERI support much more natural, but those refactorings were quite involved. +Fortunately, they were the very refactorings we needed for the other allocator hardening research we wanted to conduct. + +The core aim of our refactoring for 0.6.0 is to provide hardening that can be switched on all the time even in allocation heavy work loads. +We have been super keen to keep fast paths fast, and not lose the awesome performance. +Here we illustrate the performance using the application benchmarks from mimalloc-bench: + +![Performance graph](./data/perfgraph.png) + +The primary comparison point in the graphs is to show the introduced overheads of the checks by comparing `sn-0.6.0` with `sn-0.6.0-checks`. +Here you can see the switching the hardening on leads to regressions under 5%. This is running on a 72-core VM in Azure with each run benchmark repeated 20 times. + +We have also included a few other allocators. +Firstly, [jemalloc](https://github.com/jemalloc/jemalloc) v5.2.1 (labelled `je`) as a baseline for a world-class allocator, and two secure allocators [mimalloc](https://github.com/microsoft/mimalloc) v1.7.6 with its security features enabled (labelled mi-secure), and [SCUDO](https://www.llvm.org/docs/ScudoHardenedAllocator.html) (Commit hash bf0bcd5e, labelled `scudo`). +The precise hardening features in these allocators is different to snmalloc, hence the performance is not directly comparable. +We present them to show the hardenings snmalloc hit a lower performance penalty. + +To really understand the performance security trade-off, you need to understand the hardening features we have implemented. We have a series of short explanations to explain these mechanisms, and what protections we get: + +* [Enabling variable sized slabs](./VariableSizedChunks.md) +* [Enabling guarded `memcpy`](./GuardedMemcpy.md) +* [Protecting free lists from user corruption](./FreelistProtection.md) +* [Randomisation of allocations](./Randomisation.md) + +To try out the hardening features of snmalloc on Elf platforms (e.g. Linux, BSD) you can simply [build](../BUILDING.md) and then preload with: +``` +LD_PRELOAD=[snmalloc_build]/libsnmalloc-checks.so ./my_app +``` diff --git a/docs/security/Randomisation.md b/docs/security/Randomisation.md new file mode 100644 index 000000000..dc08df947 --- /dev/null +++ b/docs/security/Randomisation.md @@ -0,0 +1,69 @@ +# Randomisation + +The relative allocation pattern of objects can also be used to increase the power of an exploit. +This is a weak defence as spraying can defeat pretty much any randomisation, so this is just a case of doing enough to raise the bar. + +There are three things we randomise about the allocation pattern in snmalloc: + +* Initial order of allocations on a slab +* Subsequent order of allocations on a slab +* When we consume all allocations on a slab + +## Initial slab order + +We build the initial order of allocation using a classic algorithm for building a permutation of a set. +When I started writing this code, I remembered my undergraduate lectures on creating a permutation using a Fisher–Yates shuffle. +Unfortunately, I couldn't find my very old notes, so I had to use Wikipedia to refresh my knowledge. + +After reading Wikipedia I realised, I actually wanted Sattolo's algorithm for generating a cyclic permutation using the "inside-out" algorithm. +This algorithm builds a cyclic permutation of a set, which is exactly what we need to build all possible free lists. +Using the "inside-out" algorithm gives much better cache performance. + +The algorithm is: +```C++ + object[0].next = &(object[0]); // 1 element cycle + for (i = 1; i < n; i++) + { + auto j = random(0, i-1); // Pick element in cycle + // Cyclic list insert of i after j + object[i].next = object[j].next; + object[j].next = &(object[i]); + } + auto end_index = random(0,n-1); // Select last element of cycle. + auto start = object[end_index].next; // Find start + object[end_index].next = nullptr; // Terminate list +``` +When this completes you are guaranteed that `start` will be a list where next takes you through all the other elements. + +Now, to generate all possible free lists with equal probabilty `random` has to be a uniform distribution, but that is prohibitively expensive. +Here we cut a corner and approximate the distribution for performance. + +Another complexity is that to build the protected free list from the previous blog post, we actually require a second pass over this list as we cannot build the back edges until we know the order of the list. + +## Preserving randomness + +We have an amazing amount of randomness within a slab, but that becomes predictable if we can't introduce more entropy as the system runs. +To address this, we actually build pairs of free-queue for each slab. + +Each slab has two free-queues, when we deallocate an object we use a cheap coin flip to decide which queue to add the element to. +When we want a new free-queue to start allocating from, we take the longer of the free-queues from the meta-data and use that in our thread local allocator. + +## Almost full slabs + +Now the two randomisations above make relative addresses hard to guess, but those alone do not prevent it being easy to predict when a slab will be full. +We use two mechanisms to handle this + +* Only consider a slab for reuse when it has a certain percentage of free elements +* If there is a single slab that can currently be used, use a random coin flip to decide whether we allocate a new slab instead of using the existing slab. + +These two mechanisms are aimed at making it hard to allocate an object that is with high probability adjacent to another allocated object. +This is important for using the free-queue protection to catch various corruptions. + + +## Improving protection + +Now the free-queue protection with randomisation will make exploits considerably harder, but it will not catch all corruptions. +We have been working on adding support for both CHERI and memory tagging to snmalloc, which are more comprehensive defences to memory corruption. +Our aim with the hardening of snmalloc has been to provide something that can be always on in production. + +[Now, we have explained the various hardening concepts, you are better placed to judge the performance we achieve.](./README.md) diff --git a/docs/security/VariableSizedChunks.md b/docs/security/VariableSizedChunks.md new file mode 100644 index 000000000..001b3c831 --- /dev/null +++ b/docs/security/VariableSizedChunks.md @@ -0,0 +1,97 @@ +# Supporting variable sized slabs + +Before we explain the hardening features, we need to give a bit of background on how snmalloc is structured. +In snmalloc, we have effectively two layers of allocation: + +1) an underlying allocator that returns power-of-two sized, naturally aligned blocks of memory, called chunks +2) a slab allocator layered on top of the chunk allocator + +Large allocations are served directly by the chunk allocator and small allocations through slabs. + +## What is a slab? + +A slab is a naturally aligned, power-of-two sized chunk split into a series of allocations of exactly the same size. +For instance, a 16KiB slab could be split into 341 48-byte allocations with 16 bytes that are unused at the end. + +## What size should a slab be? + +Finding a new slab is inherently going to be a slower path than just allocating an object. +So we want a slab size that means all our common allocations can fit multiple times onto a slab. +But making larger slabs means that we can potentially waste a lot of space for small allocations. + +In our redesign of snmalloc, we allow multiple slab sizes so that we can ensure a minimum number of allocations on a slab. +The rest of the article will describe how we achieve this, while efficiently accessing the meta-data associated to a slab. + +## Finding meta-data quickly + +Allocators must map allocations to associated meta-data. +There are two common approaches for locating this associated meta-data: + +* At some specified aligned position relative to a current pointer +* In a global map + +Most allocators use some combination of both. +In the original snmalloc design we had a concept of superslab, where the first part represented the meta-data for all the slabs contained in the superslab. +A superslab was initially 16MiB, with the first 64KiB treated specially as it contained meta-data. +There was then a global map to specify if memory was a superslab or not, that global map kept a byte per 16MiB of address space. + +This worked well for fixed sizes of slabs, but the granularity was hard coded. + +## Chunk map representation + +In snmalloc 0.6.0, we are using a two-level global map. +The top-level entries each contain two pointers (with other fields and flags bit-packed into known-zero bits). +For a region of memory being used as a slab, its top-level entry contains + +* sizeclass of memory in the chunk (it may be either part of a large allocation, or a slab of small allocations) +* which allocator is responsible for this memory +* a pointer to the associated second-level entry of the map. + A given second level entry may be pointed to by each of a contiguous span of one or more top-level entries. + +This representation allows multiple 16KiB chunks of memory to have the same meta-data. +For instance: + +![ChunkMap](./data/ChunkMap.png) + +This illustrates how a 32KiB slab, a 64KiB slab, and a 16KiB slab would be represented. +The first (yellow) has two contiguous entries in the chunk map, and the second (blue) has four contiguous entries in the chunk map, and the final (green) has a single entry in the chunk map. + +This representation means we can find the meta-data for any slab in a handful of instructions. +(Unlike the original design, this does not need any branching on the particular size of the slab.) + +```C++ + SlabMetadata* get_slab_metadata(address_t addr) + { + return chunk_map[addr >> CHUNK_BITS].meta; + } +``` + +By having a shared `SlabMetadata` across all the entries for the slab, we can have a single free list that covers the whole slab. +This is quite important as it means our fast path for deallocation can handle deallocations for multiple slab sizes without branching, while having the granularity that particular size requires. + +The following annotated asm snippet covers the fast path for deallocation: +```x86asm +: + mov rax,rdi + mov rcx,QWORD PTR [rip+0x99a6] # TLS OFFSET for allocator + mov rdx,QWORD PTR [rip+0x6df7] # Chunk Map root + shr rdi,0xa # Calculate chunk map entry + and rdi,0xfffffffffffffff0 # | + lea rsi,[rdx+rdi*1] # | + mov rdx,QWORD PTR [rdx+rdi*1+0x8] # Load owning allocator + mov rdi,rdx # | + and rdi,0xffffffffffffff80 # | + cmp QWORD PTR fs:[rcx+0x1a0],rdi # Check if allocator is current one + jne REMOTE_DEALLOCATION # Slow path remote deallocation + mov rdx,QWORD PTR [rsi] # Get SlabMetadata + mov rdi,QWORD PTR [rdx+0x18] # Add to free list + mov QWORD PTR [rdi],rax # | + mov QWORD PTR [rdx+0x18],rax # | + add WORD PTR [rdx+0x28],0xffff # Decrement count to slow path + je SLOW_PATH # Check if more complex slab management is required. + ret +``` + +As you can see this representation gives a very compact code sequence for deallocation that handles multiple slab sizes. +It also means the majority of meta-data can be stored away from the memory space it is describing. +[Next, we discuss how we can capitalise on this meta-data representation to provide an efficient checked memcpy.](./GuardedMemcpy.md) diff --git a/docs/security/data/ChunkMap.png b/docs/security/data/ChunkMap.png new file mode 100644 index 0000000000000000000000000000000000000000..cb3a703e94187cb1c50ea8678b972ac3a457618f GIT binary patch literal 146696 zcmeEuc{tQv`1hBlo}pwHvagXf6xo*;*-9$evXrGsVNxPYp0Q?`RS$eW(3sG6d zmgzxc$&ipNWA~o%Y%`PVy{`A|_vde}XBnkJ{7mP8IER9u zL$#!TX%1Yl0wXrF4E3}veC)pUzw;XHPx$C!_xsh?_|HlExK0KhiXHH~diCUd9doio zAGg)lGf3B{PhW4|Gyf28^+>qV{#fCeKcvj=DHVmNN^pzM@vwz4hQkgqVmXt=hZ#6# z?>??oL7efAd^@-UO#Z*G|MkHCdf@-!f!fIRSkDc-@{EhvTIqc8rl-o!AO1aS{G|0a z*SDA^PSr5Bw&JPsdJG{nb!r3gGT8MjelpX&4v%|{og*;<#fwAg?9zCDB>U-!h|Dm7 zyG5V)-Sn^lM^QJYijw-uX(&Dme&BHZwsaEZ)r8ffT$n(ILErjaeORvl^!kdM zYEJ&orU+n6dTE9H#cu_odaLti`ngk?wbNrtR)wSBoy+C1NF8_T8ERV?7&w9p7-t{I z`C~lZDZO}TEu<)fnD4kAn8~btRMY?bYPl_{ntWK_+;a2ACr+nG`NpmPjHWD2?)$;b zqIG;W!x~d)+~<^%fYc>_2+^0JIH@^{+}@ALB}c}&oPM;e;(v28ZeIIVv*ZjOSY+B) zlN%}T9L^fJuA||<12Ht|^_#;=!bdm$a! z690YL>G>}y>u0~L;XOAJlsBhhmI<4-iJKa2n@ohP!+}z5o1BEe75UA#LQjYuyWt$=(e$xC4MpN` zi2qXIK9$9^@_kdQ-^BwL&R1^co>y(f1uriI&Tix&=Uf6t6g2}A1D7}EYHr4#51Vrd zoK&od8!O%XvP!aUGpTDWSgyt|zJ`+!8!)37DA76-x_ovfP^Insa)RbqLCCW8=GXOt zH_MZuF>}pAP917)Xp62&{?^oZFZToEfa# z)OQb1Xc=UjyBafR?XUqptlGlA5jVJ8A9y1^&?kP-WUglI6>(6ttwytLW-}x3e6aqv z-~9_2>xn}}`;JWO_TL%Pu<+|};%}RJBYW0wwqSMUSoy}3-^pp+l&Y{*4>#GT{$+*x zCI(N|{D|3XSRGty3Ov-7;=B1lZgUmKOTb6Z<&3O7CZ6Av)7;$H-?SV|99$Bw`M!~` zIZE1z!N6znn#7I1rscU0Yi*j#yJu?V0x`>XW@`Kr&p#a_M$g3#uYDm7n$4MQzF!?w zYdf#rHn@@#I91QHF>zkAb(3o>HfF9gX71pJTgx?dYf+arJ}cMw#&4>&6jW^{G0sgk zEx+0{iyM40Hp5FeGPpUgnJ`v5u(=VsJVgxrj{7>NIumHwI#azlq^a4exkjiV4hss8U?qobL!9KvO zEh%nrD~E4Dg@2G)`_i}H;rGVx0H-cGv}+_T4mbLT`E~S%UXsoz9`bblTTPE?S8d=$Dv8q|F`)RW8f_9X1=Z6 zTXSA}qu1Fl%+f91qKNr-g{BpD2}7$n9@A>|v;QM=iUh%o?%63Ugx3TN9WHm$EuRlp zV4%yt7JO03(l7fWW8;#8qu z`C!9uX{G$r*$M8pBq9%vi_F}z%qa$6Eoc8aRUyQe(Xg5BdR4**cpv4@q|M_0f^qS+ z#-85LF1!9RMo-!J)e}>}@J>%XkR0s|JCWHMvv^sX0%N;Gb_n?g7r6U9 zmm7M~d(Gbrk7;UVrYj#yyvoe?a;m&;Pu0F&KH-+~Cc4QF3v0Ft^ZgY?>H+aQe!T?l zrjLEqGN0z&brYR!`%ocY6M?_z(4Q20%vjaeh4{0x5KaE7#hPc{zIR3*xEXfzQu9+A4ocD1Al-gPUnP{S6d zdLbm)DWRmv1cm&Th4YSnQJ`y6L7eQ}@QZ(OE$RHBQSN;236(34JsA6a3%rxg&%HoZ zZq&_@r%P1NFB^@LxaB#+X>`T*BzC2E3?8IN=uI^oM73p2_J-=40xuh{d)8GH`L{}= zCV`Wk2_6g2GJg8>w>v`H0xJIA# z)*(J}FZUKzX*0fkZsubE%T45&P4e|2+`0wczhD$Xq{{-3d<@lI_2|8`pC~we48{Mn zib?xuH2fZ%MauARqL@@1VQ=L+=;$R2EM{xtQn=z}!jp+pAv{pH)V0Xz&^Z#xY;TBa z?k!Tci%0H|1-6zqI8{uS%Xyy~OtB>z`>%)C)H68^yd*e}Ps&VK{qDi*WM||rO9p0i zxvSsreAsg--=pE?Dk{UO3*Xz8m%w``egN6?TY2gCMe@8P7<7$?S%g`{^4@%WuG!kc zZoFH_!T6VlW9tt1>nN4Boj7IST+w6i8ksdEUx#UWEF`8df9rlp({YqZ|3yQZ%+H;Z znYmNQ&e0z4E8e;VX)zMhf1*&W>m9|r{0l_;U%3HP;9Dk=$$c3Fq!P{J1-h9qfrs~n zt4}e`B*uXEN{iYs!}&KF0J3tf{HA`MBN`1=uG6~3`-Bip%-Ss4L868Qk5_>Fk?YSw zAj0gPw`L)y;|uwQR_ifk2JzRPPxCUSRrp2g~i4UjDZ?T-9aU)S5xpTjUU_S&%Sq?>FPA7!r2KDg^til75FAI0Ys zQ;|Pp!}%q#q^V5NAD=yAw^yBzI_259=g(If#CmK~!uzaZ_xP`T49E|QJPz@b%JbYZ z6E9=&{m9o@C=-QwyNT7Z1uicJf4;!Tl#^sAsVxTL^0x~;;+4Wd7N*flrs|a_jcTCt zKmNun(ymY(lc$Z!FAtIEA@efyHtSdUd5?Hdw|Xf?RYo0$ z{cMJJvSr;{kvUn->1$S`uHV=HM~{=9C7=tFlqM@LNTTf8X34vogNa>>A02x6Hsg`X zgwBAm-ewKr52o0jRE<&_-D!#%tFiM}sVoGkFBz{uKr|Px=xck~`1OtS~vvn z<4z^zMFlFj)y}FH^N_iRn|dF}D$%P&2aCE&lJPe6D(-_RVnpQk&UJ-`EG2}({6fcA z=S_moUb6lEeCAgbfJ2#cX1WTpTb%yh!5+_v@|6&e3O?~0XwmVA$9rr+DsVNGagRxI z9zR;Ft{8;Kb=hMwee#xiRBX~28`XRHi}^vBjB(lO$Cf( zhI(B6A22laZ$pj39+FaPwfueHN?+j05$}nkGij6Ei=1w4Q`Y#ooOiq}T+LbdcPc6~ zN!cY$dr((Q3aUb~6N#t!%LA^!VtWLP+x&iX&PjjS`0YO{(F>P;IqsEqu(zJeFuEJv zfAW3_0nv4YzRSA}z|bq?axY(ohDyc(6(=S4n7i3iiS;i^3i5FGXysz<#0in!1=sUt zRPc7X3E$To>YPFhiB?NnVlJ>;?k6IFTi)H&G|rp4)ioL+5H-Z0mZ`ewqF7k*5@0qO$j9gb5_2q{Vo@%UXKRM6yKr3w_3dQ z1flIq=Q?Uq$ymDo)KKCpyFiB!JxoQ<=gyT+=~K&{k;|22rqzo64+Lvsx{9vjexnZd zosW?xT*C>8fRf|;iTEtJ4w8*y=9$?VM*a-mK(NU?9_FhkDk-(AkE<&Wo!dRBI2u5& zFq-_;>i4u&IlaEQEUvkmWEvA~^y^u!Rrwg|5x2)gC;Lu1`Fs6d4fc{wj4?J9P5X#AKXPeR;E*+TVHFHXY?q}kX-zZPa~^3cr3|8m}c#TG^P z@^ER$xsb0Bw=t**r@wtvtcUINzLsYlC%Sy!?ycjN5W6QF`Io_7 zGU(2t1{A3-G@16BG|zMnE19`!+P8co&lb2Dxo6AG`BgVwE&fqpgJ-g-C~Y$-YxLlA z2UEoaX38!dLFE ztc%yjk(nlaPC0+Uqraf0B0t5?sBDvrA?>SvnIXMpf}QGGUT38m#%WaWxkM@(oZXXo zRqEnb)D=LE{x}skApgsSrDCVQ2=-QMwr0&uqY9$AQr``AD(&%XMW#+6>-U^1wZXT0 zZ@78Y?CnbyHpy4woa4LfSFK1`woX)SRm@+#B^NvIny@N8rh!uSVV1aU7F>U-! zqAfljwfDYtA1af}S9U7{F+ck55j=;~hvtGi6_ z!MgtXtU{AGLBV_ZlWf`*Yc{$)M_ox6-Jumb{csDT*tu%94z7{pWKs8T@}2h}Ah-UMJ)P^8>Woij6bkk8KMig_Od4!ksUc{M%Pp_mK(~wa+C=yFpwCQ{Kax_j=W`~qrleNj1l&=q*?5#nLmW7neJTtrE zj_XJiEO>2%WCGd2yL^5np--LM)V^Qt%>F22`vali+@5}r$07we%nLe)^7*-wRw0Rj zeMCIrj6czTJiZi2W~8Hzt&DB`Ru+h#n|@hc?x?78t;)2{=GoMXIt>0@Iid2pt(OfT zt`Fh_b+txZl@Ssv*5qw-G>{Ap_IrO74d9T|iO0c9U#re(!GHR@H-0=K$YwRpwj$pJ z(K6au(eE|wPpS?RQ6v|84XD1(n&br9q07c^Aa+r%#l|LrVDk*e4Z67tZJrFVd`{z& zLy3ZuhelOolLF(d0|H>NZJRw!WVIvY^~*x%CPTOvrl3m6RbhTP{{x9=yF0vX%bmpV ziJMs-Cs2Qzd!@=HUdFb0%gc}EuJ~21M0zsXjdZSG&(k1!qVgY(aIaKqT&aqA6`)_# zip77%F6#Pjn#{3_E5>0L6mzB~SK6d$;!_XOv$RQ< z=1=l+|Hj3lNQc{Bt?~S<|B9OaQboy#Lr5|T*`1s zMA3!k@7TKlz$2Nzja$9)kkO5*+BcEdyI~!dM(79dz7j>J^lzC6zzf=VH$%8a?0C{A zy;;UMV)e>qS}$RxiyC9=!{q5eNlM!`wtJ^8T}aA@dY zgA8}OX4}Ns5uLwes2rQN0=t%xP0D_pk9W{tQl))15`@-`rL!6#dVK+9I*Z3Pf*) zbuKc^02$|k_DN%0Wfw=Fd}EjZms6yQru%00p~g5-Q*hSN#ig)|MXW)kk=|`SJZQ9= z-LHIs`xT$x+oCtlz11=;?LfswDwgVBiI@2l3o8AnKkK2b@>cG&mkPdPDPz@@9V1lR<0$={H7Q-qQbi1%hgAfIh%+I(?lp3f$Cunyxg2 zb3gCYm=ZW2v!S~xm$8fsSkTneKe};Wb5S*K^OLcH!+O02X5Dpl#(8blDz0fQr)H@s zW|c{!Wc>!XtwKxyXLs5Ag7*14iusG{Db|fd;;gidZSHUU%SUbE*k@}T*o`NX(0>N+ZiVI$W3na2rlUb|fIDc@8 zGsF`wlXtFfzMQ-kwnMArgF#)9Zb$tO~R}JUZLD(px}9<+-dx=XlPFw4Yh z7L_(Hx4y-LTMoTXZcYAa~XPpmx@ROfREd2^@uRaaf8!ph=OOQ1y$YJWdpK=z0E+r;RXQ z1N3~RCznu7E3)REKDU^4Xd(N0>VaSqr>R(@Fhaod_4P1;Pi9uDa5EAEqYjfp0caO%c$WTix(`S`khRHR5)`Zi|_QyaU)-$1ds{wyv zfSDf;FpH2In`+ChB`MD5C`5~xJaNnr<{&1j<*hl99FubBF5AW*i_K?w7GIr$xoz6< z@a@b4Zh30T`cY;XJ>k$E=Jm?D=&Oe#`)9C0a1@ zi&M~2E|Ug}R=i#D^p^%*DI0cii2;n1m{8C){+@M>rys3UW|NLKQ@KDajQ0EF5gNru<|Csq) zT92p?xDg>RHF&2!!+#fK4_a~LZbtK8lIOcxl%ge!zR-GBQqMu>iHM%#u%P~#oi`gC z#88z@&EYAB{$SJkV{DQrk15*6wF9z`qI$-C$+yCd3UoA??z!n|ZxYaNVCDk)Ft?*Y zYF%D2SB?Hb)f=i|p^VzeK4Hvs0`6vE{Jg^QxIw!SLH!F=ITx#PZrIFt4d-00{l{B) z^6ML<1xyORRY>xRy3ym8U|u8wa{?v{y;sOH!(Eka@_V($$tW*0S!Jt3aVZ82du|S& z#R%Yt1&jkHG!FrbjfPkplHGXl6^Tm{!Yp4NKl1Z?MdmX6esXVt_s!z(6{L|S5wW)< z2d>XD3DdohK&mq!)p=h_kmAk`+_$E9Nm+-@hW#)eS;dFq3wwr~)EahWSsE_^`@p;t zJTntunUQa;SlL3G_K~)g;>A%wFYu5Re3VAy9@8kZ{NoUK=e7Q>uBG|4MX$=7=9pQg z{fmFc;OHPZS(6?kb?WaBfWxq^PlwfX#4=bSAM0BYJkA3&jh3|yC37qav4DW5c-g<$ zqLEu4j>fVwzSf1ug*8Uvj+*ISzQo=0_%NVV)nnMjnKwfnRTQ)ea@`^LoN-2V+n1CZ zvvNGaayWTc^UCrg`a(g944%NpGZ!a~@5BgZR@I!Mm=J>EYPN76XMk`Jp+(pTwJSui zMFZT?Y8TAXnRpuhjUb9q_RL>B0eo6u*l{B@=UYOEk(Z4lMo6h3y;$7&WIHp#OmR<& zECNQ4J>wuSn{#Rdut!a>1&KA4y_-AoI^r(eBjAY?uADIrf`9dP$u%*O{95W zp4sW!a8q9p-JoQyrJw1-114no!F_%~K?fGjM}`iV3a){%(wAW{zTh~SDx9MKGV?)Y9Cf2jH7g5 zZZ;Dxzb{;T0*@C@P}Bhg9YBRPY9K;EJj3RyKbk4pPP2hTeN!O-!8wIBg2+ASnAjEkZDajq&bnh{cI{k}K4k zH}nKfxw9f9JoK~ko1xiba0)U98(qcur7k`M#O{UD|K+exAmLNIM$g79 zH0bt&jgUC~n$Pp`Wj*YVm)RrVdtK;BiEESk=CnY8)pOE(yP}|bN*7rKrC4=Ew0A6k6D^dEPuB~ry- zY6{b&;SgY@gIRGFfa70FVO(Q;%=ntTrNjf?Kg9U`FeGLcUyvaw2Z|S`$WE-z9tpl- zdRj!wpE<+s;sf^$=#K|0MTM5>6i3&oA7Q@l7){PHtQ6xR^c;z_C$7^Xq;8KJ^9dnt zVwCogpC^^^2N*$xV~em-T%spPgVIm7Hk;YX_pv@2<+~+Lwt1HXYv|zT|Bi zV8Z86wwdLPh1dH#3q;TjEaABRHtqpa#MkR?4D}7Va@IfLW=&6eKv>H&SRZ=F{l@{p z0o0y$wyVRoXPLAslHi?F=A>W4$nuiKLGs_d#P;v_|&%+h);+{q(^9{g`R7^uoi0nF@K}*bb~8MimmlsHc8r=t)V5AT}T%3zdIz&VGxXSW{;Hj%XUN8_y(1g|iREF|{r{ z|FqEA8yp^Lf0qn2DWorjk-x0JuTR!L3&VMn#*)&^i7!V#^a9(XLdOkZ6TUssO3R!& zy}PebNrdO;!ibqzGc`Rk2|jj`U6O*ssK}>mS83UHgbCQ<)NfRcsGl`mW8)}LR;z|n zQ{!c=zl4b%BQ4_c?Ks>5%q`#PQ~un<@|5UvYf{`u-rD#v?@KSJzU)VSU;%8mkJ?fv z$}}P+A~0;U=fh$5eT*Hy$Kkx6_jy0xj})WMiG}8;q1;5DFuv#etyzZ+nrh$A9hnWM zX~+mc71$EG=K7jv?G9>^)<6a!fAW?tn6;2YVw5#qW0q-O_YTCDlwgyOkxp^??Ji8r znL`b?^6%TF%$~B$+KRGgAT#=m+;MNhHz3Ce3r|;(d;JXt# z&Qxar_WI8)hBl=3(1~OIFn<|IysJ2vSJXlw($yj`U8A1od}z;|sDpcBeoiI3PwvFN z4-Zw=h3G2iR_9&vR#>&a%I2Mp^MN0p{5zi;CX!#f0CU>6RR{X}urEv)fKkDzIQiNi zYbGAUv<@j7u#6QW8WAFU;!jv@u?btwECf|Mmjnk1>RNXs>x3-34CS8ITr+!PrXn0% z{G5I5#D9tW&1!_&Ry9*S?)L&_L= zrhJ#@OAq!AYW_EIeWFTS9#OY22_aUA!{5g#dYiyS za>Y#Z%)|*XH0&nE)dc<$6K0~Or^~*?T%a=P!}yR@GfQwM_Br!Bsp=$NrBd=}iPN`J zemJS=nUJLXXZ_QoIXRW#anp#rm2HTL zW=`1ZA1U?ZBrKGFE7SPWV0N44UnfOU5p-NoE$${QLN1xjjZQnkCk!!~)0aaijUA~v z6%ZNv)}R0R95N_K0B`M_S7DreX>4fa6hb&5jWF_UDsnqcih}g|Dl;NWoGp24Z3<@aY;-|%0x`r z*NapK4)*NSUm(GT$WCc%t&%w;p%*xu*7__o?Y*X-u4@17&t|XKO2SjjSz1;k91g^p zZWmW&=IpD?T2BGIZ-w=(J%D$box7iM94Qn8>stC-=CDjtZQl4nSU^JnpxL0CbTqL} zEnk)sp6H3}SvYAK%C;6BdRVsV3Jhy+^;1IMar!k0*+p(79e?{V;`aWI6jtWU7|b65 zPkmd%1YT#7`-Y!#fm*S?-^KZQ6=_7`3fR_}%T)|J_-`k$PR^dRiUGBzTpB4wS@Kd2 zs#OWn)C6m*^am}CbQw*HNkMvVk)ZjLzIm36WhvWfo%Xbd4fqD8_#>Ub>$Tv|F@??H z@J@pdRVVX})QZkYz}dz4Xs7=xGA~h1h_2YebGC?Pbg`P5Fkh*Ded>^e>H(>%>qYMAeLi_teF_O5=C2r3Gc-5aw}MOF7PIciF* zu`;?pf>@-?*eh>tEOLV%1iB6BH!(4@?at?R$K?Qbtd9u?P)rHcHqvb-{?)b=1hptS zm}yhx#a?kRvm0++gKll)cD)+=Edg!_(LKtKs8S2;n@3u%5gLvrTK*PS^G`owa~G8W zE6LgF@3`NCSf1C*2eAdol0v*U(POH~`=ieuNt+r!!GYa??rV*Iik^gre-*;+(M)X5 z4d@3it3V<_*Mv4s4eSOzFb4(@`|!A2assB3#g}#2H<`7L3qApCFJ-|F^?I!7!`^X= zSO-eic5X4tD>h2USe7!a{pq^~(r1qZ=Bm!p7oeV;cU-lO8N^f#uBh8H*gxKmbPMUW z^v@DbYy=KKtp$7j7F0N;3-A-{L6UeOzFsyO0YNFtKmK-84&H}a^Vxo-N%e6(1>$AB zgpbB+eRkytw>Hb|pf9Y{csb+VOd~)e@-c*I6I_yFst18~agsT4WFT~M@DfbYE^v*_ zGL_6JgU>0KSnF%Soa(*{3ks=1UO{xx2PV@90e2ISTzS zcdMC^Ac%(b0g-9VKvRC526VJ7L$`~b=Z$BCwZH_nr3m)x2v zbpEG8NQitzMePTfhcyLh=KASS&z|Pb_u%S#XW3?hW>llGIW=8}u zF(p2q3Xg54dMzKYY0=AP`nWEXa0k$~^*J&f2=)Xxc3pV-`E>cH&0h)O5b={^xn&2t+t;0Bge3$!2KAv z9d26M9t8p1)G}?U>^)x|@QHx_X4A>^O5KDQ_P({>eGPzHGobc|WXffJv2}%C+wrmba$=a(UJY z6za8&?Ef`tHV;#EJ*S-xlBWC20UAo5&d?tEP^>|!D$f{wr~6nzVqF=EZ`uL)#Oa68 z1RrPzE`7Gw7T`t&bD&*b=u~CiQI;23@Xj-fiptY&Dzm51$G6MSVx?JfC&JX+1s7qF z$WxxFQ7}}ASJGCi&70I}tNrbtsM@I7LkU)6VN47R3x$C53nya`k8gCJ6+KkOPQsS7 zfAWa&R>#Cjltx6Llu7gGu8YYvc`JnX9FAtze>QKOJyc17S`x}5%h4%RM;Em|K!h}4 zhtod%d3>BMZa_ZqR6|c(@}6 z>QgJsj$zTa|B-iRR*97EfjoZOqHpDPNhdhKfrYu2OWxg76Fsb>z=42@qk^}9Jnh$^ zcGcUbd~ksAEM_g1Tafg*o33zh_t)nt-ybYW z1ypcL*c3M+cT5+Ml>tYn*t9*5ncnJj9H@2U?n|n@{KFCp@BEQDNl(0>2E%>;#bdzf z%^0doae;$Y8@XqvKk4u_{~%B?&~$zTj_v9*I-TcAK}Efj$K~_Ah6T|lx6?AIr~ZQZ zP5_*Iof9rm4oOP--sVyVz;n(x$#Xs! zHXT|SPJ3hRBz3^Ht9M${e&8oposr-uYJ|)y^FLse(cb5r;<>nvi^#R5-+X@noK6$h)@n7f9E?^MAZ?ObWBYvWlAh}5|-O(WKYBf6F-qaY9lo_VIO=)G}rPI&^8;e!<{-B*$sLlG4 zF#_ghHBkvv6ZFhtn~W<&52WQ+>>jhzY<0}opjpv+gAr*u*qwh zRAJMi4d>?t?dS&GJD_3PjsxkP_6TW_@l_uAO>=gZl@f_Fj^Jhkh`xnwfZ{ClS7S0B z$Y1n^d7O)(cG??#EC(m+3bX4R`sBNxIinE7+{$Z1Va z5AtAP*pU@_0h<0RZYR^;B+uvwA&epq(?p8Gu+)jjUPw^LVmE!O#;ta^0G$$KBLQw! z&5Jeo4yT>7w#p=wl?>gAWpR%n?AP4+v52fd*i@lht-bMS3s5xKqjVrQiL_)kZL8x& zZiXk*vUX-e+229sbY8EW>Y#xmhmSoX)Y%B46R2U*v^sA2wmh2PlP`pTG*4~^96h6O z`iF#4RxMgi1NK^p0cYMDmKHETyB8%+rimsU1TqRPs4gvZnQG_mFgt>Hg)mRr&4+cG z=D_jfQEMam-C0WP+O*2DKl?-=W2UmtwEg%Yl4YmykLz!bKdx3St!JYtzTXAH|#YdV&c181o{MyjjNZQi0Z6Mb&C zcAAsrlSCiaM?f@_@W^QxwvZ`cs-|dUbO+RvE)tlSM;^%#%6uE|eAbshvpFJW+rI(6 zFuqE6JB@9c4%AG4<_4L6@om(7`G6LIvNJ-ftT2Of%1tUxle2*Hx2mHplAJ)Z5n{E} zsdQl`lwnSsQfmUOG-$^*gtMmzQ+8sVr&)7m*Z~q_0iH#h%^q$}c@55ELFt-^W43Q2 zuMDy%@zQP}Eu+|cWP$|KK^7rhiO;N>NlAZ(W;$lNwISh!u&ruxld45#Yi>UdZWo=? zmj*mHCcM*YjC=On`(@x8@xO{m^ug^IxJ*DVc#Mbh5)2FDf~LQ@fZJCwf~3x);=zeE zJ#2^(O=fvAe=W;8z)$Os852rV%U($yQmn82E8Ul!p#Rb$uc%0Xn-^bKU_fNS%}FeV z)dt~1b@r!q$M?bkzIX;25%ls?9bJXUFD)vgFEOaytl;w$RD76nLTroDA&X?3DF~-( z4fddG!sw%G~*R+0h zA@YnEqimmTRh7k(q^dIVIa!skOqr`{OEMwzFu^yvH^4zdp-odEt|dtKWu!o>;^7C2 zvS6utBtm|Mrzm2 z0y>$zORI?r-`jd9i0yaX*{)Zsq|0r41Rjz+Jq^xysp;9WSW+1zA=y2LA zX5iQwmEcftI+2BXZ1kKS9kZ3r6DeCJ_RC`|ypkz_r0ok3R91|wZA+}veHoz}y#DxI z1=KFHjVCIZ_JeJ+zjXQnq3v~w%Ff4~@eO7(V!>%+9x_lO*$vrsO{!KdXsDC)@3anx zC%>odZAl}#LI}-=ZNlG9s=;cFNzpXJ%|XTBug6&UzPjLMkw0_ z$}aI(S_Dz;H|PrKT(ZX6lBy@zc6>ICNhl&z>@*{(QIl;01?YuEl)hT4!qRF9A&9m+ zpdSaeIni!0XBQxus;{#)&Pn4-;x&^?lQ@2!lp1g`71~V%jB~BD#+nkE3BuVJO~$6Ty2&HOu^`nMep+3o;WYS}h~@T^x;g^~wMd~PD$#%{VCtAH z2bXT zmrv5hqk%X|lWRMwgupo?wCAp)7P$AP_}DKbEE;cPx^~T+cx*IPkZ<2=V0E}l2VJW?!z_YsCEchoTO>+R+Pa@jN3ZFof1K16 z0CTOqM0ys`&|JI5a)VCelSCz~R&Bdh1c|(XnNsKISxmToJC5VhOlnY1IR&G$f3-z1 zn^d=#Z=R;f^cqg<_yoRX3l4L}RT_PhYfLy?z#p88++o6IM~}N7q&M3>-dO=82oo(r zhwA$Dg;+;DL1Q+?sHc2dlACIVinM5V#O>7FbiMlq*lA|p<4DOMSm8}F9v-C3Qc!H%l)L=A)8&4N!6pu7khw@`ytpuCz9^Wj$E|B zm3EGsTQ%n3lN7uJBp1&l5--`W;0?>IDEV=7B&{R!vpKjc z0!fl6=l&Wm<$+V}`|b?j23Me`;>_8_;tveG>?Op@iS4XC6#R3QdG@j7aZ(VeThOGC zE+a)bKb7lv;F-gu5&yLuvdkwVoK33U>Ua)4YZSj$FG-^?o|YRY)tHL$xj{w}#-!_B z8ION}2b7TFXnaDi(<8RO?w&M;Zg+J7P` zn~a@4l)$mXZdEJ~KC$6wAUu31m1%K#e(kUo+~b?i31#Ao0*43!WWRqq0r-$joIWkx z=fE>xC>MN}ql{MWASvy9yqhjrWTA0Z?ZF4O+vzXxaJ2g}C>(sFLMgZAW4h!C5+FjN zcqQV$%^E}qm@uECwfFnkL-zGwZ-aRbf;Bx7Y6#aE6>q={r#)|shV$ao)ix~eW-ToN zih7&=AMemlQ7-eciS&sF>#f<~VD-pLS~)*KwBi}Wfn(Ul2&pJ832xQTu-Xz*Hh+Vn z99NRgClL|)2M{Or;W}%s&rfui^Q$Urlz^`s32z6W{R(F?NdOXk7Nq&cx(BL%vXy0C z2px0MI1F`@L|zoj0%bGUUjONdlN(Sc5bbEu-FMSpI38mGAzd462;%{nH26{+NAy>sjTru z(#QQx>LpH1(pP5R)5__(6(1XTSXuS;#a%q&DeIzb`&8c;eh(=gXCHG)sq|#fx`+@$$iks1p9REvQ#LmrCDj$_UrsDt< zsnz#~z=={J-KwK@D=~AAj$!Ye?~r)9N7Awa>ymA9mg8lx@8O^2Whd`b{Z{{b46|%wG+% z+>c-EhSnRS8_v6LXTJnmd~Y>rWY7hINs%57Mr<0D|~O=IlVLYr=!L zqdD)gHsO$wm1ljs6*Z(pFNvWQ%fS?Y4$wO)xMjbZdpxmj;yMtMXA*sXEk1w_dQ`bX zFmtVP^9p6vl6t3tSq5MN{ktelX0GK4&u>`pn29r$uDQIwK^#^rTGZb!veda}fAUD( zF@RluGEGLl9=3u__Ij>f&sg$b|<1x$d9foYD`SpjvaVJ4I)xy6g7LqC*+EQ0Hi; zY_SPM2gUC{5Y2mtUHka3_Wc^On*K+Romzx5A5>n~AYkKWyunkv1TYVXHExvK#bL#v zb=(m#5!>4t6~+QRXFG>}(iL{{tPsMj_P}~m3B!Y?9@CxJzx`mhjTh||Xfl9u6$=bR ztXzcb_bM=F>=o9V^Y>4;F4(6{o0`IQgl+HRXaRN|&w6al2qnIK^A6jCrAmW6Xn*|r z$I@F1y(h{&9fK)z;he2{55#bS>*m(?z_(^tp!G}F;klm!36vn6H#ez$G8G2Cg9BZ= zQZ1OitoLASzkK}BtKj>LE0)_qjIUWpAM8`f$Zl^M5gHgYUVBSN{|>wMd%HfjN&v4r z0A7_fb5wSwb%l|PxMsj2_f_$LVhXmSa@gqI^7W-^sIXt;XKv(+qvHaxQ=1+iPG8I= z5^ts@;fI^&w|*2Hd-SF>BxOJmc{?c6)iquBw+GhEjl|C12qR63H@1Az8rR{dhy4RD zN=O7jurlzC$viF3Rglt5<|Irc(2A)?oE`NIZCrm4Ct?%6Dh74WPX1xp1&~5h8*y?Q zET!(ghbFB~O4<;O-@~JL7(8 zhfY=YeGe9BK4UnfP7qv2(m8O2^9NCgVx3yg?tSmDfWV(|jg2RTeDHm}@O$kiSR{_u zbaxxKKZ#QA==L*~S#Qez12R`Fpg{*5R>fGjsz_3*>%#PlG?^kH0Dozw>4s44T8my( zgp>46c4yjSQ<(d`d$=#obeOsVZ`*DMB#u^C)KR#A$;QB@f;c3F6$$7z?S6tEOo?0B z*rK@1y%_8d>CGz2XfGV@-W3H_41^hU=ir^aqOSo#qb{iiS@2n_dVhNdQi)RcG;v4m z@%ukZ67DB-+)7dVK($*t++3%d7ZR0SeZy-)^Kkc-?k7o3V30-s3FQk%gDl{07n9Gilt3q9-7xtK9OGJ)Uu7u|vkZ(aHORTQMMIqTrb!CYXNa_hSkUS?@-& z&&PLl6HR0QtEL_^{ORn`a@L9t+)$yZ?d-W!^yJG7Qt46mFp&oO?ckLIJcoF-B{9Px zf`PG=TyoHiMUe!j4J=qxklvdYx}CDTnSIQexlZ9A={J$c8@0hpc~H`R1BjpdMm20C z1UKtGyPY9r1GGf<sCeYw zEwdlU6;a5Zu9?;#YWUcnM3EY^dc&^Q8AC5LlUDu@dw>29b^pbWHNLXhb>rQ3|yWs>N!`%>0mT}Yb>9dAdt_X~3;(}$hi?K24h;a?i^XC=KJks|Hway?kG(p8#WxI8_?%$U5Wq*M@Le?Sv0Khft%Dz(I!jV%fr8 zxsx9c$u&xDCk3O)&l7k7C)WLVC?5 zH)O&Zo8Y^w;)E=l&v;Q&F|!}tcHJT{G5M;)ag{YIJp#{fLsJYDmcZsly_cGM^>iJp zIBZzV!Ce9UmtzMf_EZEOJv%4=-?_fh|^aN8he*Mvg zEiaa)%zbjo53<1|t(r~sfK5KSfA6gb`X5zFrn1vMyHMRn z15XO-uh?zd(?>mrF8}o?^!~-tz;O=vrD!DhR6h^P89a_v3xht!b}3kj=qTv<>c)Op z+a$I#)RYIfmqRP$8es{vZbSSC(FmEFK&7_zC8Z46)AS$HLKwe&#BKo=$qNW%N{4&B zX}q-urPb_@YFwDR;kZVh z*)CA~QE_`Cd#0d#6lHoDp@h?a2Ym=3A;vd_F~||E7T*J#S4#KUEhAa0%|nEH1}hb;uXcQ z&K)SlHYs4b8m~>?{rU`l$;6v`u+i|2_sp$3iElHko*rka!iyYBtEzxHgpKCxc>C#I z-o5AJZKzNVZ*%XNTBSRe!de1pOZ5`Uxm{YzogKolhx5izS z?JOVrRdG!^=mR!_l$P$xT$U>SX6pqbX_;l7GTl)>52LvB@dsJhE0lZpp++BDmj}VR zxoDs?rVSnk$!hQ8i)QOTBAU~tX1B_*7jHzow#)4P;mgnUU4^z@W^s4yTE8^Tw!w|V zZ5L`SR-Gm8asO+w8tT#RM*rigb}trefx(si#Zr^qW-uZ^P%~dmknm*n%=~{c%)yc0 zlMQ(rt28l*BYr2c@Rb8`Pj{hc+ExwZA7Y;Ftmg&OjJ2yD$;aj@T5@u)hltS=cN~uU znPsxgufkH!2W!()(q-|_vZ6FDA~6_yFO~+9F9b!JZ}4$dj>~ZiLtt4;#bvE_gTEbv zA1O{o4R>67P8QhL>&u+XbxMBLqVDw8G|HtLEgR$&h`Kv8F5ra#uA*`QUtGVBCmGcG zHD($Lv-^JRXdeIU;C@N}zf?}ibNW%Ju6ASqxNSO$qPpneI4(Cfk6ozDV^_`v2JWY> zSS9X6aXm;bh3_3fjUL5G$#ZJI@Rg5EtI|1oIWmT#@%t%H>Y^pft-<-VQSg}1^Id`F z;bno1AfjDYD3)y)Rhjxez&e(*6ICTWa9p*gcM5z$gE=mgTlRK>%AiTgGv*u0|CddS zJYF+(RN^n_EAM3}`%8W&V8B=o+=6ja%m=VUO5b<)m2^SBgWQ>I7X{u>G%iR7snPsD ztvdh$w$(eZ^*NmAp1S5Km+_Z;86R1E2g?1u1W>^fsIUz1(kVLc+rivM@V zpQYrI{caDsX#I3=v5Kb#LJ>iDw(UmI!t3|~iRVtCf zXX#&+p2x$x7JVWj7MEe2Qn-9)e_Bw%L$NRuyP`z1`x!UoNa;FP=ao|F;eGi=eBo`_ z`~QF5`!Pl131j2dnWL8t9vfYLp*Z7GQeC~`1`bBJzH6+71dpxb@Z9Cj;pbTz=50^6 z`wq>i6n^zC4%ZoYV{PAg?Q_yW{|Kx^9E0!K*rYNVD+i^6Y6bW+Jg^i;DA_0tbkIb4 zc+SYt%UXHq%;kouJvPS1?h=RDMnyHL*|NE!LUvs>z{DgI3}zy@j>5v=Uy#{|>=!-K z^Zt^qB!n!^iCypm9x^S5R!*M?h*nIWtP9~T&Ew8go4GvaUYXA@q!4hzwa%!Lv~+E4 z-Uhu^>GELhS(~e|#ezo`;U+o1!(NZezm`{f>LJf?-lnAv)~7ueIy8T;fiE-oLDVzm zWSvv;#>Sk0TJ=uprm~hvO((ZB?6)_Q|rJ3{BIhn{Mq^D(TL{tNr$V#DWcjrDUydpJ+j+e1@cYo zt=>swbpO1lJuRuf=||V>nLQn{VT(!9K^n}OhX&?cd%%|@un@1IyZN#w^*0xfnKirP zu52tyj<IbJ}KyGtlhB_u74L+FX_+#A4 z>=jYU`#&~Zbma7G_M4~;iyCl4471MeFmxIi$)_Ko4>_69DQQwU8`e}u`^UI6RR*{Z z-VnSQHjhT0rZl%dM);SPWZJ=xx=sG<>v1scrfkMdd0v6%NzKpK9%fsYO1K<8RoQAL z#*R)}PGZ2n=xt6>zZYRW-ZFAZJAu$UdV+mKj0wN`@MEu=hlgQBRaiH{TED_=#wvO6 zq~T0Mvg5qYX2#1`-c?y`x%nhkyTJpM*M5BVVk z6Zp_^Fk|+m?7h(CavuYOnUJJ)yg_e>mBdI^{kL$mfiR9xXOgV(aa4IN%;4kwcQZ1g zSbynQY>U#&$Q80!b4vGwa%n)u%{O%>=yx;os;kSJF8t3k#A)N6G2^79gQ~05Y9TT3 z9A<;$#=j9?6s*}Rk8Wm&$O#KR&u#W|SI$;qm7nsG5B8OxTf^N{oLisTWp7a7s}P`B z)HZ8Q>v|*Gm`*%!g}qmluqMx@Z$m6}SUKtXF4Xm^V&7S6axco$w&A4L2Shr6@{>ic*{u$(j&|3O1z{ zi%=e9%PVi=vmi|4|6xltbLeRww%5<+Mtfvg5cP6gsPY#zCkFMh?L4AA%C(mBj-uh9^Qgdtz0j6Pms%WY|Y5 zzmC_7ombALKE;jCt;mY(exfEx>+j*y;cnc0DZv<)%~Wo35trpEX`W?Qo9w|8$zV|! z$$*i70{!>vEo3e5!g?fk{Z68s1OOV{UjX1)7_$4k!5i36QjcXf#rx`Gx)Gd>$bY7M z%nT|%>I$duHuy1D>JYg#6CTIN@v02m46+_)g0>?bxdMKcNy6;h%VvVJ1%0a%dTs+j7prgP>=npkWdE+%o6YWp|ppi z(?Cv(&Rrt^p*3#6#A5NH(28Z2eM*f)Y_e>GR!a@h4`1lfeTgh9=~}7v(AqzsA$i^s zjoDnUumA`Ozb!#Qko>VFC~&hwqMInJUn-=Y8>_Pabh}&iu`yF>>u2tmY_)#Y2;;{= z)_e4i;E0v2)*1;a((%!xAhV??*9Vexx>pN`v7Ii_v60wQEX+lIVcS zj=Vt1HL?8V2(S`eDL($8&DF?aNKY`I&@Dz7bgp#0k4-eHbhjGm*~~W!J+PT&zY!By z13jZ#)2~R_U=8t`_AWn+vG6aZ$Ia9UVPBW)umma9#Wzfhm+4`XqX#l$RerK*in<(A zQHU{_TQRD1HJWLntaNS0x@Hf$)84XUG8hT9IxAz8|BxwU;xlRl5UfK^}A3VVBHrR(W0un>lcWvb4mhkCcNP1);nk)UV zcI-b2vHyH`>pz1VV7FjfT5H!54`=f|;_9Alz30}2nw}g6|5NAi`&u2eW9wQ}DZZ7~ za(1fjYvEcXwaHILWk6BUY~?@wFx_}WqL;8K1uuldo+@tPY{KFTVZRd{{HXKKj#65d zww{Sl%N*6zgD%?*HDRO;V$IC^mft{R<_*)BEXgf|S;! zdLhXnhpKfBW0#}k5*vcrEHV}^(eV=;IV?u-^b96>rMH6TUqoZl^(3FF30x}bVDHU& z%@-H%!9@Qo+CJM_Cdg&ca+s8H(p7gYcz`P$Yx3I+)T7#sxO$6V6VSdIsv#+k1D|Apn zG_qMNJiOLgiysXg6f+!YHLY}HpSa`9@u2~)vyX_=TUw<@GJ!pnVQO6~=RAIGO!Ppf%reexC6iZHjImT$L*%cob@#(dXr~I0EHh6n<*S8v)eUa>rh+g@1hkI{{Et^HzRZE70POo0uQL9_ ztHfBd)ePuxg!>j88W+^}-!B}g|2`2i{{oNxlyt}6^8)1wS%qm6ntX*;>rvwr_6p5& z`HnZ*#mz@d;*VAG8P0@TvP>JU&Cj&1nU6?7C?h>NbT(*des-CHraOzih)w)rF%6uDLq*8&Va zN-0doX)fxIFQLMoLgXZFNt8KO&Q<@*JL)v0&Vyyzn_*>C+IsDLtY%W|G5g)xTK=J> zdT)|LEnZKog@4yNkRb59-T_ehI58=FHd0A2e`H?Mu1gK;dOdce$)EH(N;@?L-_ViA zfg^BC42^tjFQV$aNH(sBtjk}U*^Kf>)1p8EIyNB~mooP!%Aqp*;g_>XXUcq&TW-ou zX0>ZQ^Qb;{Ypk4`?*+P_$1Gf^ZiK>zP%USt>~dn4`QkFcd?6xc z#(i=qOGsOkcxq|I2)}w#khDCylo2%(HeRM_%y<2VAl#T~?whh#lj*?b&gK)*n;M3t<|u_A?lH&Oa_(f%=Is#MtmqU7bI zZi|^RcT1X|V-A(eM%c&FldkDmM-M;8OK9>+%GFG(h>qy<=1ErORRoCk){K1F+%OkI zsI7(tf2ScpicU_OH=6{R3h$ipCj}gS(UyKjfxo7h$-nxTg&D_fBnhjf+)J_7IALhlr*ZPMgRH2VzJ{ci(<)XLWJ zx+}jItU!4x3d357S4?4DVvJ7F9@*5I>lrpUiYdb2BPdiCt$vpf{DZ*%szaYK-VlpXG6ZSHR#(=(J)VX_dF` zw#g0F#-(CbV3@tgod_2?c{^b{Y^m7~jBr4!|3L3F@Dv9L*FOwP`81!P37;pp= ziy=B1R-Hyl|5InC#HzIXV=UH!XQF5D<_jKcN6Zvp`-oW{$)m&WFBn*}2XiY)wd#{? z?H$W*8<_zcYfR?Rq^RUKU6}EdohYB*^#0L9)ai!GZ_f7k5wGiJOaf+W9-<=PuFzoKpf@TtDo z;{W(qpI@8Sv*c!kGji!&P`?}hW~Or-2V2-Wz04IR`z=x_h2`OmNsX)tW7OG(Q973s zshaybp0NlvjkagmGa06)>gHe-?5LoAOQt^@XM}G|CiPGzuZnR@4EAohC#Ks8>IUSo zzimp{W)rb!yGmIfRn6&g(;#w8LCxmslp9qBcrt{-EwJR+(zmCqn`O(5TA zXzt{2q9(-|O@4n58f|=_KkPMaonf9_5W)mPu;27d7x=?0KSgo;3B$C;O$`3MHEJo4 zN+*873y3?E`lYSb`(rh~#ez8v;MmQX)s5|KoUd}i8)==jg= zBRI|drYYD|qvW0nJv;A(h{I!^5##BwWc-n?n*X?-C z!@sDfqhrt7cQ+YL#b7dSUgMpvuzQwaRL9|}SDELio#EOrnej48f6}p13zkSixzgob zyKDA5KOj>!=tZK#r0OG;#Ac7eU;=y5OtvgBkCwQZ65c{N584ls65wX$jWT+JuZ+?C zfxBW_oa2#RQ)>NB{*Ox6#WXn#Dk@uR1xta~s^1GT?t5jFY)&m?M${B?38>WCo1dB3 z?#g2v72j?@o^3osm|Pdm4~Nx5nAW&vX-^`0%5B;J@vgF-ShT;xyh+Y6{~jAYOzodt zL>&aNAs>nM-(o}Eu+x@QKw8oRQRaV&+RszjO8xZy{oL)d!}~HF=z4uY#t#>9!mlPbKHKo&FMx_WX;e;a)Qr^@hHPq>)FH0wanKO^oN_$uVDmTvzzIB3P z07>M39Is$V_ji&9ZLJ5d9`Q#$1;=YTE*tw+F7}6Vf4(BDK9x|9Xg#|Ow$)542wxSPcMtQg&1x~Bb+wwJ8F#TD)YeH)E-p@#kL0$Gyl=4& zzzb;$*YY<_jwHXS+sqPg?I9$btcA8%mNT?fBo%i}1qBsUKLmf=4r_~kd#N=gZIxcb zF?U#0FB_fuPP3j(Q}`51vb27OJ63k2r*h4oApzYDA$gxK%Pyt6ev-fKyGcGh*4sKt zj#Fz{^PkpQ*K$3kPB)Jj?d9v4F001GciKcz^kl~-=5QVO=#x32XB0V9-d#@(!0K4Q9(S}%ijCM)9hv$&D4082ZCW8U_ENIx zf~M(Wo+JMc{%~VrhR4*ZScRS>;Zk0J+O)FhNaxf*nLt7Bytu?4x0fP9&E9RWS+i@9 z+k}79VnDzGq7EqQ|8x7*Q`yVA_<{-L0T&6wtmrS6(K=T-`GSq6{ck7U!RiQanZSr;d zCA<&1PVymEWB=)8t%7giZ4a~>#xHV=XfsJiC{7Utv@%FhQQGNhSGfFJ$M9nx&?zh= zoodIPN^FkHmq8L}5&VSYfo7hI#u+9ds}B<^-dI55om@=jdKfq;Hj(AJUNbFseJ-*M zXHZpZpF-61o!|XuSx2K`XA5#5;}$9ZQctHbtIAu~bxCKl;6SlvYf!{7;0()KtB4VG zxX!)74i@tVUX}*5e}`_1L2s|ALR35&H}i~pxkoydP)*Ee)j7jA47&+!#*t$q{=9J8;~ zibc8$*JTNeq;*StuE~5uqDGBdN=8G5`H*G>8zJvz!&_r)7E-_^t*;b3xPf22j-HP( zv2d8#;`Tn^Kfs$VbOKR3^YcKKq!~ABa(OsaLg|Jmb*}P`E zD9g*rHubj!#O9q0lMc&du@J~%wRt>o%RS*nm04MKm`F1Ro_RcG-k*g%)oQ6-d0lJ? zes6UWjN?MDMeE(N!)jj)Umjmr9nZNrvU&XAJO-q1etvB}Me@E!G0-zJhKeX`{n*!i z&?tcJ-VoB=yBn|g{**!#n>G#oC*8TlK@ttCR4{%6q|-hOl+k|Kpdx7N1d{v$)oB<;8+7D;88SN`wot<$y8;+{_wsx{ABihmBK>hnlOk=GYG|NDC|RFtr4yDpsZ zqoGWhiB_q5^=Sso+cKV0n`64B|9@`bc})DZzHv4OhZRFPZw@_ITXQE#jArg&1TO>i zRrn+FyW2JTnjN-nsR_J${*Q-^UjGzHo@cXl=xz;xwqv`{Oc6&{zPptO@slzWwe-0= za|M2(#XkmTFl_&szCqqf%Hn&=9XVr_Sh=NcyPLZ{p{a_T?Q_u!gSH)EfhSK&f2I)% z6`~1HqS}m+-xkzsd}Jv$yvYBqV_7n_HkndU6HuTQo+-sAjp^0;dJkMOc<>prUY zN1d9WZNKF;n)2*rS6)bg0F3g%>+T71$I**Hi3*5kCSUC}XFdzSXW8U2X1$f^X)#+VrwYrOfo7n|Dc zl$3pMh`sdwC$lmRkgN|7Y{UhlX4@L}?0I?4n}dV+$zHaHlswkCS)anYX4)9A3CG9; z85_vQ<|US}=GiC|ZGfQ0-`84-m)LGPjPdI~m-X-H-N~4t4Y4 z1g{sv8Bk_f2IA3}T=$XPeV(an58U^keRXbGn8T^|M|p|IX>sm9?+pKVl>01o#KZCQ zqfBDZp~45G1wAo?(_R9{uF_R@i{My^PEEf?_7!h_Qq5)CdpJ6Oi0Z1CX>;Yl0A_K$ zjPHsQ#Roc-s7%CXaZjGex2zI4Fl(!ku1^a&#bvfVXD19g)hT%glw+8^oNUAnUB0%o zV#YGLTG?@^5qsT(%=CeGW1YfYt7iSM3I16x7|Y?~T>r6~;iS|xHTuh+yVqP(52#D1 z@8ghM`PR-CcdgHc!$~Wf%6wwBcAE^4pJO)RL%MIbOcA zk6FF#yT|{M<_K&Iy?!Bomw?Cc;6wNRf0 zwxNya^`kax_J@ikue;H&t*Pl2oz3c^ihsHr_2~W#A)uRNHD=;X$ zqb>`f1y{t(o2T=SOI=$UpX|<>Xtb^5*H5Bi@rIx_2`SRU8Q)qbG*)eknUym5s0kdh zb{NrP=R4N4S={6aww#COv@)j(t~isvpBMFPzPx`dnevMBdbq_^d9tIO zzI-pt^oIFA?wd_a*eTxx3$X?#byj~Y1j0fj^_PFc*1f;%G?mX*8Y??xeeW;%Mb}a9 zH^o`89NtaYZfr-S&slO=>p0$aY2Ui&U+wt$f=LE6ocmJUgi3eHw{Lv@<)bW?YfBx+ z8_Um!krKwVccQ)q3yFN0isJHESXNx}VO8o@p})G~ykdnyDXQXvC3&CiivF^Em4#7Y z+{o`J1dVu|{$ZZAmGBaXedDsA=b^>=G>P65%56d9=!NZf1t@lxX5AZFrSN4L&J!z3f@Gha82CK3)H}*gTrOjDj7U*uD2U zom$I-7^ztEu!_o%L&9Es6YP`>^$w75YUufdQq2|>!qZEjr>DU#sfUx*3lM0EvY%9s zS9?lp&G1cL%BVk zKXTEo3RlWV59Ywlz~6el>PgoZRWuKKyn?M<-*Ghw6`deJMk?_bt(Wq z=?x18o*jT>Ouun24s~}qTqJo$;pfN)96tgA(+3YSv-_3%YiSA?HIE(K6=Y6qOPnZ~ z60N18bz6L`$rtyD85}_Nf#LdOOXxSOlFhc0-DyF9z(nOezJ-EbIXwGk(^NS8b_EW1 zh!VWgMx2nOOs$UKq5iHqb35{ksY!jp`y*8f)3k)>QrVVYDu zX|N~^E~j7Z7}-G6Fee~T#F@O)I?FnIQ46Dc#tr4ki5lEpx=Gag#*(*y6;=qB81}N2PfF z4-5U*$F5#eOQI=MNgCwU(PdaETWHE)W@mU4VsLVM2w8^9=Rejko#dQeT`d2|F-4<=Yj z%poXGzKhfPDdkeI%{DbS3FxE@yI6Dkw&_S6!%6W-i-xx<2@nEwQ}MADtn!2p$e9ye zzMG6iEPltL?FKj->Kk<4a6Wo9^?lLCr$o8u0kh9-I?+q++Neh{XVrW>B>fh?J`h%LHycQua{zl16tbYZ*MveBpvF( zPyPFEgO1N)6Y-y5a}=ai~D|NZfES@Z2jhZ;r1G7B15v$YbN zr>u2t7s2H&EOq`;gudJG?oV?#u)E%_It~P(W40LYU zKu})`zg&-bEwSwo)Z5n5)~-D!f60>q_BZW_>qxeMD45o@hR%%{I zKujXa`&mFyf0^m1X-XCy*rUA|{X=d<$|fOjTlJ~-0U*jz^X8c^9$LP8g3NPLvSgmfMyK$2W=@Pi z;$Y3g@B@`}ir-kj3%40XFHjzKpIa60d(1I_i*37ffVpLQKK#r1qgNukxX0i;sDlhn zt1YgO!+o$6gdlJ^%`pGd|?@N9K4jT2A` zJcaXsdTtnBxj+MLLI;w*CYKKBT{2&BOJJ2}Qb1~f;rn-MYOlhyJ6U+|AqMxQ`M&9z zUf=Qng9%xbnG5Dfq1%ujFdBY!M2Yg>zq$Y3^5U2PbHjJYHckRqfIBzWLy6olpdPuJ_cmT zdD0?Lscn(ibZa+iR34`)a(3_JtHi|OngJ7USS-=f64@y@dbM^#F38*oD&XQFeeyMp zj4^jdNNE^u>*M)Zx$kV{SB`H@8*huQzZgvU$MZ^GR>&1GyXNs$L47V@+~_p}8mX0e zpoxVGswmW3Lrj`6$@Ooaie3fJt52gd#dXN_+DP8VQ!w8bT`X4zczrU2>#ltGAz?qN z9?77^lV6osbP)cWq8sk>WI#4~@?Vr{x@p(PmS5-z*=^QYB?oQ0>uhJ(8qwQPw3ZTh z;cLx0GxN~miIAe0J3Ue4PYW-%g2LB1p;;a4yQd#56yIZE6f$?3(9#AU{;sz^y0EMy z7bFpbq!I_qmc0e*>wpR|TtW_IE5h%=`HcUPuY7+mRt;f|o$WbO2&%=5h+Zw5)mKnv zK)tm}6dn#dqee-j<@A2m@wN8s8%gRE#uEpr3W3z^2{f3f1M{7~e58je&7d=Frqy;W z`&LOrAjSc`-^{Med_AJyXO(a0DQ05kL*O z4`q8)IiSqgawL!oxnGCFp`a#BqtW*TdV%1X+P*+HDW~16piX;Oh~Y~!hJ)n`J5ZPG zCRv~K-b$hg;8fceyZAUd1N&1t*KD#n4My#*hSM6M>=qpqdzxrzEyq>@WA zxdT+s-84-M7ywV?K4x}TE0NNxvh&^ftu&O3WlO86MmyrjVmN*x zRg>Zl)T01kcal+M-nE)Ua{;sff#%$!1*!b1X;;>yOJ=0zJf#>S`NX+Eh6ugCQD47A z{rK@2lbs(aS)@@}B+8k_pGR4=vx3{Nkh3RoPItpF6oL{HjnquzPILL6ft+o znfYQnm*?iuY7g{axJaerX6{|Pq96&atp;mEB1h5S$o|3=iPD|2-Cg;^elk!!&v7Kw zIbHHx$ou8DPll{=jCaJ{vTbRrf>_?S^#A$6>>n^mc z2u^6dyO6&ar#0dldkWT0g1K4i=qlWCWk-m~iSSk629 zlnHy#2oP?N(C>;l&zQ5aoU1<$Oi8AZh&uEsE-5FK+E?Fc)|s1!uI`NgUQcS0+oquT zalROA{lNF`ke)6>|F`#g*1Gd8azSYFO7pT3u!&1zA*9)jrDT0FG?ieq;~lSl9xt+# z-FtzoKMhPT>zBLbYT)8z?IP?p_oW+B=(on~`6j5zNAr&gr>s=k#oHQ#K$k3$X8%n+ zaBt3x28AgDY$_{}P3B?b9WOWUm+98H0d&V{@{0(PH!@ zTh0-SBLB1h8c>=PT|6q);cDtD_nZb<(<4$YLm5pYZ={1_su;pI62V~oT-yUpkx|O< z!b1lbYO-YDc%>q%d}XWTI!E7YKVq#{5M;GfW!|Hxw{Iz$v zA1*Ci9g79r`SW_z^X7P^)ti0${K+>dX1b0S7vw1M(45tRd>A-RY(2(eNlF<*HJVO0 zEL?z;%laf97n^an9^m{aVtG7`)aPy}jvqiW&|uWY>hYkYl)ZyUAMDu4vp_&J-0zMo zR0QlJSc$WXrO$*pFMRa)ZLp@8UUM=S+&HSTX3i>~<$8DKK;QUF!MuzcNZ#$5A`Pat zi-7Mw+BsNykc*uZ_5j2V$aJ@@Ok0z?{B{S8I}uWVOL7Bpm0`C4vLZqV zA-bJ|j)m)c;&J-&u$Xthg?_%u1l}6D^0n4XTjFHO331R)iv3v_`Nxf*08aF)@XN<7 zcK5FTGQ1FzbP-%%A zuI!WyWu|zMsC%KpE;H?E>;}Wr<;yfCQ5^`Y$T7yf4Y789*nmJR1jWmLR5(Rv0@SIz zg1#GKBpmt!dT${1sw90nxCYYi86|6V_q!%jA=rI6FR%93FH19vDTLE zPC1zAGFCZ%A=t@YP@NcgNT51J-k5o_m~@(`9)E2;Y_9yy%nc^)Kk)_s5D44l1u*aG z!BJ``=TEbjdeMaxD0)bWt|obd1Llu!sGD$ruSS#0bg_j5ibD_-9=K zk5^nr1jmikd&$Pm7lZBi`>B_-HV2_a!5+2zlj)bM#P9&*h>oNX*Q2*b-q+ix zF)vWY($xd@&gQC!`bqocPVh;WDM??W6jSzy%6-3fhEI%8#-$U8Vu+J~v{GJ7T0Fbi| zThod1-`3A(g)pl81IZHwNKmrM)zAcjct%3&A4_pYJBo}oP*uCAg^-w_w^iF`?r~=O z(kIfdCe6{;`}8D&&Vjvp! zWvTph1F*^Lnf-mqRaeA}1+X%wWaryOg`b{ZzY4jzp-TfvC24Q}{@$r>^6qV5R2FB~ zm%eCF6$Q1#++J~muZNZUWEm1ruvxor^QYxf)$ zeTr!V-Hm>@Hf|#wQj%|=2i1jOOBMFutglKA&0adkbo^5#tj&?j9g(COgmK*cb|9sF z{Sz1U!ul;I9}7ljP6%ooIA=SOVw4@67NL4)aBZgq&2tD>(1;k`_??Q}+RJVI57VvB z)G;YkliYgKm_y$7WHa&&vgrkK*)cE(2Ykh+?Op%L8plD0uv~n?v<*SFc zr^L6tUM6;u5}?u9K%kCbxJyf9y7k!1n``xS#g9tuW6jGNi$|fpzQ33$u(S?KNsjRH z+{a-}72Zvfg~HUFmWt`VUwL4&DC107h+lgqijf{yGdD(LLX#ctk+`h}n5ncqD63?SSFZhl#uIOX=amAFi(XFF z7t>7PGlKp$PVBI?LXExOaB78%nX8K!0TDz!-5u>V!1;~|$VY36t0VBbn~da~t7TT? z3wQ@;#hD>$hu&vuC7wH!r171iBjStgww+5^_vSf?UfWW3Mp=opUseHH_Xy9!VZV1@ z9<)HQmqfjO3l~}R;pb=&Sz;>@)i?;H6Y<6#??QGj;o3J0m$fnvsiw=Xp6Og-ssdfC zu?kvO>|?%$M30dTVU=q`k^}DeEAVU(bq<)-{esaHLdi`Rs}PdSlafx#4L7hT^Y`xM zQQJp2#<^Vd@xUbTHX0uH;h(m|`S7RCkmvuG^F$!$`I_?OT`W1aUNt&djN3-R3qh6Wh8aD@Ca|P zJT5b+mWU=SalE>-1_Hkl-jXzt2I0~2s~^$oT|`;u-m?fwie!FSiygba6Sd&2>Nv_8 zf2~jpNU2rT(-Nd(4D;!H8;v(0B>w|;miJ{kG?zQRkb5G;e-@MN z><#kvT>sgRhmH$7lhieuy-Y1LD^Gj&$;IH{i=72PSXSJy*T3Zr)!NfC^Fk|G_{k*4tC! z;rg^KVc03$9jIz@s8g|A-NSV$icex_Ed_n&VV+zi8BuNgZF6(l-cH9zvjC zQx?0s;Um9{HTHYzOJiU27nCT_{4GiDB>2o1uH!<6r(f|_zW0ZBUq z8XHr@Ugjy1E!1h24L4Qb%iBpCLV$w?qL!vj?hmVIHG20pw$4MMOY5O1qvSQjA^cAY zYO%pzI7zPf{}`iOs)lm#X$6lWfcIM)*ib5J&gDH5J4Hcyzko>ah!hWWIBN@VgA)^! ze$A`g%C(+2+xMb^Y}%S}O=x+45%laOT#G{z3t6|X>_TxV!LC;ju!I0u5-mwj1w$bk zaB+$2T4o2T(QzZ2r3;=HLtm%axAO0pXh{|oK5G)uf+g4ee?YGmsawArP9UH+YEgEB zX|)GwS{MixA!YW`Q7naUK=cWuu|Wg+KGz0Fdx4>5P+t?L3oq!bPwwq2mFtz9(1cG( zmRO!(0J`yR`?A*G04@AY28j1-FPz&xc9#{)ba&NaV-7O=5B3}Yomw0Z$9FVZeZGqZ zoa725_p(ttv?+9!3=#1SA&ArmqPa{ll=pig)5g{7VZ!==wYj zvVQm@-moDHAwYITF6fURbU%lAi|(cQlHK}i1Kkf&%c{R8R#K0zH_Ku3A#By`vZ?LQ5628x??b=IaTWU1sx~ zO|Ec^tK@*TGNp<5Iu4qaV?%NvjJ5AFMf==WA+v>Az>QI z6OQ@9wPHxo(*27&hra9onF{{fQ)6)1q&HiV4vIS@1^am+0zR~FUeW8vfS9Q_LV48! zy(aB4hC*?E$7u)rtnDvf-w!k1L8Vpjxcf_r9 z=a9y2hce1+=VFkhyh5vDj=a3e%khFll20}|8aV^2X+tffez;T$F&iO1hZ+p5SwPAtaIf(0d!Ool57a`U&GVMk z_%b;k5BBZxRtox?8qy+g))&)i_A3LYwCVpbxKWJ7Tfn8b2|NCwqi~HatVE>$XT7bXB#IpoI>;5qU$-s|vez zE!T+{I{UB6+JCZdv0PC4ouy<-VQW#=sQFS3m;h-?9CU)YprbxHCug5*_h@FA;Ba+3 zcRF(bTWTnS*WIv4#m3KC3I+e^~=2F8^g=dvjfq-1el52i(Q`Aw_;{n(;A|MgAZlZ2US=%z|w zAy=S2wP^p10o1^%CQ<8s21ZY8HrW6MIbtuXr$DbvzijkmZ8Ysu1W=$MVl18rY8VrT zljB3fb>Z!5XIlDou{WX)h-l1m9ib1;fIi^hY9PH8Xdj$D45}8#hRS`+5m?g{$fRM#woP~eA?NysQsT>#6C6wt|UejZll zl3XpsyxJytK3W$Bk9YvzFSW+B(HA@gc+5)!KsLM^gydGPM-G?`NH#bSmV*rj5-}Pz zQeqYA6ox=D1AXsate6cEc(7?rn>XSO=b!eM-;o2J<`4WAhKWRU`0WgwN4@EXPXO(p zI?O#BricLBG~Qz_A}PX6h$&(*>4RFO4UhF1H!k<2tqf<)%)nyWb3+91V_^jKjZLk2 z0)*rka_EMpS#&*y9JtX|-$|WPca~yoWujEtdP5xqC)43 z(*_wXIE%MLq3A@`UzT8;7k%E@;k29!CDJlrt)SOidk}7j;4j@M>JVEE!B!ZQ63|(x zp1tQh(3cT&=Tk75H>9ZNcE`=1>f57hlNI<=IBU!agKWVV;%$>j2 z&9+nBuvF+7)k2m3lL`XrMu^|;K=3}!@u8kCB3{WD4oYS#d8Bpt@xkbiwtArH7(fTw z?nM_d=mo>*ks9p*NL}&^Xa1+chPaO7D+1k|5OByG^*Z296(&T(m;~{ZzWk4Ei~W!- zxn~0D9sLS@B84jG&AnH7yN;(Yh+jkoRWo8*7UuN1txm2~Uxe|csqG7@fdH_HGNKrg z?R@0|v^2p362-|DzqDeAJ6B}{gr%Y%3&s}lX)uxqpl?Pwrj|dETiif00q6mp!om1% z@1RotI-KqE~WXb?;SAImn>@d9VV?Arr~jXSM^`{8xtJJ4@#F zGB4^QLoZ^Ohq;qbb#hp{kdsUVmsi(6+&+J~JO9p4JzUh+`tb*}NkP(W-RSv(J`gve;7-&Ewp=eVIbqtDQ&S5N66 z(1L76+q>p+CMs9lpT&P|O3G|vPZLP4cVlYXVTvk@GjTolb`1s(w03xaDf;;g(%UeF zI|s8D5b45o ztY1{PZUGjS+Dsbm+QG!q=&b5ywmvjFwe!#9A24`V;}vu|HR}}upt@=)gXFijxx0k& z$ZD|H51J15&81@4y9!9p;j%>9jlKT*@y&*UHvkG=yRw@BD0nWNRrK@vCkn}U$=BO~ zs3+oh%>g9Tp|^5~ie}OLhJAL^LAtfceJ=Ff- z@Kxea&Cu|Sx3nU|ZHqb=k?i%Wxyd>-{+V?<+}GGyzOL)OzepWA2K+cVo!NMF>uY4L8AQwsQ@#aYlYPLY|E1~dozhZt zh=m8#3&15oIb@wqaJT~{y4uAQqFBvfQVxlu3Eg0B4Ljvi^p<6GCtg|_)yN+Dps+uT z4z%BAdb&-+yujVEIjqGes;qV=c&M~M%rZ#GAnRA8Lvr|sXQR+^5%!GU!$lysz+4vZ zdwb0qlPBo7p3iOS7#I`7QM2Tay{FThfTS@?#L@mJD*3W(EQTanQ%W*`wbbwI~W@vnR0{rUS zZo(*v95_rkwekAX;7DIj%!N}w+UH$`Bc>9?y~Txh!GD^h^U5L)zVcwZ(|FMj3xaW^ z4tsHP3I;AOLMxksRKlp)*~@_O?uZxOgMHTegMHJlp=5se^`y;Km5bnZ!tcaZ5C)D({C%*0JnPwzsVjk+j=Wlg{-oMM&l~@vBWnKDh zBS6`;t*$>1WqkLcXT039$vC7B7aaFSEIz|w^AH(n!%S=++|>`y^?;Stg^gKqd_Jrf zis!7(2GSP_o|^4_D&zXt|=WOZPg0HOW=p%W|i6A$*YKzEE1>$n84 zXE0818{9bxn50=|stC7>`V&oPZ5qqzGBebTW%TVI`*1Z&Z9vi%K`nJ!^55M}x#I>` zVV$XTJ&3&G4b)2>qF$JoEl?!FrrO0!V3LcW^SmE#2GNSu?Y(l9PK>_O?c; z1gt~Rb-Zn+E5{kj)%8JE>t6dhZ4kn_+lj+3r{KAb~ZESYX846BAk)XdU?CBO_l*KFNtX}lwEe`2qoDGCX4lCur^7qFEkksOOGKCYvSeQQS z)$Baupab+a8F)n-8>7KjigE{$-j|E7y8-lulz$nr6?teCxV*67Web8jFX*LJ;Afc2 z4=n$4JA@@x?Z+_!wx5VSKNPzcx;DPX`TCDDovfgPFPlKmH4+9`WBzgp0;`~$X{&7g z8IX-XRzO0%z`A1(1%$7b%GF}y!H3vIi18~hKP5C?y5{5g@vQ^%@$_ekpc-PdU3W4k zj&bS+bFA>q9d#{Rot}QHou;a>1dSFPN9uz`HFC!&wy(wv%t)E+dDYQ1xflYT6m9j= zWXss*zR3mJ?@h_McfVz944S1sOhla-4%fa#DYy`9q}}%GiK6KX6H%JY$~agJG*F}X zVedH|Zh_lt`rwaGdXw;)(ZlQt{bAA*Ki`!%6pcvH!`paSf^bg|mQc|d?4I$X@#|Rtss1oikv}%*2lO7unn2)1uCxNT|FxO(=F;TG zBUiP9tZ53}LdBTWDR|i=3n0m#fPvP%Cu3;x>iMUfst7{rnC#-}o^OZN96*_G3WI3c zzBQQMH9@bIC02A}1`Zf8c=r<%nvjcA04gaT20>aO`N3G+9&iz87t0i+gQ)dxHTTJ` zGYu60} zDDzxkrCgXeAiQcx5?>ua0If>rm%zg5R!XWCzelo`kn}A;8@-wFB{_QPyI}uN{5B1C zHs-@Q1t55HfscH@TDwQzTl0{&IyG|oZ~NpnatKBj!Vgw-#cc5@wlXh6Z4Y)_3Cr73 zp1(r(%Pmij!qRqVMw}RxNCcplMQ^dG8AuUbUAI?Dd(--`qOrU+`2A%1C5#a_w~U7D@!sX z_l*A3RAyF;=DUi)S)LQl&BMdVwEuJ6`+@md_^!${RLWWkuH(}jGH}(R)d2Ea>}}UF6~v!k?JoC13DV4NedJGTyh~m=fOJXqTm^zP9v0PEif>AW^}d(RkWV^>obB$Z4UZJ2f+Yk~?ija8s3CDiyr^U0_* zxx`2Neh(QuP)64qFqC$KtOkue@TZs$J!omtp{V6ZAaA;lm#VU=!R2HrE)Scfbnv^myGF*U%x;^{|Ak5h~DY!IS$9 zrH;cgi|Hf2`vbTko%1x?4yt3`6k52;xmm8&gIBu;@yyYo;A>~;TpJ%I5cYuOa+xcg z8)cD|=C~451=J*GRAUgzR&XbKijNEsP&w+Jr#hvbZoZ*7C?dNpcnjAh5-f5SPm-jZ z4G$xYbCuvh36SW|IVt}9l;4u=27Kb>BYHN{eFdV$XBK12Y=`MD0pR@l(<^p4QOugc zql_h#vS0klxk2p4<0ZRL5KeZaK=CQHWnH;ejv>q zO^(gBXsKG1%dw3AxV~gN-2D=0Da)Ug(qqp;A~{&og$+r3-U^-$e5L#)qQ`i*9bOQ< z$m&Ay`Z(;9UhJnv`)^_bQsFld0G$eeih2N%@J9vh)RxiSg^1m3P9XaJ zOV-#O_q~ah$(Wsozo}TPG-I>z;Q2OS5xj~>-x!QUTf1W0UB-iT z=0Ck6il2>HaL1y8U}ER8_!5$K#iX+zRh+U9Mr5;uz&!#E8f2k3ug-|6==$+pujgA< zEE*z03?_E-->NvbdKx(%UkNeEIb#U9Z{eZ3HzzrI&nb9zssAD>sE(@_&*_*CGB^k3 z7DMB0n4<&7E->eqF#I|0{~X`=Yo5OGnw1<1PEb*qGQ8lFb`*DX&irxxop|jo0M(j#iC!D9!at4cLp?A#;Pz zE&ViHH#~w4F8gs*F8geM2eH%ZfDs5U-%Z4b_)Lu-G^mz9r5NRYZEx(mg>p&l!V%Q~Cyv6c=d59j~ML zPB-{>9izC+Vcl)dxF3e1e1ILE^>l{8E@qI7@*u!(1rY`Hi>8~9lfw_tUmP1Nn2X?! z8%yH>HDe>J#2Ifp@OVRboR}pi1o*f=;`H6M(P#Bp>0lMtIUtCm(5P5qK~e#DBZQag z*K`8R`pR_E>z^K*O;7??j)R+nSZ;-6pVbnHUG#Z%VK$jZHsCbG1|-H4wWBje7+9v> zF%bepA1H$zP$_R381W$vu-JxL$2_idJ)Ql(Gb>y!>@IVtn9TiRMU+OA@za$?Xz8xI zHX0XgEbf0B!gmUyk;`0>m-`$zs^!7P&pGImF-?WeF+OiAMjzY^R#1`hbo^U-nGee2 zTfNzGz?kcg%%g*dBt*eR?rvzK%fZ~043@EfhLE=WTc?I*2sP$4xa7aSp7pDuEQi>^ zvH(99kgxsAA4}enhMa`h3mvvgl{4Sb{3|(zt=8>pZrwg`TZ1q)Fo{UZD5ZF zG4@!F7uorBkHM36TB@9y3mjOs{5sSbgayglGf3eCGu{toI6UZ{bVPBA*m~YZ&Isw5 z+G}?gwUJ}f$VYPcNj#-cr-qoX!!Lc~U(giK=ZASZ;E&Kjdk0pt#(trX!$?a677@lJ z{9&FQdBGZFw7}4Bw4tmR-HwFLCK@QjZUqHt{U(2mq)SkTA&E0E&K`uJy_;`n@HMRk zpbT(zCYSoHaJn2I97H9u15V4gZ$%9+(0EJzA0nguFx0eTHpI`*)5I~y&MgxZfq(g= zvbu;J;~$|#7sUL$!Sx<&tYB%Ku8;pgf6x|o|Ax;5@>df@4@#{kmpN`%E|7VG0gxC?23H|A{3bJK@f*Pv%7de`20?21HiDR z%`R%`j7|-&)}P@fgbjG)py}S&xQVtWh+O*cyqQyb$q;-UnOYSqgB>5xQizg-y2Cuc zdffTTFQDTVcFQPnu)-~a)Y1V$vE}S1ie5_rChf^uDTd8rg<^< z*XK9SN-=)q5A|JzT)$K~Bn!vvZi&S?5y1W~w|1C%m!; zXIG8$TG_kNc;ReA;_7Tl0&0Dk%nlvqE)ay)cOK^BVCUoM1qkI49ybxd@2kD;cn><| zmm_>Kl0V}6l|8Q}X|(C_wB1_hODwr+YVI?vLiR)HaU2#D`1_%L_Zs|3ieTyu;wi;! zfu(gGXMyjtUZox{%#BQ!L`@Vu+W<>-mW!2a9ZsDTo%xgZy`y6|_`#|VY55d9&7n!d6g_fz;$zS4;8Jx4LI*J!Y2i6n7`vt`oxFc>9rapdQ^12YhQ@dj3 zYTA>mA+|CIrR1LD4Mc$!b4dZE_2W7T#(@95Slz!$5jY0}Rq8}Te$vI}JavK_aL<-= zVZf{&M|=l?qn+F;^1t1gYQi1nL94E$kkD@$>ze*n(38bZkHChO@k&q!nIcbVYjfR7?&C?qxF^<< z8+9Qi^D?cQ_q-}wy;*STEX*3RryaVrPmMVG2FPc(IX>l(;>6lRt_<=)RnN0x-?Ocym3`p0-`2 z%K7qo{FBq%B9>0$7wezs08dop8(ePzpOUf^D0IT-W=V@z6PKUzGMz|b&e|Bw9 ze44nfDe@YRCkuoySdZM8od=aylB4$5%$FrOTt#x<3t?pr?sqlF2nU~#0&8NeE5cUZ zvN1Y)+08mS5?qiEtFM^Clz&f z^ov_&{(XpCxQCUK#aVrqAe(^XVX13ONu*(!9IEb)5VBm908tPfI`Z7~!#V;nEu8Wm zZxC=ly1niNyKR@~(YI84MZ^{W4j2U-a1@20r&YP_lkfHSomNN*#zO+F-V|rfiJN&E z^Ems`>P>SR`7*BSr~4!(+zK@6rndT=C*rm-PVWLjPsIPDf0+p*bz9tG31~NTE$w?? zR{roN+i_}9$BObk35+{$J%qR=B%5M7i*&$y>->gE!^WQOW`D-J$$1c$%~lz!thSZO z>pK7bMxL6GbFH&uoC=$yBJ4_-!+gbLS``tuSN=E>vZzBYKxHx8u{NQ-QXIAPWPRG~ zY22X+wTk7FdaA-N^eq=c`e1nZzrVhGAL$Cu&zJMk>at?@{cb2|CK?Hr{P?a~byg3& zBd2Dp5*80R>iU!sDP3k6^(K=`t%9_DtXy2y=#m9FMp(qXw5TkzlmiW#Zu596{^u(R z37_pP?*TLM_@VyU#tyQ(ed&~3^H!(V))GR3uKm#1&m`M%K5G!8Z&@&0X|6VBG?D$^k-^NgREcT0_!lE?y%2KPW4`4DbdqfJlu2|itP$64BGKlRxr|1JJyZbxCB zo78uoT#_V0nxeH71;#Kx&4RjGT}@bg!Kij589WzAz*+1Zt#+GdHoiUE^nJQ3B`9lK zLr10_w0($`Db7U@LpkQdVh7b-3 zqHxx}x1~aoI9}v>?;dpakMs01`v>2ZPQBGXJ85QWWWGGf?k6Q+uP~I8!^aXuY1T&Zq1(?mwq(8U5TK6F1dujSE*Tms>rPZ@bz%27a{Tb3^)+U->z!AqpPvHxYRK8@~snH&2lD{-qL*=HQ@-$uC z9GvXGIncL2ikq0e^16f2toID=^-D%YYTaU(KEJ(tTKICo{0FtT$-ayq{Q@N&ss0Mm zwxTZ}Nhq2?l2~d9?1CK?X}afP{>nrA45Pai@9l5fb^-Dnzt+uT=71E9!wh0C2y3g#wI6KDn@P$l zd6X9T{Zs7?u*gV$UdFwtK2g ztT!InSrm39mCyevA4P$ohC!e2-KiBfdEDy%uD5`NKBX;_ak@-*%&}Clr?De*?t$*f zZ`p_Rir@Kl%MQSbu7%;R27mItYVqh_?gL3g;@620wYds?ewpKXstj8QTz7i0ox7{k zNqQmQN0cD8KH4=4V6ZHes{jb(8e5k=V4cjmZJaxvknZC#_`5>)DLv&CM^gALu{(tM z*=HMy#_Y(Lnn4@S#bjFsd+{<;!!~)P=F5LxsX4t@CYhE-y3a_x5`6x(8kQ_1Uh3|U zNhIw*wW7hY<@W0-@5@%S$lcJmrx}vWiQAH{d4Am&#+VO8RZhZWB6^5caX>_1ent+k z?3{)>(PgkZ)xX$f$B`ST?Z-nd<2YyObQIzcztk<(L#k<=_Dde7VNXt%^gKp(ZoDP- z_ZYTy8A2neyC#WeY)pCS2JxE3NV@mz*5G0C{O8py!8BCUX~e{Dtt_O5lAFXSUFLrE zoB;vib^k3j29su@I-)I7op9iL=V&%(XTjsJ1?vMLC-x(pw=? zQo}jf6URmNj_11cNHK_m>@><2a7e76{&Mi}?Dtx|VhENqlSB5tSRBkpRE$uIJZgIf zJ@hstio@wpH&}OW_CYIv(wGsEc9Rtk>pilpfVW&!B|=J*ns&p&;revD{+y}M+=Nt) z&gUF*(H<7|fabCjD4lv}Q3(~VQ#}dTHOD7p%pI)&QQA&xe~jd`;=P2aSNdlQ6X4{7 z?UNyf`S#4YjVjmaWnL1iK#b?1tKW;@$Uq!&}`8G{F~gGJ6_Uut-Wa zt={rJs`eY!BV8|E0Q@*(`@(Hblh&wlQcll*%hh~WJg5hwfj1!y&&`f9+f>*P7Ro~x z2WBvo#(poMn!nv-pvRhji2-Er9rYL7s4Ph55?n>~MQ;N1{x2py!qq;OO6BVJv*LY~F4WEisoFtN%B!h}g|^e5K@@m#&PGMZdw<7-3{JHc znw-0)4GLp&!9uqwp#lj79Gj%FAh~ts-*M9+FU-JKVMS51fTEZMN`~TjKx+%6uKl6! z?_CX_|2T+Z;rVi_^;!7#QFz5Mb>{y)x(~m7rbbKJpYh6GnA*+C8Q_rO2d)ka2`XXu z!|ctYN>Lh#$FS4t3C&dwLpFPQSxQ0d;(mWl7W3ZY03qNLHT!DA^Be*7XuQ#nc)`NN z)T>6ynht{^w+Q9628#p3T+z;EdxEG%Q#DZ7^C+7sT&bad+Y-#-tbN;Xr9hMdN1Q96 z@ZdkC)_DkS2!X)Mg?bvaE(qCpC=#K#ov_rx0!8fQz+EKr>oL|NjtRCHsysNW%N(*L z=i2AFR>n5x14VlR4s%&6o`Odv>ZxU50Kw3VBZ|L}GG8#-HuSM`Aff_%Kxb^Po=%`b zL=?hfiG_5gdHw|(y1HEA!cz;li}lek4r}SgGP}q=H=y}yXuy z695z~jks-q(tQD~2bKly%3edmpp|BE!Uh=fwua@decWvnQ9**EM)*wMpx^uE4-G+P zDiL+ZHw1=D3pGU25e3Hc-K+-#`r*xGe!p(xuEz7D+j4V~l;f4v!zUzMqhv#2Ug}6S zo-1Me7OyzX$pC}DLB2&RLOwW()CK5t1Ghipwpy99Hg~&8dfwf7uq|R`v9zbQvUB<3 zH9_cy(2UYrHRh!K^r|d%D1wKL;?f@MoSep3T6TvD*D*U8#JBN`d%4vKdU)sIXcQQ% zelFSH?5wB1vw^lKmL&@xet!t{MdEc%f0cQ5U)72s%a$DVvF+mBD>vavshN8(CXYQj zUa(a`Db?rRS?rXM6{v|>rBhrG3k-+_E>5-W;)Nc!bMe%#CpIZHuhn6C8_bFT%o#JD zer;9q!n}@WuYxj3NKbk2gq+*g|8vUv7HV{U9_SW$HhF0s2MMy9OG?lYnYiM=D#Yfn)U|NpH;C5?aRye*1kyb$ugv|4<=MA$p-|4=Tk}t9 znHmjtB-ZW{$d@cWDy1EGF)h$(G6J_oL|6&_PrXXA7}M){l_c}$*;o7Qqw3U9 z$=Yt(kPnuj&~wk=Ikl91sD)R;nj((l=PchEV5aKv=BGDw(dsiS|X$ zYpWJ@&GwEi=1&Q4EU!w_PKg6p81yFZW(8Lg1a6$oLco0{4eenbH{9<_e_(0!Zd}4O zLw#UpU6%thUh4#~mzpSM7RtH7!UDKu7E^K=unPh_1o0Ol?pQ;Ki-<(Ec3(?h?NkyZ zt0&fH8-c8ifQ-J%q22Pd)GrE31ik0jp*#A6$je4h@O4Sy zL#XZX9C)8$**FFLIvM2cX>e0Tc71$!{^=g<-SytIN?Y^+*wuGOulVZ%ZCAM)(;d5L+RiXqwma4? z2x4VRz(*rrNW1zi@JuUYVD$PQHWokCPTM|<$Nl<1^zmh=T$f+MbE`KB%>z zNQnB1W?apytv=AmgJ1e*!&3{y8JiC80}R3yy8~8FWnpAky1^NYY)o104BYwHDs6;( zCggxynq;)IpOA-gWfjUegr3#hzO>^DRDFh{7K!Mo+7p?(t<8=A2N2YP<55!4kH=g$!29^+eb{9NyVd z7kb0|)}p;%_rzx+@v)k;q6Tn)Q5e!h#QNL6)g)=t%y7Tmx`GxR!>#daHSGN3u9r{Z z{}cl3VpC?5&I(v?GGtbFLeEnSJGRwaZOguUrBl&){{t}oILf)VUHQ4ZcB1gbRxz^6 z<|I-&gCYbC#Fg;L(J~d0$bE<5Y?K+SSrsfCcB8+Kn%3E4utKSm_Js7Z}%vdtnV)kR=$Yu4K7@6K5+) zn7PWZ#&<|P=4@Z;(*q0L6K^-3G_X6Z$sgYS1gkGhF!6E=O6mitwJXOtY=$tx3i&%< zL-@Ek9M^8Q{h%-2hF?n(FhWY`SOlUuY;&D)pHzo_H&N;*T2iqdQSUl4>rTrB3!vx3|F4c|-m zNl}4HqH!+YP761T5YAb#-4F@oY29=!(khMX;ty(<}gwDa!2RRgmac=4q5fMk-q@~0SM1h3Ul&u=! z6_6M!z=E7NzeC~|0g>_Vm{JpfQI5K9A zm;1P6gTEiD{g72x$$`gL3UuMom(Rk)i2#m0jkx@hmw_5rPXWDZeetBMrNIq_RAOFU zdsq&z%W@L1E@Qn=(tBYa!_mxRUQ=ZUU=vPz1ebjh&q3!v-z2aePy1)Lh6-iE2>tkHe$F$Y-T{PH53ebkPmU*)0<230;nrUm(HRGc z4N~)Ij5WRCfx7C z_%(|&G`YQLx+Hgqib!F(F4-YY9wn3ut{f@o@#d*?G)sevk6NQ(-VVobb5jX=e{F@- z_!T=oY8TF7$FD!lCM)@w-sNVRJJg?2{G25bS{$H6hzlgAd;DD+>)De=DEc87drvtC zL1vk$RNvECTer&tFZLjte|ll{!s9~uLughY@x2F$k1)ttpIdJSdq%jdpP+wW=H{}< z)0^1^uWce^V!LL(=dY+JSjgKF222SByg(Hj6O0n`w+QQV(37i~$L;EMP6MI#8E{Jn z^bqxe+-Et)OfutpMgu2aoLTWv@qMr927FvE?5-9uaSl5CX@A;4 zfdhEBojOH7UGv{L~!>RXzg0_uE96%@DXi6U0&57Ed{pj2TY zsp!+xL+>nKSXO{q33Q}vUn;aBMYCg1peso{v7Yrf<5p;p#QZe;+BWyvz^x#l>3p!r zrZ(#`AMyDdGOvo-2`0Z=3$F%cKSKZ;K+{%etuBf>iN0?}F#76S00du~0O4qL(Lzly z{a`NiK24G7FJ8QWPeVL@viNKW2N9kHN+}hn(cu9(;@#YM=|s7U`&kcjSu(Ky!OYVP z10N9Od*bU)bD!s^@Vx@d6U}cM=Jxoc?_86q%v6S=-f?z#9$nAoc|Bn^Na7g+lK(bQ z)nY`r{OA?aOdV*jK<+=|)igUMjzJ2$%{hGntS$0NID_{BkP?JAAW9MTIL!ffue{*g zOThbRzQ!l{1k0<$-SfnfOWC^-k=Tj@jm`tgL&P_%qYl;*#|1s-D(x68oC?xT4u7v+ zJg{k^CjVZ%B}8nmVt~7K>oDp-&MdSGvpnRz5hnDSJX6@;_xnf(QFO!kd$t=SEo^-t zPup8ei5LfQ{4}C>-8{{J-i>w>!2&1#@0@G5Zb1*~P*S_>^*IwJ)NSSz@O~m@Hi*qR z;FgA3WTstz<{8t;aM(#`kwt(kiBON`y6JGBxI_xS9a8QRWRy)PA%d2&?yf-(`g3e~ zOa<%nR`F1B6`RONR&<2O#w?!E9gt)XlOqJ`(j4N(1li#7oN_=UMmZ$}1Dan>-l+&l zPxpBMrBJ3blU`^l3shq&(k`P5P;GoFj0?kR|9JN^!tEdAb^9s`O zETH=Y@W9^y!O^7%qM(4|_t;*oTSVHWuSXK$liqFrh@$Gy2J#=RWWu=(*Sh2utFm+v zsJuEr8hXq2(jXbI`vZN8X9JLtuPK1lMu|!UBcL|6I8uu%6}umzGN;sLk?ZdZajo>@ z&0S}iWy4&fWU-5h>zx~f$3-NB*C5K_ON_Q49;pC zZ#FY+K=x~nN@KB)U783!sNqOpkfpSh)g!blSCZDF53gPW!0=ar<)69A+|#Ak&BHIb z*S{3bBE$oGwjC3?nR%yeRx|b{1k0dAHkw&)SK)S%;+D45KV~i33A+?a434=t{N2?^ zR$(7YX6C&?zlx$F@NfO3rBQu;1C%;khO09?OixSan!1WQIQ$J6F9(=&NPi}xUT!}T+B75j+NK!t)-Jp*OFbB*_D8OxL5fWbY)X0N?J(@Yom z?NXQz1~@qa=iAj~uYmn9@`Dusq_nFS4QN4RbAOxI4H}CCw|82TGXf`8pc7Wr`2oLT z>F@n0@S_QQEJ1T*lM;Cqk?tap;QQntT;Q5bE`c>;G91EG8}uC8i5hpPH>q>V+EtIo zJOS z(t#tf1$LjLZnVyNF1$nX{`m7)mG#0<_!}rU;6jj#yO#)V3e~DUxo^8-)=IW z`7fwKQdS0GC=Rv#>!YPT5{wXT4jD4prk8axGk2TG5^96V96Iwp)@SX7TnHy7(! z%b#9BQ+!XfQ>L2?Z*&LhUsc=fhJ(q&?nOlU!gx0rR3g`()J_l{oDkGS^1`3;GAsPB zy3tSfm=8XoO-S{roCGr9RlhqXv2Z{u54S7mwx8N5hDYFO zM?X;_^BQPMaL}@c6$vC<1zRI?Rzfuo47w!ieJ@S`)!hfU#6 zFNZ!ly(?Ra1MEdF@5c9}i7o;kxRqL*vrt^o==O`LQfl4=JJ##nJ)u|q2SA0Tc5pW# z@hbL*zRpA?80k~ReP96|D1-ltSZZS@fbs=rZ5Nk8p`Oq*et=1}sjR%mRh)rSt6JzD zTi8%Q$_hS*SaQMdZ^=BKTd&0akS(Bp=Ea5Yh0WXGPT#NI>2+xB$3SyC&g|^#eiW-% zC|pZ$g7-vU2ob)F+09p{0)@73vOoPeSTmS_quylIjQmWpxlEqv>MeG5v=@NTlCAf%y zH;YV6D^tJ!c1cUn7Ea*N7j0Mj!Ui*K)J-7@(CBcKzV%Y6^;sD{Xhk%6+LCNB7f+?V zyx8IWK^R6l7*N{H1F)dwao3D!Q4az~imtXg5I73m{Oz&8gq6-|%(rg5Y<~dK0ZY)} zcjroi4Ljc87{EiDr4WoO#=E&|$b}*;=ELUCPt#RXpcSW($fE)*#LPF> zMG6RA?n`$QI|xEPm0!_;zZ?wi3Z7FXJ0>}V?1N;F%xTo)&waq(=Z8_ZpAPl8q z$ZFKa#ji*+8ILcMc{kaT@y=a0e5u-X8uqmwXw*Y)gksIaWu*=GUf%%C%g{ee4Ym6e zq>Vs7|M)nvJ;wf*Rgx(GB$g$%zsk9m_jXMwZ2uF_1AXoCfvrvz7{S`II8cBEwCTqS zL9vrI&sxdE3wp|ORE!a1!ltw3`7EgoUcnRprO4Gr`?g9#jhEMp=c*kVvvCFVRO^d^cNr? zM2qI|3FhAG8a`kMeZXkiFy}s8~Rq~ z8)>Y_0MVy1wpz^UD5;;EUlaM(}M;S%CVab<$S=@5a*nWKJRATYSYtlRiwDnyhs{R!UlQML;Q1& ze9M1q_=$DkoJ({graR@^dAUyk+5w?3yKs;N6Nyr=)TL_2H*lWt{y1(qURUs22Nf5z z73jByPhd(AD^P+0TCfmWbbHk^+dbDF7!!LHs|3dFwKMkbN*&CRuB8dO7T_4bGm2Q{ z!Wtu#1r&b;R|BwsBks?vY19{hazJ&F15ynPBR3(pul;58cFHN?6ea=qey6{ql>U5K z(elZc>%HAEJU|SmZ3&iqCIbJz5^C9w)(du4aqkMClYTdx;{tr8KmZyl&a&FZ(|i-O zzVn3X5l7xvmEzQQAvBBx2)wH;<2j+OYDf#M3q42W+$3cmBO}+DKh%Mw3&nV7B3D~E zII--1%+-)qapTtBh=H83qLN< zA~xWJGoWc{14igy*cbx>|t9Lx=funRq~lgKyHi=n?-vgG-bnub8sO7udCKf$~PU= znS;i3xnOS!5S$$Jmd9f=qGa@6=15u?n&V!;5(&>in;bn?uhTC ze|29fn2RHq+-kK!lKnsx*Xn^oajg?6b0%)aVA+c2y&p{xa_MQXGrvtY-ge{$*=&@Q zI2QmPqH$b+d8O15s!jh$F-V@$ebG+|N8#ro#Dycv&OK? z-_JGOKomm_&dI?jfSLZ(o#?j$vRJsfo%^QDOl|JYHRIqBXg-H2dWe@r`qr&rto7VF zk>CI(RM#Wgsw;Z1%71uYEVGJXo@6K4F>_nP9zAK0WEG@tdLJ<0GHz)cZ1D@Wq=qw8 zi8+ieiro5Gm5_zLAqI>DHp};Sck5=$JrFfCRbmJNW2}$^lM)qZ^5na$kWYQ0VOxgSnz)99m*wxVh1qBumz>aG~(e)Hu5U=ZqY+Yw{7Uwm`P7;~R2ok{Og9?}Mi zu=!i9kbx$&^V+^HU?f_tu7L?~pVP?433ffZ#wzXU@?>9%_WVlKfkxKp!v2c&S>Xd1 z|E0w8kVeqW+zrV=2pdBe;usnGQHU8h4X-Y_VVuuz(cz3a6M!|9*+M;KISVggh6uT= z(pikH#1>}Ov{qT*fAd`Z4X+A-S4vaW{zT9igZ{I7HkiBvkA?ww@ydI{5R^SgfoHznk zypZa$_~vdsY&=0gY&0i*=P$wX+LoAci0QDFRUBWV?wdtZvHeXg*SEK(yNY1kFW5#) zH=$8o7<%1Ly`kdV>Jwa97lZ75&#kjLAR1nDj&oln8ZcmW-O$a;}<^r&LeBPS4BF>i1g@#JnCRkOijy z0Cs9}*7tkMBAR`-fWMVM*)8ZOVW^kkm6uUr+DlQ9+ zTYxM$cyg$=J0Q+as1n!w>imVa_majQuO}nEw6#)0@?WA5iJ9!?f!UCkXz=_}ycwyt zXeXk2nxf~&Fcfi+6k0qGl)b+a52CFmo?s&nCj6)tXZ10ZdZX(GDtWM;C(xd&^cnHI zwrbs{kcM;s=mRp&#<@0=Zc3f~X)O19#(-lx@r1G>yQfl-j&6J#XcD87|SyWMp8*D=QxEJre5fMbVazefkiIEf$ zq=mDvTI1NzKny&-ACMTG|M*!+!C4Xc5i>@cE0*w$KMbMWe@Kark{0;)rh&V~L2*-A zfY=$kz=zr&2GRpEKi6L(O^RT4E*rI_OIWUWtZPNNM-=j&3gL7v1=MouEE;^i%K|?N z?8VfgzJx|Isgb;H$bzYxPrGulTt7)BR^YX>D1?NxyJxOr<73RT0faU+W&j*gR4xWJ zGG?UYW!P!fRk#B#4;&npR1rNEtGu2ODGP#nw$cW+PAa(fT5#KYqNARr9;Bk0fj+(I z-kS_>_i7U7YJ5~@?>{8~JQ^l1rV@qG=*^??(mxdgto{PnLZET_L&X}sPcDwgB4G+h z6dxaIHxFO4C5+(vm52J_7{n&jzyq<$Q{OHwRK7&so`0t?y!KS!C5E)^6Q*A3pIxHC&d=sG zG-L~8R_FF3fZr}~&?(*IX7K-w#1>fq;Riwi4cOMncj|vE+pkALb}m!4s$JU5JAA|w z{ZI`#U|L}D<0MH>{@YvxYUwR;m>_w04}gt-2QLy5r@ZH!i!ShjDFwWCv%s1i$9Yct zH+kW;rz#CE#SeXPxWMgYSiTOzSe?&ta7MM*8XQJC;IB$Kzzdfch=Id?t8?dJTly*Q zTk1q(rJYG=+R4(1FrNZFO!4f7ST>EHZEj*mErqYpDfx3MI5_-<@Xh#X|w_aK5@)mxfJoqS8?L=MJS zFJ3_th)vbqoFeU>&d(ZGhB<_+F1(_|)Z< zwh$PlLBI*-SQ2l_g4B#pDt$0P9t|i9O5X6mAdk0p&&)6H)}Txz+Hgh5MIaD#=ES`O z(ms%ly1@Ir0!a;=abkj=L!fnqUj9|7zO)x&K-6VFmA>R|QU1tIrS-X87ktZ}nU_US zuTpE?^Sxxz=_Q^G2mW)U>$xNW+PD#Ld~=8uK)`}*-deFXJSK~^o=l3(i~+KTvr62e zHTaaL-^pBs=`kz?k8Fi)@&Qiv47a1%$H~)UJbDvRRMd7dl~wmU6TV8l3rZBO4mnrX zT<(}01)UmlsDA%4bHLc+N-)B<@L@ib%dnF-<>oUrv^vvd|CqHcJPY%O!)L}S>ejrh z`4oce`hajL$`GE8hU$l(0Udf#|1ZDCWhRcGXVYr}=b0L~u%vLtHhZ9owH1>TtpNBW z7o_0DI}3zgNUzRXNV+54M$cBwlS*5CRjKP%+V>f}?aGB3gJ~+qwK1=!*oFEjFj=N$xG5f!OqthyoP)W_J zL{o0>U8yC*<=tqTkRyT-)=SH!KLdwJfWmfwwU7(QN}Ds@4kl^mM37_gO9uSWgEM0f zL9%$9Si1vRyO70e9JwA~zzC+u0RI4hDLr4zq|r;KI%(?655GDC!&^Y1yki01BKuSG z|4m4LacQK6xTPTKSHzgeG^FGM73McEt%@hM!os$z$O-6mH#Gqpv@pk!g*q2MOZDiW zaW(%VUDH3@m#!yHrlT4LL`4{yAY?kc2Y{uVnUsi+HGftXZ`!{95KmiLGcPx^U|g_| z3+SHC%YQ0n2AqRvC^$7)T+Cj?u>*6nLn9B!$ITI8)b%~xnfhs`gwSS?fHQDa^D z^)dv-=(mg1iF#e8A)@!a;XhYHmY(A@rLTeZ6?O<2k1yeU&Ta>g<7t$M`l*X(%Ho|J z@&x@mW%1>&NPWwdF_`qRr!Bt(^Or!5pO3I18_X2*pgh}#&>?*C&@>17(Omyuz@7(# zCGa6U(FU3Pk(T9)&7Ff3{aLk~1YDgXsqT|(=$!h-qeD^4fi}*G{-^SF&##ndU@H__^)>d6Cls^U~Q^iHR>G+f_!Q9GNaG0v8 zD9(wb`b^NEHv;bLD8T(%gm8p8`~%SRjabzTn!K3^(N>GNzHEdhr9WK1>u6rZst#>r zMd-N@I5#wx%xFkn(J5`mct*XF)2pLOAf4E;vv8L>ZH5`Wf)sO4ovDUa`PCUc+Hxk| z3Z?S(Xa1rCI)-l3zof8}Lk7CDLf>r-Ex1ESUS8H_v@``_%-V;u{L#WIZYpsTHJ}&H z54U~=gNn`42NU?Ubwq%ufSQ7w?l#UUX^ZyApaL1ZZ+fgKROq%XNb$tZ-EG^M(R|kR z7)~gp3S+2Y{;wuOXa`6Y!~cbwvlDpYnpR(!dyrUlyi>_}0P$s?t|!>aAAyKLJOq-_ z4^>#p4qlV@7J^~$mCP3+!2j4N8(`CVOy${iPB0#(68Am}2I)KBiNoUhwL`aA0QR|B zG(C;k#ugR%21p*#sM#~p57&pa1qk~HnJweiSRfeAanu1KWj8CN3M?)6z=#u51Jp}L z)JEBMhqEH**yn>9D-G#WOD?y{{{(2V{>;6%@YL83 zo{{moDIe?f!*@f6x^N%RVgmGdk^Sz_oX)RhXHM7)WYgL;UIuRwPo(O$bw0(aw5|U1 zkL|RuF;$`%%s^X1EpeL%FHr0A@81LUVEw%ob@SsREbyk;S=j^nO^^m&xJm|=Jm*t` zodtviGPJfADbH=y*Y!?b<0l*v1~C8lSPIOIgVF^&vo$px77XWWuC##~)3};I;sC*q zfSOo^?>G>|^Ka^~7Tl!de4=2PE?)AxraQ=qK-|YhW5tiIuc=BD)vK(A{LB#B?1aWM zcag)=nH7OxK#8LBa{O9I_`)n44Mt(6i&T2`rpJOn8+y5zABwxW{-p7#&Zfi9R;}bc z(YIl}W4swVGHSb1B*0EyAdRJq6R$wpsD(1})JKXB%{_54+yy-79Y{B%7=&Ucv%P9a z>wYSfzyK|FMGHn9-t{zL)95mKY`*dPEFKRf6Y%R4@$j)p1j1 zUQGhuJ%ju)Y?vH-cwgGB)|sA)gzDa>mp?5Ow|Mmf4CG4sH% z@>OLt_U6@lMMbf$AM|4w$zf5U69qWVaQJh>CDnKN8|F*Dg=s-qxM0)VtE^v8o-I78 zZCR!6y4V-1v_4KVn?O2FkUOchY_tt8yaO6WYuFZ=g~?oG+ik{^v#mcX)nWee4|fw& zGx}dZwD#SR>G5@+K_0~R1%OyG-JKZK_>R*VIi8vO%aM`+LveE&RCfE)sb?Sw98!-y z5Ly_?0;3rm@MQ&93$uFUGVFT&U`J+A4wcO>AzgJxW303(R@sKCVJMb&z07qb1;!uh zp+eX)wxE>;ZSbz@1^3EWcFxcaA%u3vemHEkYs+;oi385*ChS{6GeXtaIXm9^>zV8Pa#)%!M=>HIIafo-BdB{xzrYqxgZcbVL1#pGU)d5dTHs+3>0GgHOaxaQJpUl zp|r+{k%ppY0%kXXb1wqZAIPm}1cTZ6<^X)y7G@A&f(uIv@f@%u{n0MQO+Q7a)SW5k zuQJ=mE=)jOa0CfA&%~|l2I4%|w-vsybZ1&0_`k_nKOlm!K^^$Sf6Md?n)kv z<1xg>Oqk|9oRs?q^>*|Z=IC52uPj7@+W8n%%M7MFpe%^tTp1rLqKHCREtCHg)=~HA zKw}V)tFPr;ujtHIzRGZ-!q@^hq_8aeZ{ZEJ-F#;nw(ZjN1l=9)<-#fp80Q90?tD8i zYB<^d$JUpCL%qK5JKq+SLLHSPTav_3IiZNsqOye|yKIB8oeHO9sR-FBTL_gc zdy##REm>x!!q_rXwrPyz|Gb}}17`d*#-%scPrS?=e)?`QD4bD8Awhc*Lg z(SsR;56$WQtpQdVU`O^99IJYK9X=5g);-7*>IqxSKZhpN#Vt$(m06m zJ8rmWP%OyPD3|RtVP-&A&pvAH&HfrXKkk8Bg}8qGBtU=|Wv2@#sr<_M+`&B%V9;AP z1nn}x7v^djEH}&MDS%@m14Fn|*CXolT)7y(oXxJ^+Yu2dUDK!Wq!0rMYuvq z>jC)mH%jsC7pO8WcEMPootR#dgPYpKP3hKq=JEjyR1ID!2|(GP3Xp)+Q?TyiR( z+DySi!+}W*SzRk;a$> z)fMW^JSm&=1q?e)*GsB*5*R#SsDa2oU_5NyyAjG?ybIKlKTg|Wv!ilvD}?r83@*6B z0BuIp!QjbozCDfdU|DtZ=6flS+roFcRJ8krxVWYw)=5(-T&T$P899}_OYnZ^_ z>M5V6$wGj!1H3s906_NGun>bK*V@4Vjx}#6>+L})!tfnM1(a!d#tb(C2z3qB=nxxR zT(E9KqoeAJ#>>pESKE4jjQ?(0AJ7Ij7&9agcmA&-fy{UR$TO;K;1X1W8Ny(u8tm8x zc&{;Q1S>wLSK@GTP8zxI*SAbj}Yn5cU|1g4UcrNTkD|s*--odC(q>3pUkvvQr2D zx~fSHm>R+4M*SzmBhh?EJyZaSb9vetz$9bVz+fYjtSpBdn1TU{wD|u!FzrftALm&D z%7|elwT}vN?*OsJZY(HrKJHO9hGj`wsHfmif0lWwtO&BO`G*fLCjTm^M#G5xjxNtK zgOE|@7*e1}MU}J1^w}q{E+`AgK_ALUF`(W_1AOl)d%AV2xS;pF7rDa*>QDYH;{edfSsQS#H!V-{rOGaR zy7qdJno7ykEzq5*=EQ)$5UoP9hUTk5!xsz9sEzQ*Am{>ygNb1J+XV39_0c#8L$sFQ zZGfleRhg+LQZhg1kGrB$2jjFS6MdWAEFS9kD9_#g*Mmy}S`xS#IN^L1Wy9Mcp`$Vv z((#e>*?z?D%TvHg$IUAwyXxuSOyg?|tb77%l=lzlD}jcHq_Oz(I#I9Y#MsYiQ^*~? za9`r8ee>^azWNw-XDeT+i^pr|9^np=Him5D=H3>Z4fq2{ABtTad1xhyy{)*NM4#yr z>=FR?)>b?&X14#fYW(tW@B_>UW?GT;OR+8wrtMq>5`rCmxj9!MJ#0)0PJ4*KQJ3!~ zap6SlB+pI*r}NDDGCpn~wk;j+d!TO2(9%dBd@b`?k=P>-J^06ODf)Z+MboZ~E=S6A zGADW`QF9YwkESf>7m_Q89JcN)1$q6>Y(pW)f7OVES`=%^8!)fkLNI6;ZSq~b6!4|?FyFG2nQ9*ot2?1ar_ zU+`!Mucz-sr9)V4w$gwY!2RL2vZcj~{yJzdD5~VZt+8*}Re<`=NEsjpr9BAk^V!+2 zq$>cu8J*DyeBpEXa(xTa|3FdIk&w(I*Y6dO6Bzpg+MW*+a*!SjFp9w& z(^i$?LLNXQbvw}Fvt;7G%f$7uG(AbGM9MtTt_0U0^a-+-!D4bH;A_A;~RqPxaa$s2e_R^YR zi6A!^*5w@o#I~!5AsFYIa*nQqwza34#1>s^Ag$s@iK*Cp)Gax=^PqOa5 z=#Q|e09kh;$h@M~wASXk^?Z3T*>T(y7+@fi`p2;V6cWqjFOF9x5o8J|g9rm*y%m6H zO(B(NvF4|tQv`D-fd9|2v)Qd&zC994f<&)5k~wl)g0&ny!8Vk22?>~I%_C`(2dKMz zb42@6=X_$Jo~K?`6uA{hhroFRhHTpB&v6BRf>VL*HICtH;QtLmDYobZhQY$aJwiA@ z{36Q)4jqWy2a=tt$pugw;emRu@~nW8pR{C{ zOI!nXx>6D3d-$71Kt%@v9hlXn16YVFA)G@L>nK+5Dm8MNFDQJxRhybrqv!pXhqLPQ zFj{ejmOX9^c|##jAveUJbBn_`JgwF3!x*^m<}?V52A9TzNZNo9vnqZrp!-3rMs5Rh z&~DlvT$n*LnKFY&EE9tUP>%|)+=L_401R-@ZWq(x0T}1pYs-eGpiw%dRK!kh?(eSU z!Xg;;qUtqzbF`ka9eaDCxaAU5EU&FkbAXr631(TXvqd@OO3UhBzm~PddXyH38S1{L zo()#QsPzf@1XX{)X^@Fvq3N1TW9N{Nrwf4Vy+km_bTv|Z*gAixNrU5G8 z1`{su6gtqK2~?YMXj4ETpuKDPGgE=Hcphm3rfdMdgNkn-M2w9|lH~_6!;s@>+{M1r z;dy*}62Oh@oZ69SahxgnkwPDc(iRWhVg*bM%iI}&eX#`@z6PkF<2YICmOrnT9~GYf z22zSUUbGd!@bX}9z6WN5gT)GP7G=kZ$3XKA5)yLHSdAa0+OWt2Y@ulQ{Qu9J$?r1= zx$<}r9E`*$NYBQn1xtc-VMqX;?ikbSE^orv$I0P`UIW>HLq&u088QMh@!U zjL0$Y3;??k!1w8?np>Vehv2SWlsHTS{R=vTgSud86xHH=XgHJq#R+m;XJHmBpwt*t<8 zWL*BotpW#F(GFen1)%o#%(TF`K-#k)X-fN_yrS#Wt_w#YMlv=Hl*{bV0X=6fG9(5Bo2 zVgRgTJu{@NxYu)d+)=+7jI+TmHmC+{0+u~0tnvWY4^B5J9*H8Q3Rm#xLbPVPLbF}} zwKwqVI}OOMF2srPC*Po+m_u0_{KvWRtdzhP!#$WAM<7NQ_A~zz@be2W@dOGT=iO~p zN5M_Qku%JmHB&qnQ$THWY|V{-yIIdhgUN-(=dHq&`(wn|NXe`5Y#rBg$ZmC?%j|%3 z3RDxmpUL^>U3k-?3Da-?&nd8Uv@KzF7St{#vPSh0-Ay{=e^gutjUiSbNnAPsLBHUG zvjd8F*c%ww{UH=HIq(%M(o?;My?a~LfM(ePDG=Qw1s}LK(#5acTLf!YK@gI2ij#$gEep}3>B4BgaA7S0|R*ffN1z} zI#wQ$Lt8V_wAgpU{M4dCgIqFZ+5uS*Ue6Jbn5G+yzcKjywW|jcZ^aL@Px_ZGv2GPm z5joWH$@#**qO{d`L`-wkxM~joh<>ByZhH>r*TksE{Cc=M>QVyY3v+C( z1jJ7q`cVLaxwJ&wVKbw(n}#SHw)6@|`o!0RsJX#$;*Lef{jUz!P(>HP}fw1Qr@rv|ex& zTqY3S+5qEbXVm7Ex*f-6Fq-e-kynMK_e_ssIPt=;xBL{F;`2r}i>=C->CH2!e46I~ z{-B~HV1JXQKh7KFUPi-3d6Pm3G_&;+wf&E!6Pk8`E#g|#_XBO@BLvFD>Ik$i=>Iq$ z-1rBevBES;{XFS(G%xyIP7Nu0uswW+dg7Q()4a=LU5GriQWTJ!*)7b0AEA7XCiEKe ztHY+fOtJ8h)&4uX_E_2Sk4nR^oZ1Lg&uj21I>(r5vD*(ts8sQFWIqS)+{)OqCthit znWn^+sFd!2Ir(td8yyN?iKEqP&}~8c4o=M2kyo32f7YT29v(v1GO8`O5eZfWqZT`? zl~7;I&G7&0*;hgDdJ;O7aoC~!p#tMjpreP;J^2wYh!4_Mg~e-Q zz`(93K-G8+xFX6kt8Q43zN;fpAh-ZPZ)gQ@h%c&dY#ld0&a(?ODvp~@(zFvENnRC( z8(xY*M6ts5p3HQx7WX&xf~pk5k%1Nu=PdxnQC0ub3^pjXyAC7se#fykOJLC6|6NXc zKxK6CSXgMm-2S$XALy-Rq7&m?;8^`X2Xx)ecVZm4*v@7Mnm-dmxhBQ%rIz+n} zcdfEn4Pf%;8kmFzKwkJRaQIT3Et>=cb~R1dU=PbN8ePcd=~Mqyw8X1qZ9GJj4|0P6 zzZ(JPBV^$7DO90i^6`J}3OqY!w zaY1Y@dY4y7H+R7)E8I8?TnB8_E{hq0MS{W#B(+~tdU<;5vM1z%t2 z0K70NnB&%i+k2+eAbm@~0e3V^{|VidkOVxgMK8txdNQscMwMX8W-iCpWjbgL-igkG za(V$80OZq+8NED`oH!P?AwK{>xWr)4E>Wg;B9oPL@Cufe2f$dxTL*N0pj}tP1k6APFhc&`=`Sy;K{q6r zfeQWZeX1Bv2oNJ0fYLjA7=5A-l>Y64u+<*ogb5HCrx4@E`;T`w(P$sRS#bS9eS)C_ zC)hRfR_nS*dKUQS7{8vtAVMw?dHc6GR)rO2BQB!0CtMm+UW2l)wQGKYX+|vqBG?BM zBLERT0xgsJv+7%4j)1S%91KsU=ij%yUAzHfFk})a@SGsZEHHxt*=8BKV%fUV(Ya-F z3d@K38v+3q$bsm93~S01(a;8n251-x`|UCXkM8z;!FSk`O(#AW0`0yQbXGt~*$6EZ z4V90KkALwaLT?a^Mq?FikPDo=2@N}GQ25bpRk!PFQ@RR%8XtHKXMApo_dswIZ81RH zvFrHwd^;E+2;ByzD}#Q4577fhSv%EN`zBFpjl&0G-IY}mM=!aHDN z!$<)Q^E5l`7D(9DC^|FW!-s}>J^&o%K?YRJDe#9~sW{l)FrvQoi09WgV8yl}ZvYX2 zz*XQMWyc%YKfn;TKWJEb;7)WZG*yt=B*2=~=|b^X%CE?52L)uqduq}a!C*ZVk z|7biowk85LL}~oJO#*-}6R`6Kr%iuBVfeT2goINhI zkTzPVvXU2J<#=0JIaj^^m_%*Z<&r7|ZS}c=!RLRyIkf695D&9c!X(MD@f8kWjSojX8g4W7efFWbo}anle+;Amj|K0S@Nb`*ViK2S=@FFa zS`hIAx?%d00FrUE5EMrePu_cshZWR6+~=DhnL@AS>rshY7Nk!Bj5ITISbIoG&E5IZ z-x|=PA@o)gxLlw{Kz8F2d7>E9{0T(tF{-V4yUW2IeZav3WJBqcmtfB?>DxeNGi!8q z>Ca!l&YAa5P=C2+wU_MxdDX>5gP`A#QB*MWLTJ_MVlgx}1hoc8GJwZwoe_{`-?d+! z=s{}$EUCy=yj0r2w%8uEmCKz_Yf^#bAA|*uc`w+5D!`uk0#?`aeRwgiFdZ~j!Ua)J z@@}Py*wsZU0OZ`cw}%%DMeut;AQcqR@^4v+s!?Xy8Ne3RL|5H5XX2SX?NOpm;9IM)Eojv|%85l;1045^ZQny<;ezlLnCUXtZ->qY?Aln~~@rq0d-IeQZce?wXq>I@){W)Dwtym7|nXN-A^&_zdLu~+UHj+OCfJwO1dQbJOlaCAn=1=F+8>!231CC9K%x* zoGUaTy9{6*35WSpd6Vcfi(aU+Z}An^P;>>Hz#gi^?uRPh+!jveu`Y4N3}VDx$Jp9z z7Kz@JrEBs?GT8)NP|qeRA-87lp2%BBjPsv&4(3sf4ppCMz5B9N_2hx}4=6b>Oc@AB zr8Mg}p0Wltu@7|6Bp`9Z%rGW)N7t49UOL2iBGhe_$VR zWeTiwXoo3?N&1t=E*d+DWteK>QpDGcAdDk4J|L!gotZ_)NUO4ubJH%cfc z!INDrb8u}jbm8n{7vOSg7Wv8 zoMaa%1EMWV_|O>o#nWO@Whf#M=Rpn9CMtxnbq3g`E~Jm-0>|9^dYG>VnXFy(Mu=h- z!w!gMyV}{vWh1Fi--bw0^@w&3n}0#myMd3?G7NmP61-~)$cRYw zckn5&;6p4ouv8cH2P2wfzQaY*Sc!Jo@S6Bpd+>1q21ugnflctcauR#F@wp9;pse+jB9AW}m*Y-s}Ydzk<=vSbX@4B&nysAQjPHCJgCiGOi=gQT|u!+b9HUTd^7 zMoKy*JmJFp`<@nP$J}yZ&_r^f^ejTL-Z9wT501+FB*z&ZpjSu$>6HpYh3A${+=%cB zRDGmtQaXe%A7D*aslk;4eXhlKz$jpU-3*B=r?l5=_!V0M2&T^qu@V-vF@xn^^)maO z#Lf_r14*iyJHeD<27M7+1Z$^aRRzqLnhO((a#Y7XJerK2SkcA+Y5y=Ho-x!pq}fkO zpFyVr!w`U2TCtS2`QCrsor>p~K^2)X*+OFM$_9(Fa7GuA#RjM)1qz(n+GQA+IP{gn zYQ;TE+Mpqz*AC5geYSegk=BeVNhg%a%rK(!B84UD&_#KaNkZ zt_0+hHb~K0x!6+3vzHA%hCOx=D)_;8-i_6OunnM1aqOqGStoMMOI*KTHI zUh04e(mPI_Vui=cAciSKylQ9o%-OxKZDek>U)s%&F`^$^MXI`vFtnxey>}X%DDU0; zF-;x+(2f{@2(Rv7(=LfM&}fk#ymfrg^usS9Yy2dUJIJ_9S=;+dF6cvn7*$?|YiCTH zxu5F)tV~suQv0x*c4B@f`>_EoRURUR&&;72lC{~+Qej_1D5 zrZ*q68EZan>dkW4<%^sru%DGeJ1b4Y#fuc3%5vQ!O)ZjsM%2Qs~TJ1lr0y*#*=UhjXgIks5!R<17&@}J)4 zd%Qk7JT8Tpd;1F0GUvK#!GZEY`TE7rz@dEwQehVsV~SAq@hucJj4n8pMv~D1@td45sMZGsfoO)_DpxI zalIRjDE2+OvK89BMf5bmITW1}^-e42^Ye2p$Fp2q&-Ud#PIoE(V5N}tg zf}xDv4^jb-nFl8Bpw^h>4&8?97N{E>Vllp}^32KIvpPAChGW=|%tn|U``Ca&at^Pi z>;n7mfFowK1B%f2oJCT*5gSo*Bw8orj&TA7G!lm15|Llyk;nKV2?8TvSECnwbMdy%Dpva!oSHyhBX zFZc(s$f=7nw^~@phLNF_C2RgF^Q9OYvg8o@Keh0Y(P{jz&*4uyi-)o2tjJOK(V1U) z5YH|Q)_h9&<)W!pc`6{0d)P_pC?u&5(2tuGUhjO3SVce%umz=y#kGWPxy@>aWDA&I zu}FH*cFAo@-hepUIR9E_7to~EDU|$(KpXAQ>@1&M2tw70N?GV44Z!f~N51lN()BYJ z$Zj)aLeUfcUAc;dC~I|B1W{&WYG!NQ5kJB??i7q(nTU>+UUTKZP} zZi7L&d?_mV$P;SHw(Ue-e$*Y1f4a0WMK+S%oo#H#Wl&ZwC9j>4PVF!?6r(*4g}M|3MhdX*g-2CM04-~a2l%m0zx~O<@w8H_n_Ucb^0k-%L(;zN1xcjyZ$LVCajc8wa<4fhgw+Yp8!@R)3^{T~%Z(ouDifV-lolsuh2!e~>5A~=AxPqzK#_PN`DqU;5mQ0C=pJG)R*kn4gwcWo;Dw+ zRe6UNA~OgUznF5xK}YJUg>g>SSY=(C)B5^{t*mr<;iu&6^)P9UVO03%70f?4tLd#Z z-taRgnuAts53*~6uLE?ZX+9V~^-oCaW91m!3BHK>>^cO5LLuWw*fN+!60m=5X@VKJ zE`CGM%+sL5=7tT515C6U1%Pv~WZHjCJlv8oCp>40<>7~j^tz~wwYQkzAL0s+&Q z?I@%V?=pVLce&&mwsZO-z=If^K-ul^nL6fSSM$c zW{Hvb6>fRUsVV!kzeYoWV}Ahs%b+vx)cj@eb&+MmsIQ&=5zn}c*=1`Wz9Uc{x+EJi>nMOee_B(QZNu#FvSeFrT{29~ z2NKrq-Xaf>bw0~i7FNCkJ3Xv>C1X+o3Y)z>mRU<=iKnQ7+i9P67Ia4=29iEg>DN{c zXW31tm9?vUOMxBT4}5>--Gd)Mp=(6KZfMtbVdX>bF+lz|t9DVrMG){aXuEl#<6MU@A?%v`LIGK}LQRk_BJ#=2{ZKUzbO(F%-9*P#yia29wwy34e&=CRgQ;)1^!^mOcm^CGxgu*@3NBYw&~0 zQos5iu$L?m)~8Vz)t?4E7>tUG@i6u(s%6L@{Rl z*+YQ28YqpcVP9%!!?uZ*{b}v|F){wDn)wgyEN=ELlt&2-qdEH#P- zNUXbmy?Gyn5fDJIq~zm^Q9fwxK-WgU_mn)IRsb7n;{Y_dszAF_gYpCvhp-{$!qCvE z7!Q^pXn-uY%|6JUEyq+q<7>xmiTC(ErIQQN@dd7D!9$9&zCqXvwrE+h4sY6Ha`DkD zGOMb7;w3xR1<54EVc|kh?%B?lpTl2#OU@4Hk#IhW)hYk?K0^ez>-t0^mF?c!b5Ie) z7hE^#XASbh)0UZv4an5<^`xBlrEtZG1rqRkLjpnr?5{`7w@0s_p9AyC+0doL!cm<= zmP|`nPf9CC=5%udU{3cdDqs17z~M9KsQcJ`FtV7%a2XDPT&e`)!%yN&DiM z1MvCpnmZTk{FpYBbv{Cg&j=kotAKDv#57{qLKTg8#7hks>@RDQ3`J%1>fm|bX_~Tf z2 zdi?BW(uR?3;sVNk1KaN;>HKd8hwwJ>`1?_}WP5V0VrE;5PqrnI=nj^OH;wkH#*(7v znwp5?bDr@G(MKy`GhFeH70oNWwx*205W6`&BjCi3V zuT*aJC#?=XdSFJB%+Y#;vDg7P3LBLshI&*gni%f47dBxX8Q}~o5qAHKn}{Y%pIpD) zm0#Vnw}*C+QBk4q#SXj)YK>(H8|hB zk>XS8VdkE4(fiwVUizB*0^)ayc6xq$rlM%EX%|dAC}#+?eL=?MR;{D`;UdKOK&Mb$ zX`MMCMza4Z=3nCFgK^)_h&sRfbT2)-wYM|N)g@PmVgR4FY-h zH!E1DB$~u+KD?(~$D5YPK7q$;`57Tzm4dB>Bk|tUzxZ`id^oJqdhYA1$aG2zBtS_j zbys^7pv_(JOJ6!Boh{<0&rOIL_NEAICZK!3=#0ngP=ozt6~V@fyjXBX%DR4dO; z3t0@5u?C)<4x9XR??mvXV_m=D+>`F#R5xwCsWGJp1rx3qf6rrRtVbgI}g z;la3i>@f3fuZiN|kM}($*;@{cXB@gQ+3KDUUEVj7lU;Rr+MBWcXE1@k+RtV8SQF7% z-3>mucd}8ms^daYUpSmZ^M-Za#j-UMp0jYluC<>q8>-E25y@V^Eh2j^iU2e}^t0x1 zhRpaehf{;Ex(JOo26bv%61&^OPJcwdN}!Xq_x$t;y9Gl6jQleh8Tc#r@Tw->L*M#; zNqV#~iP;<<`{2$MMmT^szoA1 zHG zcG-G{3+WwBxacg8aGbQF1)RiJP1kRs8QFD?7Iz4XzItn})!)6RRYcI3|FDG^z0bV8 zH)rs*a-cz-cF$a658MYm(PO9c z|Kt%)>wkPY3SAU#ku+mr-*?I?z3@iPR~^rOaE3Xh6fZ&~#Zh8eTObykQc3(}u!L$? zo@LQrn_UMF7F2H{7?C4FoU5@?O7V$geM5iu($C>n5pI?~c(ZM_f}}0=Tcx-yRR(-C z)1wRaiHK7g$}@U<$O&y~8jU+RgD)LlFXACooax3fe4TNg{Rcn@@cF@Oo6zENIuq;%Y8ArJ4G!6jh%qW^2-l zWyX#1K0hK!2D-Uf@huXiqh(a`3hKQMk^Nf_JIzdzSbGxWlJU8n-n-HByjk`5#dg*T zTve~ac-7aTckhFGdOKAd9a_AW#0e6_iL(h?<qmdzgA%Z+zdvOy6=iUGRvL=eI_K zlAL*#!z6y;HEaLR1A+emmm)yU_so{HhrE(yHSG6WOlsH!hUtX6Svt`IQ%*3uRYysI zicuYv>ZTB}GSABSxx6U$oGzA&EG$?Ix?EE6Wps_Ab%~GsDv~W$fClGVw`^t{@Mob8L2nB|Qi4 zlEzMFeq7q76FbYh&k&_(TZr-Bt6fX@#!Bjx^DK4ze%T==uea%{7J64U2>2i85$;G` z9b7Rr^ru#!WfW5>D`E0boFrk72~m>hP&@aO)oSbr@1A!AV{BBR5F(IMO(^=N>*p9T z7wfSD6SLUN15Z#HWLo!5K9+{{1;prZ#X0g%r+M#FiG&$oI;}r0yDzkUgIoDtkAVe;pB;vl%=jU2ci=`0!_hEh0E5Qo4 zCI>iq>P-{3rEO1qQ3v_x9Z2CBD$W&#hX~n9urs;HQzIMiptSK-X|eAsuX*1QLj!y2 zvuFmf?(hmtodTTK$KrxHS|xT(!>(65$8HqDYg?MFtR9ZH>x|OCpHz?*cCSscUO5~K zf5R{Y9pU}!+6qNA^CU&$EQP%$_~e`?et%$`;p9uddr7hHJWZ?TWeU#~k?S;%=?@Y% z>LIfu8M|O-WlY|{3A%FmunLy?md+#0F!?iaE&}``L=+yVhm}-IKiYiMNyGDeIJtJm z^uqPb{o#dPUp#r-BIs{{%W!f%)FlG)>sp`Kjxl|zmoCnq&x-N=#~pSL!Jb-+;clM( zH>VR(Qbza2Y*N#eG9lI&fv3k{)k*I#+3mShk@*m-e$eMlaaQp|As%JF{&3&f+OCaB z6}2WsJkZQT_}RhaE;J$XJ9824be*C7O8k?{&8tqikdS}dVWrj2Tk;leNfczGZ!Jtu zGsfN=$HB;5r8*I-b1dt#KYyv9YWZ|{TPdAj;%4}pyrq0bKcM(KlYSdEmkN{Q?Y?ZC zxOry2*89ZLiJ*UPizc3;HbndqPomiMuesmWVsLj0D{bUd^C_xHBrBHn>uu8m7G;TE zMyY!`6{jBz*yuyu3N`DCwCjl$hkPu@I9exu7Hr%_>&&@IkHp&yipYdM_cYlc6elfz zWzzvg&$BU`l@6JNMjFhvvZ&4s8mitN$sTAf9sB6=ySfP@3l?hE9Od4oNSc~_d7D*q z?d|)(-azawA083cDLkkeWfTpGwE6b+I48LSCKkf(6MQ*llg6@Vv6yq(+ljba+w+@d zGG}l_QcZ}!RK+}}L?4b(;x_hfv_C@HLW8i{iL%?acHWQGn$kUava6w#VDOSb{&_|- z)Dq7;f=^c`DRr>~XP?vkwEeNLJM30XY}8daq0YBix8KIsUxNcWCU>9tjCJaz1f`*m zw-Y~UGhlNbqzrI6=k?Sye8%Pqb1W5OnFZ8LqDHR)yEr@?;{r`e$&t8bA_?yFcbp>R z6oGx~Vz%PaGwu6pr7bz?Mq4^WqNMk3P1wAC4`uW>Mt)D$?CfaCXw>|VmiO=~O(ftH zuj>}hmL*(IU&jur0ed`ytcf!$VzN&&=Kr4BL96;j-McaA|7UaBl0#17C2-A|pgpw?tm9x;)pmaZCr7~c7^2QE=p_x2=T6M-hL z7Vp;|!?R86-+A9%xX!%P@#&t4+56qL)==#YwRSs~G%dYbxfk`DYfqA&H4Wg$J*QvFfWn~9F4d@FCDkY#E%+Vud)G@Z9#T%+w=OC0?xp|xemy%afh!!8 z%K0`G1Mip4KqsAoDn-xLMZP`WR%bqE1(QU?%NI^G;gpMPE-i-PPp7@5iyC`;NL>>G z#1PQ-vu_XkpF7T29h^|zW3Ej1`4B5~Wc)V>^qe`oD^wyjCVVDJBp1l3eo?v76OpiR zVc?RwZ;6+;b^o*IcaKj?qPufg+HVmO zT__QPM<=h(FY;*fhCKztk8BS31QJH??NRMCqkA|ED=B+$q=7SA;R?;rBE{@`6H#z< zA33|RTRD8dyRTiBs&F-OAgFNRtkR3N;^E(%7X(dY8#D&*$R@s!Y=QQmP8OO&c2s^K zyw5s0l4N}FTbPIKZU<@^bpxyYK$H+&Tt_l|Vb+|c6>|Z%qA;7#Svvk^g}j=^V1|Q} zyt&3;1C&3jlS&33sT>^0pzU0KMjUl#1e&&Hu3sZGYcA)pGVSfJtuK}CcCnoE)HHp~ z1NND?iR9Z6L5G9qrVi=Xcc!ckxv>K(jFi=3vD0%;75pYUQFFOe2l;I_>Xn-bLAEKn_VCiei%8dtb=zjP-;YIW;^49Xv?|HaRWF?q{zW5b{uuX^~H>H!uS*PAzKFW z>VxH-7ky+{Oeceqmd!sqNGj8t5d_Q<)GeJIybHw@DtBt`RwQZMz0@dpBz-Kpl&ewA z!IwlNdv+vs*4sHZAEsSmTs;{(KEgwgY^{L@~Ig!obZU;}9FXJ&7#AmfzBRM_9w2d~6(R?S3=6JWBm$4cpPEhS?n zGNC;D#D>1mzai!P(=g!Db$Or`4BZT=^zfrzQEJ*fMQT}Rx$U|bsp!c|2$M*erKA#8 zr_TPxYORKK9%LHKdwsDq}m_qJRe57nYD@?XcZLknIk&8y*- zU&Vrj50C!_m0L~fmK+8@RL%1QW_dx8?#AfxJdx+hgF~U=+2*GeUNtSfS37EtwFS&u>6^NgD}0#mEP4!3m z>jL|#D!FQvLEL=a1kVML!%juRf88WcIk+Y8p_}8UsSPku-0bD)?@kME1@9~NwAX)Y zf`%I4u#V-;#4v(gwkMu%t<*3M)vn(%3s9LkbeaB-paLX3x0F$`M3~iBS?M%;L=ioOicyvpgS$IiMM1TF#&ta!(`DkRT67L zM|j~H1xhJcFjUh+wsTt^h*Xzb==I){Q6K7^z)+^v1`iD4p+0JVeEK6qppVmYckl-m z@+RG=Vn35ug?9AyNkTC}1Mg|BQZOy_Fj4&o`cIDvedhHsUPhjtadiJB;(Q&*c+K| z=eas3HO|HzdM@h#R3RQ98$yZMRmvqsQ#B#t`i*~%HqcXqZME3lhLhdp;uO+?a``Vi zviop(=BiDyN!`2Ju~a)LYQ!T!O`8x_$QcK@k*~yeJM)WbZg1<;F!uabTOQchS6ON?A2KRds$(xQx!!^w zz^;D&-pCI{@n+Lk2McLrx0CGGtvW_4Si~b!`5T~bXPG!wEaBqPxuAc;s|MJn$(G+e z83KtlA+za%s_A3z)|*eyjX=wNb>%gEivU*clkjX;^1D8v!m?Rq>mSclu#hxVE6rFB zSYf%Vdg{+BbOMTF;YoG8O8F;0k1HLVPG2>5YJeA0G84g4A7b}u^7YZhR@X(8LECAdrv+iq5hPI6GBTmVhKBYiv_&Ewj zXLkPy)Yp&q?Qwb0GxQ|U0+mLAsy4)Udd7zB+*ip5)S^xB31wCm`qYAw8W~QXg4vJO z-)g1huWdRY>y~uL%XO+OP!9ik5&rNCAEpUK4|CHGCLZA_`wiQQ}OGce6K~tyDKPIB=*_`5RTzf=X=w zfDCuDtDL@tm?^a>=L{+GK<>DC?^hgcf<*@y`H+eC13`ya0i0eV+4J1Mp?Z4r5ym(= zm!%)SE4#U`8fUDZ`nqs3ku&7c$@SYj;y`Znqg+LH&Dkxh1nPUojE|mytMjCOFvP1p z>cK>PXhuMYK=Nufa2OY54yQS_IxN4ifZc@ng^$pLN~;7Zg=S60Ip3HMoMvl}v%?d1 zO7K>X2Cz2h+~tCaN`kWks{{bp)uo}$(zx!25F65Rs+)`l@4l2SjKsdm!(Vz8qvn#m zWD8_eqgW4aUl%G^rDQLo`!)XehBfyW!?ipPAjO)ywz)10?VQV4>uOKgv))1^_H^v{ zbw)CGlBEnIW#ZsOLv|qy)v~j@-JR1QQ38&5oAt<;sB>L<);-#MWE|c)^J%Xnw>4I}R(L9&q@|eM>S&SAP^6MZ>qC>FV1%Jk#V#lqds|Y+ z^A;$r*QorYrOxaQ#Ov(o0N>c{@r4pPE~qkU zw-zdMbc~ofGBhQ=N_UFy6dq=O=3ppDXe@&!mjb;Ec=}aC} zDI=(}BXqT3YIVr$I86d>qmwl6f6qiNbDz}y@r{WZ!4*3I64oJ`6Xz)i>Il5S_d3!! zWV{@LQO%(QG58H|9GB88IEG0!88A$0>mx4$w1?h=Z!zb`^Q$l9DEx0fYuLPzXPY9_ zK&Z)vPArGijsGaPdEWrbD~FX;pz?+6c}(AQ@-&^|;Qa$e5Rj_9`sF=1#J{d-b>-S{ zHohFD8Fv>$Wm1K_U^cVKOO{~nBU9jPcA>_uDMLRdkirbOSV;dRQ6psb|D-U!V_kP% zdQjgMlv!BVF(;v*7B3esuS8R|DgN3;tn9dcQF#Yhd0P6^bPoMuMh z7G8UK<0Dn|xe*w&e_hX^p-sxh*h88X~W((jv&zhLzeWwCM5nP04@t4@Bju`J* zev5w1M(;)G_CK6w-OR}`COX^#{#ev0>5N)Yt{{8s?3N7z^~yuWkL3K?<-R-6eVXP< zW;TinLXr;~TgV(KEs0}B{zQc;wWD9METLH3KuA1{mr{HRZN0IT6bx5WtPv{B0tzCC|J@p~<;|qmums9a= z$<%p;GP8%lz2=iY?jdA;SuvfL_T=gAj1Aj5HvA4L`>}`1wW4TN_n?@uShkle{_@P5 z@CqS8>A9AQlr(-1--XE`+ymjh*{O3VARn|&NmZww&GwKX6tfN>dF0QZFVKxbn=Nq2 zDNa%8#udVvx-{_C%NOkwlpB9J;{N^=mDHL=JPB~4>e+ewh3wnSXUx?zcc};8n6zik zmJmKh6gJ?@*ngbMv!HRcg(P@C@)b6?^i&4&!kLZ$5JwaXJr@|(n`Pj*?E{Q8H0V-L<cS=n<2PUc1{l?mvJrQum{rw9|<)q|({fb&XqL%+)b59a9#m`nM~`aLYN7 zD8+^lf~Y3-sZu~$vp1WQJ+wO~-(|twR7NS$QU?Fb&9KIbJS`oyQ*G76_+NnoAMc$A z-NNoN!M*n+jA}0Sfq*)GYhlA)&$)(_7sL0DXP=`e;cG)x3)t02@FP3s$CBGNM2Gq& zSBl&64}I9{gj%!$V27MOHL>ik_c*sep-^E zs8L@&iLXBuF(H5}cDM&eI(5>%rvH#cSRECnl za?c7Cf^XB%#vM>lwV$wlUu!qZ6Aa^mIfs4nSK6)7NVb2b`3>a z*ukPzoO^EKtPxM*+NNB2OEiwn5K_LBJ6?Kx!2llfn8IQ`b$6RluhPCMoDcT*ORe$$ z(%uYZw!(*y?TS$LBzp{H_7HQ-x&GLCq*YWsGX?5TL`m=$K|qQ(*8nBK07Wb)t&Vus zDi=816SsXd!o3|wE2~=miTvE5`j(RrRS{LaJPE@IYZxB3Kfc#<+jx#qhJ9g`8oUt; z-u;0vLNBG=VC7ySG|GJ9bk-D`cQC)K?OgCZ;CAM`j7;VqVXp2>7edDzxER_%hE}RE~W?_ z5s0G;D`CHp|M^i_mJKAAGN*+UxAk>apPK|59~4^(;;c9AUuqlSEix+B2%FiZKt71G z-IO%4rj?6Rc)whM;c1WT{-DBhG=qQT)obOg@kE$KdlA1YL6Xu#=dE{BJu|8L?xjp9 zF=|hC{E`F%7~1tk04F+#i;y)D5GECev!LRfZ~T%Z1KoBi;NID>cKY;>6SRBS8T-A8 zH>1I?=?JHV0=M;6AFxJ+Y<>Q2qXMLl)6kuCKrHl0^v>hbCKABNEE4(iVSANTZEifU%0cR*oBLV0!q=9;)+VR`KDJQzA#L^Gj@ zQ~4rTXB^hl*}M%AB=Qzp3IOC11K2M>FW3Y#izKS)-2qK8kKn4~q1z<4(9*n345?+V z%b(nDdJp$iH~rP#8)hZ224?Xx$845c*VYmvwmPhSaQX$|7W=>PYscA8n(>SfVYnM9 zl29!`tPVzhC)AWXp>*h0_3J--!86zCA<(=QMugEaX+UX@Heb#v?hS%A-@!)R0U;eR zoaEP4dtr3=1&}$!=QuBPc%*mZ4xC@VJR{FWg6MP^>-=vpG8p}T?7at6Q)wG6 zilc+fD4>o?FaB6iM5H$ZGAbY_2#6>Mh)8cjD3Z|BQAPxUfb^mUsRANRx{46#(mPQp zp$4gue&3y-zs#9+?z;D!a~F3lTniI+vct~zeQ$f-=lS-aAk}4PQU+Hi=Ok2MLfFl6 zl0BXmAP3GJu(tfbZ|wje7ZZG%;7$E|MOxi>TD%e*ZjV|hjk#`nrtTWK zq{)BZjBS1s6$}F)whH_(QidVUIU7q?kpx9&?=(UU8`N-D$Mv8lB1jM0xhXcQQ%<{% zEF}P)WqaP>q#sER6k)5rUo;H7(I0n}k~3nb=>yc*g65chT=^Slj)!9S-9cE;eP18xECO&HTHAwlW=J|dFHX5k{d-<#=5BYf0Dh@wg}9N9dA3H% z>=*)w_I=R6Ls&q3UcNJqmm2diZT!lZY31y}naEn$oqAx|^6kenJkG>5@aD;Bm2@+Y zYuDj60e%CpF30u(Z3B=LDg^cI!^o#Hv$}FNg^r-SE|diA+wx{XyTA>Wcm$RbSte8j zccb=_lXdazAB~^(7j(eUGSbseUVeTRJ9yaCY+!4q_sG^qz=NtqaLMg6iAc{#esa|U zeN46_x!KmlA&VNCCv~II@)PqRhEoe&tKbCfuhMZPUgJAtGzR}}NMs{ta6Jm!?N+PyCqGY4mcA`@o25LZJ5y?_DL0WPCS$EMI8(*$EY z`XPO5PhlCCLgmsaqtY_@&&7+Y*GzqYAubIEo$Z#(y&4#>g)FcauHcf@8z z&9(;I7bS|Mn-&wkU;Z1P!}sbr;PR`;9ad!b+0!I&%c(MvNEMQPRPLg5Ciz`VZWo?* z-8R9*;00@sI-vha2CP(I$DB-}V z$v>ssZbDXFvmj2Viv`b_f`cHYv!d?w0cHdZHQG4aJQCiTaFIR$vXBB1odPAAJ<(lm zZZ05A0(e$}R&;+Qs#p_cLg+^r5J=i2H>olJBpxnYu^yWOAZu=)xcR7~^7mVW7S@6! zN^I5SRG!ljP>9tS7sN-hFdvn`d`px-*h66NwOo(`D%CW>O8+C>*yvPidcKh4&*Qrz zwc+MKapkAFK^z;(<6ne3I@`J~Gm$#a=5t1km zj1i*Gs>e7Zin+OW+xa7|E&-5y-RTMEOSjJ15_r!8$;L1x}Y3^zFa+4mc)Bck0t(yc}n0O;F6swIVl3^+H*75QunhzJ7KTEu&32H zHD6w?qITdXmLT0K_z`&U6hGzUraQO2cxjXlz&}6i3EG!D*}0hVu4kMKSRUDLv1{_9 zk+fqUvS5)&T>%*a=375Aph5W<{^|ca^?#s{u5HrkyDt zXydAKPRKN&Z!1~|y7wiiumCdV9H!nx8j4pbjwi7RzS8&oo)gCb;w5Ic;PkllpkXz) zqoLf-m&q#w9`{itL&MvLzTDE55EK3z7d|kS`!3&v#liKbjOBfiZt}$HnKrG>vh!I$ z4l6s;Th1n+NnNoYMxuZp)_^bwX9|}dqk*^Er$Bj4yV}Zgox(E(lXvrkNLw@QARvFw zeq6#xv)wwP*P1NHxKBDDjXEhPNcN*lcjp|URHwP?+u9ozH)jRQ|ZdrCe*g_dS0W*{8| zc^o?>k@~)qKIc2dW@1TO6RN9NG7niPx*X{)=zTaa@!4Ri;kD?sI7qBg*JQqI*$sE1 zB`+Fy1H?iiSFL(V+8;e#I$Areec!WuIjO#snw?Q&0fNo?_uQx+#5FVRD*~R()nV>c za^lBJ5X_2tb~Kog+wG8G_AA7n>E<%;BP75XUsa(FQa}0QzDeM7@ymy~!(=YCb_mCT zkB?aBdEI^j<=|5DQX^A)Yk-W)7Nq^&@hT(lk8}lq7jhOvezqgJS>ON~ruZ(8&KckV zVpw(lK*?8_WM3(LurIEX75wc`gMYP4p3*AuC+ob6(Woh;1d|1`5!hIi>suzMA|~1% zRbFwN@m~zbo~B;llPNbn4=wUG03w?^g!d;D7%ce?VHcdIMtkm&A%j`#O5cA_to;@& zduQ6vw!3XTeW>l{JZrzH+YW?<19O~P-T|@qosgL$;T3pS?6B0_zl4Ei!ew<)dof9* zO3lO$Y(-}R8Mt8PuJTllp&4+2FP3AbpW=BijUG}oc6z$ z{&N=*c@$Kb(|e88YGoqR$0r{3?fbOLAzsxzRumd!rrCm-`& z^b@7)6YH%o=7Q8Vjy{0fV{S9LZt{=E3oiM$=8JjkaWt~$Rk}&N_(g-_F!%pJ`y~Yy z0=Re6df>CGkL9J9F7hLOV>JjwoX4@W58rcOE_NDue}$N5R9TlVw1zW!O$Yiao=5oP zz2)_OC~>|gUanjV5yHmu`m%pbvw*kA{!t9s0FB1Ik%KM*IBM^=L)N1pL;Mj1hRJM8 zHmlC<&9A2O*Jg(oPujjjpa&ps)&L9w*-JD}DT5}l?|Z%T(7F#M3FY}|lc@1je-E4d zc=2jX|1rEzs}~;Mj=dWjapKPI3f4f_?HLjQ+Sp#BApO?ukf%EgrB*ZN6BuY`osI{V zbCl#Vnt>XN9Tvg?83qvh>w1S?9;Aq2kjo;x>dP95~aVxwoUm$jZ%2*S7)+WliDD zPn%rg_=kU<|3{K{pzf&U`hz56jGoQy>RmIMqlck+mkRadiW8()R4)II^7rJV%&e2~ z>~bbBpu7P6iW+$o~#Pk@oPwjqtFv2zwn(Kog+Z*NY&9;enikd z;H*I3Zo-Z=lR`>4rCb`ma<${}Tr&0r?dzRXJ$B5uYk-lr`zBF=KAlbOJ>qF~-D?63KBn8! zf->&Yg{hy~#&q24Vxu54S6Y;+%lHTG)c{Jz9+6=(oIbrflS0{Us@}@{AHg>JyH}w3 zUz4cu5m}jPICel|SlX3g1O+8

-c*Gm(A)xqcNJqK!qWz*xoNyY~zY8J*Lg>gRwv zg1FWHT6!Hv=6I+j@The1I=jDrw}_B2uJA7~(W^`zu$*!2`n26g5^9|Y$fGh4tUAuvwV&rx0E?C+|b~VA} z5!U2RS+&;HZUnf-ykZ5ytzE6A_CxEA$YgsR;7o8zN#j6N z_|6r=v?aDrgUPV_@uI{XTHmwn@zEN;iCY6En(JFgq4Pz$&^LV`aXN@-nSNf44-9|+ z@^E+u#8DoV0ufhA;%aC+tk<@H28hnHcl+YGXz@!CD_v*l15TrLqNLz9l;kh(dSY+l zXn4t!1NRu)V?nL6weWVV8AUMp2X%m&ehR&M1UXZ(KTLR+t8u??C23|nt^HPw%WI`o z&qtf{ta!0tSG-6^=1T4ih{%EE@n6UVAzq}5nATAsCo3|LBjE~^R!hZmgAwRYTMIBs zI83jci;z6*?Bkkm*N+9LMay(_0kv9*(CLQoR2L8UE_6Y*gar7X-K2&nz*$v<0RXL| ze!rXN`Xh3zF2=)*W|ttq&Mu3>syGN81u7HtRw$=ZE>Y5RkCVK zi)`j+i^Gbl*k*>h9g|!$M6BZ!ah7Pg`1t? !HNlf} zA|3Au(crvFjzQr=q!?M#CQGPxt$$vM71HISD>bWGooU*_>N5WHOH;G#WFT;8{RsYB zATH(~-x@GqsgEAtga{D6!{CCz!1R~7D~FH`WgVIle-xHmeXVypZmF=C=**1<@H&(j zxNwHTzh>xe0xALuPm;9rLLQ%@MBx80cB#UAD+qNX3yLKjvMTLluWf&ceuB8tJ^!^J z2=@N+=LL}bmB^i}Su^2$meC{AO280=6k|k3FajU2bu} zLTF|Jh@^o4^9gfP+q^HMI~uOpW@DpP$B+7OQfi>NnsUyUw+I_e9mwl91pcugtE7pW zOGJ@QI5_a54u-3S;`lm^%@|a>aRAyl8iAalc{_(VFt__0t4?_fn6*Oo-n!bZ@%jiQ zP(swcSjdXc{UDbiI65}lU0JtB#|xC!G+X`x#CNf?lDF<@SNo%w z2k!$kQvT@~{oU$;)bN7qPj;cP+Qml;fwXVXhwU<; z2W|Jg%f}!nAGWj-wKLYcXBo37q2-7iS~#> zxh>8|TZBIX-to=KRpkQ_2#i?WF*)>L0 zxKt|Afgnl`Knh&Gat*k)BKO6jT)V=mbMwA4f;1DY1_lq&l(S=blVn6fO*ErRQm|b7 zFtYr3{JbwOkaHKdl7*>lHo7e$?jvRLUl_lgkO-#gn}1FiBe)bW%Wswj9_hP8U=0eK zC%a&FFmV*Wqax|KkwECT9(({R7OI>5*k!_TE%0Cp>tSkp>h_X{15 z^QzsGXsamTKizYm4FRN?Q#E@@YG^k$eLFQS?A638?sn18cHpwf7T|!!jC_yt3?Q+a zmEx1TcRuvueGdTbZT^eM2)#l=j{0c0$=+fQ{NddREjXT~$iY_-t2BqbBP%$D6He^XWkC?y*p?zH`y2eb4Il_GM$3ab}4VEu@DxA4M zXm#hfUF2usr{p^z*GLA1)%`u5P)JgA9RnsJ{ey8{9QYQk``{px@R@>Fx(q<7z2 zK%ixCSHy+Dl&_RWO@MLf=)rU~NQiGr2Faa^w0<(g#LGL`W?R zU3-VsRJbWjb1UZ_h9Uz(n{1=z@?XpoL(YZf3dNwnu14h2jD%7J=@SNwTm*pi@K8>!+^`~To& z*-v7u6A9Iw%lFXtanQl&ox7ex8Yw$0Kl;7OrTy6*(ZDcC7wDDsJ|>D8P1LwvO|?ZD zl&67LS&OLi+NM>R%Lfa~vHh=d5H0Kj_Q0%p{JP_N=rK(t&U2(Y#94 zpmX#hzx$vCdvL(^3X*RC{Gc|C5d~AOTgZ}}&%8dq zNK@89IYJ9+*$oZdMWz6t4lCi#)m@t{0{>|1fhL=E@vSLKz8E{3?Y(p#qv+W*2weFG zpo>^dw0?2E9xps@U;}){iy#>~OfiS{rWwc-OIOz>onru0BWK{Ub$DcG z!Y9p2Si%NOF-S)xT%9{TDzj01o0><$_Y6o3HugwEmwN(zV=e+H=>JV!MSky?%*Y$K zEomnVSee;iM-yW|>B`Por=FSXu#J9-x+WS*b7X(O|0fuQ0Av7CtTAi7V{ zSgl73xQT#Oidu>zTWL&H;q;0re64KgPZVFarOQig8qdMMsH>KS@9CG*>xiBE!H@X# zOG9(!f{L9lDFVwJrW9oa+b6gM^(xl%fkgg%wDcuu3h+~GZ+_404Ck&p<3zV3t)R7) zjkM+(n%#ZekzS>{?ZC~`O1o*dMu4o0gKH_#R`g&pw0CAO{|}YHj(F>8{%ZtV;el(R z!q}B?;ZkC}A`WWq)c$y2e3XoGuSS<+7qm=$mo!|Arg?y&jPyD8mQ2fSR%Q%@tG*<8 z$FUzaHGtB#h!5QrN-HQn6~?n9xOH(bV6NA__+gGzbbzaQ_T9JZQ*9S|z)Alq>=rB= zAT$t9@t6XhnB6LZN*i)zgap`KIav4YP)*;?{hg4~Azr%ab^V--#_@r#+E7u5gLht$ z#w)1-CivEY`SI+H{2x{=0^RtM4sa5!wG4HLvh(4kVgjA!-97-Qn;}JZ_gKE2$)}9s ztLk3O1wR$S#sFKRpFX*wC#S#0TbC-Ssryw6BlQOqNfzDTG-lJG>-7iJ#zZH(Rrq}M zvL!{M+Ap)wbx`clWQM>{VDCn-&U(3=LZ0Yq+8 z{gbW4`BhOUl^7%5nOCY*N7gf+@5w8zcAXi{|BR2SV988}MtQ8uS~|oaCk)`J#|lsC zV~-YXdQaDh#_aIun|65J2Sm+gP_RJGnmb#h=gOGBjc+{0eU-11wRr6q z$#1J5Po&OjyrH6T+5{Xo4_SQ5<*H}xR}G+1oSriv7aMo0ankhu+Q$b5cyHXW7L=m<%l#H*=Jyzv%bQfU z)kTTkg2+<2nCnpHkQ+KaPU_ZMtx|pgb;%VH4L%Gq#57Hxuy2}{bA2K5x7bKhujy*TKe8Y+`Z(pNywO0alo2G}e_i=H`djALydcwh*dAa>fGA2S`5NQ;wTx2zCA3G9 zxoEN2^;Yx3z)o?8zSd+*gt%Yh@0y+j63BP?C++*cJ|31UA}=n?3;RLf zm2kB*+mM}*fAx-|>%ptT{jWsJ&p~4)??y{l zp5v?mdQKl(gQ(ZkiA8l+k$ptM^A>8F`{g9d2N?!Ugxf`~wgD>;4$6w}+eRipVyrZ>iVz6L<1IH!Y7{V#s$RmFp z7~|q7Rb0foKg^V(P34?MM-#R3fH$n87(I|CWdz3huG(`p?#cz*&*<^+g&5CtkzuDIc ziq!CXvUV>xP|4G&F}J%m&zy_!U5R}N?#pp(1%5jIdMN$xel=4GId_7VYT~=~{ zb1Ux*y|skN*cI&D_=*cy`O3_F*JIQ1)l#y9V%cw>G@4n_M}uPDasGN9$6hWm!~F3l zc^M+Zjvu;4ePsll6{DGSlI4go$0+?YHy5@adWW5Cte`!ncj-&ipfdRwxtepYkGo}l z14`~_s!q}b-Y%q|y6DCl3j!}yQTh7gHy#e?XP=ufR@@}(oFcq6vwU>P)!5l{u&W*q zOJ!%DltO$;ACd{=nJ%tA)f^?5!t~=`*otoX+wYDkz3EDzAm&)(Y zu0+&tR;mH2UTrJLkvW&G=cGML5gt7)xsBoWLeWHvOhuDd=Z)^~5A6yG@=IwC z_eSl$tG3{={Dp0yPINHKWjL4(dx%-4<|}=4c`cJn4XlhNZPqK7$#2OqkCZBT3bJ7M zuDBT0q4iOGLZJ9A)Ld?x0R;N`+_>QwnXCQEa$9cI*s+zRp2-rPqn@?(lPfVK0}<-v zz4>b~e$>tEra7qEx)BJU$~0dbq-we)Z6%5#kEL?RRW4~8X^|&lZj*`5YXw_gn(VbF z4A{8Y7Y-xT%7coyDF3b-)!%p?$C3`!Nn)mBfeXmkUwo+}dhW_v4IJGQ*dUT}TU2fu zXbnzE{_M@b#>y-Fj0YrK#NCc)1bcVP#C^tBe$x;xfE_4ZQrGG->DUjCRl4CtPG^x8 z`~V4TylU)Ew5Wbs<=ajX=qD`OqTIxS*#sHQ_6DbaX72j2-7xigd!3ADG?nl7+abtN zlt6==pxz}_vB_iHUbj{g?Ux|H-))BycQ&o3rajhCXiW#>WF~yAq(4eeA2NLa`$-m5 zg}KPvRo^Cxn@)Szv7K;$9U5{08zrfZ-$)bZ3&-Uno1Xsk?7dBcr0zfS!gyx-a|K%6Vgd_wKM$S^#9I!kz}%4_0X z8V@-2Dz&d%usu6hS283g_VX2bIRl1-IpoI;NRnBeAS8Uzz&lqZd#@Solh>!WL#u#K zvG%3L5_m@R!rt&=7xHnm!IE|NSB<>eT>~*+g6r~_S5rUQU*J6k8zl-rwRX3EZ>Y1` zG3S-Fv%f3Jg?C}0C#)W}+pI;*kFs6VlQ2Wt&eI?LIqq%O=)0UMk9V{!-F{rNy#V0f zVGBw>4j@mI3xV=!Vb(O5rXw#a?3QJ1y_!1Kgtj87BNhOPc8K#l`LV0e{+iQQf7zi1 z$*^72Z9ItOo4-pqgQz87Sr-Rtvz9Gw7XtM)CG6TyVn_#fWdmcR1IS6{(B`}|;78|v zrIfB2&$Z`_4zg9-M2NYGg)MnPU3&g$SoE;Rnm*s!_j~D!N*c`L|L9dJ>oNz>cD6vm z+4KF9+1hvg{F|5g^!9Uyfm4Q4LpB^E8~#tO=I8S^p8x-A1b81~e56mE{-6BYEkS9$ zM4f#GHFx~k;DPF!_y4B;8@!WR;1W!G{{4UUI1i?9>My!gOWE4g!MLMEG~e}HdS=^` z)jz|cGVJa0;QJ}mM{|c^QULW(Vs@7yt=i(1n(;fR5fS;VtS-n!vK_TkQu5irnN}=C zv1{>f-5o~T_^dVl>Si&bD77vypKiOmGe_zjx6jLB2CURG?gu%{+WDSKoSSbEL19IzIRu?KQ%CZ-({vYyXeI@2=Lm#du{aZneJ5ocU--CuMa zZxdbL@fstT=RTEfcOZ9cqKSBT%)H3yk<-Dsv8Gg<^tc($kk#-gCb8k+`Bs-615GYR z%%ova=ByeQQ+|;YHINz%#HoRxbvMLpoV3Qqf9d9SmY*jq7K#8$k zh--2kufuy;;T-B~$a%>5 zM|@sAuRq}J4;ru2_WR5K{-VOPtLzc`ejmKa+rZv4ImlsH27R!dqGbTT|9e)As}%~* zq^O5QE;|%{xO5$;-oCtNh{<1*6rPG51TD^`ghME3fNMGH8ViZ@2bZ%d{NCO%sCoBK zi*t;>KNpq;Qp@t}LBhR;hI~uQ3;ce9x$T(9A=n$K@6&}{ou6XacdHmXbS1V+96Tib z=WdtHZ?*0!M2ErFq=BWU&ZFHpE>U@_YMr?^;l$`|}L0)CB0xJ0fc% z$>^E$z|yENNP~wa_=Mml8$eF91)S+Dj=Pz_py>{D_Hb4O;3l309nZ`H3AThFFznhX z9<*RZQygtcD6Ze4>9;ekC8+d_gce+eCNh9O{p^D0fj|BJ<288iE;uI$c?~%aIsX*w zu`BlP1Amk{@W^~zIvvU>GpWi5bH>W&BOtB3TZwv@Ue%waT?KSmAW|CC+arfe1IXH1 zho|G9Kya#5vl@L)f3JZRHhR(H&_^bLI(aY{O~jtQ*B5kkbbME;y&k zb&f?XWDfl~#5>FT_ACAU#g_M$z&pY(+$|M%Z6Xa>S$T3}C9 zz&gso&%pK%ezp5cqGiO-HPsfzMw@L(l7MY8$`~}o6$5HvYf))r!w6OdkoJjr|PORv9zCKgk|IhWAyWnFZ_;- z+t0_4=g51v!#T)n$a6;I{5@yd8wLY`L6HRZu*>+g{ei6yz9_>?)W5n`G4UTgFpD>bAT3Be15wBIn3^UkPTcMjp_pd@dKEe{RHp8hJjmwFs;_ABgp(kt9j@- zS#BTv6V`D>g$DkK&4_Q`Ne$UuFf~*=8>88oKh4g97M+Kij%&zHc{sX+Fg}sBC zSrud?ft3w;^RV%jBstit82BR46h?%4^Gj&z;onZ8#z%_Q#LwU2z+6)gc|Wv{r}wpv z1J2{~pMDO&^8?V_`}r94IlOoO{fE{Ek=Ky(;CVbq5Ued6E5N5T!I(b~e)c*L99bOw zgD>V+UiJZF4LO|p_c27~0M3q2SaG+G1K1Pw@b(bIMY2l#zKc(}KKFpuCl>Dq-u2LWyC`+6a;0qi~ys2WF{)f|9C`@sPhu!D29X=CZ5_HcR)t?K)Gc7r zYktlle@QB5&>X4w{GZ4qgnEnkpPzmnz;mik`so<;b$IUsIEVTgavpL%Yzv~t{_lQk zFV$}$?o8mBDiea@g3I&WS@xI{Y>0aJE@Gkw?)%+&&!9P-`a3O_FMhCc!hmqiPsSuc zd=^Ore0RnLr1~90d>GaD!Qvmbvvau0FPNHwxVkon$kg0$kmso~F&)TYT&ATKAa*u( z2;@6$FS+A{IBU~`3S={#6Txvvq=1fDfKPM~=V2ciVY%SoJ_siroazo>x;7Rlgh$(0 zVV9!jw@E)y_>P#gwB_zBo9IWJ{FX{T!W4=Yrs}C-$}Etm7R`Rp(yXVhR`d?&y5%RK zV+;O?6b}ZAC}4egn-gHisRuj3hev_q(<$*JQx9)SB)J#04lH-j0FSpxc4D_Lf*+je zyZNZ52`;q>@jpNPyocvhpY+o)>g({{_izsNHRL?x{0kbvf`0$%w*U_P(zRKiqWqc*|Eq-!&3UtqDYNERNn_HVz|TOuMuBe#|8| zbuP!usdFh>VV9uB_(!(rxEYB}AsJCtgDZoOg>D3c#lUX9iUGrofE~YQ03WUfjxU8M z??OFH*ECh!V}yFX$tVbn3-=U286B4!<~+6!{q9#Vh{dGe4Z(R?G*Sbv#@ zWkMp4l|``V*M?&-Yc{o%ncW+Pw(c;AHHP+SJ9{&8idb@VDPm~nn-;1Bj&20A^;`)@ zzp=YkmhNn{DX(o4D@Z!%-YPSth`Mz0J6o8C!lo_ z05C{)%m#%1e*i*1@&6D)_rk2gwDHkI^M|5^fqzZxQ49#!tKr8Bh1ionHtOefSwD-9 zE#AK!rNH7nFDW!`{0vF(@9f+U_9dpiQH`xjDAz(qzCkmRz{a4tLT>V(;6;qgmuh3K z!yM;%0k*e9NngDoDt_$-pE&d68v=^`93lc^lH9!CNxHn*g}w zUs>0-AoRH!APsDkVe7CAi)d!fY8u#1->k{gtG$wuvoU4s9iz9@6=%1yUoHv}-4!QS zdw6)$z74Oq?!BTwi2`o|#!s$ij*@tM!z-@yHkhSuGcL2qxmbbEj9=NkP%D`4&*E>q-*o79!H*^(6ADnBf0^B8U8LD9f2 z*>{}^5S>i}ak}I08#UbjNgCAu`x;II|1@OE&su6T=WuV)voDmwS}EmBYm{1M6lsb> z!s6o9yNAw}kT0dD=Uhr_p1V2|W`yP*cUz39n#JZePucXJt0Rmr!w-7Vfs7hXFSu0K zcGw3Es@}2qjh?}}>=}qziZOjraxV;>dad!bpoe$!@f_}iM~WnlEN9tH{!g$9!ih0v#d3GOw`$dJ4hIc#|D*++Hrl>j=jDnO*nH% zZY{sr-&bfNNw>2bCy^vMclAlzplZR3kGROC?rr{fHn?@{gl$SI*GyhZ%ixk>)%eox zgC!*Y$)d>nlvLtUtIroX>#HSv!<3h)BOoRj+h_%@AJ*j09f~Xqv91PXd>w_u;kT+5 zI^R@+M%1QVGp}>3W@JXPNAgRVva&|zEv;rzU4vHp%suKjnz53C<9a0v6sfwW%iW4b z#F7~P^wgY7=@789A~*OH>yl5=81Es5&oLz!Hjyyx+(Y(l>|SRV0(?VLMQtfgTbs!v zDXHPYY$X(}u*Kr#XSc@VhulY8R+vZ`%_Z$eUpQaYv#CJ{Gbc3@H)u}lk>(_Rp4~4- zOg5(8ys1<|Z}V|`<_a*+`?gVpo0A2_^Y0AO4^=z0mdM{RaQSF;3HMZnIOBT9AVG(( z)>x)aKN1twF)Uv!L7J~6jCVxWF;?a|&svBVThiG_aZ^y#wnq6A*bZvov<*KJ zY#oaYQlm1_=ZA%@&~gKArAKF?a8?7tk?w`2+ybLZlGx%J1FKZU@dq!hi(NSM@j4#~ zev|HYlY*F;4Zs_7L_LHW178NKv(nXks>L+Uw|xuixQ) z-eWVr!hKy>35P0$15Ldx-<|LfZRMI?`eZSX!C^7bqsQfHF(CaeN&!z`tQx>d`XSL? z=U$#iry+}9n)E1}thHP8COWVE8?(JG3CWhaV(f=9P%67_9F5yqLi$~V&UnP0(~nF{ zdi>}Q9Hy(|jzR7zl$uSdfeZe2h~l`PVys>7RW85cgHd-Zr^Af0d#&CGJQ(jZP3t^d z-FgZfKoKu)oO0Q(m_DL@P}jyi8n}J@8~xh#*{>C3XEgomDK|mj!)0Z@F;=hNFh#?{ zmR6(->H#FnX}Om(`ALMd8r=LXp~!IoCHxdaa{b7jkW@O3~cSPu zx;_zC1-u6CCEVjhR`Kke?LKW7PLgr6@vSpY>(LUT)eUK_Rrr|F5jWUfE>^(+Zz|3{ zY%y@vN*q)8(z?*4alY6|3^Oj%e!Um9vw7i34Ek6to2<0wtmW)gDC)YJBqL6O5Vi^9 z@cM(PCG^-WLGkEDl->||{^?B5mKj>!WJ0m$+ zR8&Wj5p7FOm+6R2JP4ix@18nqeb-Nu_<-vZjAL!fwV z9*>o!9;O?1H5|{*E6ItVpV`crEq#|Q^#blnBx9|3GQru>nXcSu#J-7y{na&!m)$Q} zY)VF^HD0c@D(59>tScpGYEG@kNfM1SYBJQbQ6Su4FH$qQhpN&YMi&Jx7CU|GE(xj` zv2(1cDLiC-0s!~dJ2p!C^%=>Bo^khnrMIjN_4~}uvH3U?2d>fF_khaqoTb%icYmGT zd{VVnq;VsCi|vOW^a>51D7<4}hfmLasp~!-Kj%JT?zjYHUtCOSN;^q2c4SXkE9_~h z%(vm+*QI|ysNd8JI|>oY+q{YBEv{}HxL&uYdAD?aaT%+Uw=s&nCu)ri6PsLbIaM-f z>JoEf6mdq87^Za7l1ojbmVv^93PeMoQ8sQE^Ex%=VQ)#xOmVkAX5OfzY(CD-fy2wS z3Vg`^tKR$?!S2HIZaxuaR_Wu98_O4$c)-FAls?QnYUZ&4g%sTkRKfl-CCu?QL9*sjU=|A89baC&m z`i6u_w;_(qhwoT5Qog_=@;t^?z#q=1f1o?>)f%{=HOD4{lxJDO4xK_`Ba_PWcF|xSDI<;N=oajJ^*0Zg$w2#cbBJ zq~XcOzn?2S-T82{;+}5(T&ZJXDt0_7cmL#@eeufWaX(zZmroaPMTWFbk(9}1UfQkn zw=TsVYjJ#V@59Z?TK2aR6DHv+*teOh#@Z%GX=b??xg~~v`F26b-BycpH+zOqXy{ho zBlF#!hG>iJgBhk(*>a34<|R9{)D zes1*4O2X5*hoQm9b|Eyoq8!NdIlTmdKqOTP%*?cxt*+UF0!Y z)eO5{ik0**7p!zx)`e{5s5~L;Uhm*36yBe;WkCfln!v(XfKi|=9IVLCF?`CF2a#v{_H|(9;M5lXpW0u#` zET{W3##*yYN|j~UhpR64bOd||*%Yjb%bZ=%YGoC;Qc!D!``kt3D03W3AemWWPFUew zRP|yTjnfOz`VQm%Zp7MSu8XA0vc#dQIhSQyLgGtgo%~4K=w63REBHox!A z>?2I@2+s$Ha=swm0qnMpBEZCziWv6p^1pTWY<-{`j2QWKJ$uA;SZ5vJQ(mAd8S1Clc_01qWAG7!BeQMLe|Ji`xop@|Zun%~+?+?yWv)8mv6S&!yy=REeFt zx8746CG_{hAS_Q4>#njN%r~7fBl+=ig%EH9zPzX#TZV|R!{tS8l&)rSGFhbJjrOsZ*+>GJ?8H|`ts@4X4wpUEXk5s`0l)Zh*jV&>teu}N5CC~g}{~g)n-r0t09|iYr z4T%5p^@2Jw3twK=hsbrS*~giWzXR`F299^WD_#ozlHoBSnDgm^mEwU0){n+D3=PCG z4ScG5f!~*t_2R#Sciv6##tdO2zrgBSdR=$|mRIcZLbz(_=~nCI(L%W0Ficw1&Ny#h z-X%TmlM$8F$@{+x!v=y|MKOa{wKwyJa+{4&64H!p&Y6q}$Hm6Vnl<%z#Ak$}?%(i- zo4A+fQN~6aTFIBJNLccS$yivZ7*{LnE;`>MMfHC@WmCv6l)8HH3KBg35nJEEq*7;X zt?jBnw7d72#qkd-I=U01XHP3!d{#-dL&VIHPdd_(d1zsew||Su-P3}<8=p?~fk)&y zCO?(BK=xbKPyOrc7qwAme>;Z_WFN~#CrXf+v|ltQH%s+D!W%Co4>{>uK%9eZgW4mP zkVqQ>b9t1BRol^}`^)Q({6+5kV8v~jl|9-aZ~;DdPoZW#im0E`HP|FKt;M;|vYRmW z{OuFHqfET#%RWDisf4dnW`q}X&yW71T;KB}qUKQ}YB4RMx~#a@JQsdKh}aod@({16 zx8TfCcKeyt?(f!3p;@;*)w<=ED59bFAEdP}w!88vd-qS^rTLVZ*%an;Pk4zvR>Fm_ zDd>i5^;fH42;3Q$bizZ-CAVj3uwH~u`Ax$jMt*eJDg`AY@xCD$O;Aauzi@(ICKS4WgoA3O?qjyu&c&gH2Js8VV{M_P+rdhMHmr0#St&Oc2g;&dnl_CY zBN#@j%}16ypW$hevoe-E6#;1eNp{gD=be;FJuD>z(g)(c5Ba`w|_DC{dk z`$6+mG3BOb#}%8BrQi{Hp3v!o1YX1q6Vs&}vo5jkx71+ev()Btis;YKDgJqV{ye>=0}H0tv25tDyfjzlKh4BzU4y zM)gfx;R2+jnvBO=X`kY!Hg-zBZ0EBU_bvnW?fGS2Ah>aI4<~`U)ZmopXh`Th?7w?V z|9gmxrG>~UYKZip5L>UZZ^pa;BP)=y%*wXQ)!^jMSNxGn+fj3A9$GHl=qZo32pf#3 ziH(N4v0R2CNA%8gz|J_&D#*xPkzyy##P!o)urDv0LPTY+653^?8^a#P$;{yE%43AfpNG+4iQ zZIPW>=!s?Yi|BFBZY5HO@eT`|)h~3T`~Fva@P^<0-|csclDHca$x-(H$w(fI9_=vr zF|U%Sw3sMaFEP~eP{uhMHX#)PESrnHT?@|66}|F=jyW~bt5DY+Lv3hh6+PHl=E=Q{ z9C%&jz^2lFWmt2!wPjAbFn7w-W;j<1BDoW(?M;oiqEbuwl@{((^K*Ga`P2KOjV&>S zdq^?Wo7ZxS z(Bbu=sii)UYR8SGVdv%x&{lQh9q3mxb8U?suC>~+cJmx)q?o0A$UBzjqEu{WroBQR z1)#-=em9B;N++REaX>dlfbOQxh9R#jL%w0de`A>4_)w5BNjUpVGhO9#g0#dzwBOrD z(=Mcz<>d@g9LrD_Qwn!s5;ud4bK%ceMVNp1a!ZqVTB~p)I^WqJZiawT-N2;Q6DFk| z(#uknc6e#2G-vyRi;Pb$avoYDclGr0Tao0mCEC!VE))^w>~kx!(WOm}F+*2z(Y+z0 z`HKYlJ1uDz4x_h7x&;ymS3{?<=*A3+nHwg( zu1xw%8*6#Cw`_~z>G+)Oq10I?we@y19`IafTe7d*Q6t?zjL_=|B5)JADPKEK!r3$V zoCUR{{OKO9nc`^eLg(DevLqC_RF$)!7+aLH$?Nwazk}NWi|H@>{H(G8DiEZJ@V#0B zpkoi9SJZ0fpC}~YAlq5A_aemIcmEAN{aOI!#GZ^5uNktBM6KEP-%6~DAHQUpn??ur4)?pCEQaS8asuQP%+xG zX|GN|MaIcFoIx3(eTrhj?9qVHwnB{b-Pi5uERHZS&s>BS06!Cl%~0dmZY7`D zW4_EYnw$VE*4g+Vt%?3gq~TDqFm83sZxgBoFeiOZ@s5t%PJX7{`uO93KH5VEqB!?3 z696f=swx00z=IXn1wvrlX2t0MJeV z@5n2dAyaXVUqS|27mO%|dShqoPO%coP2$y12@sQgdO5LIu?D1R%7B{m3fL-^u^_HQ zJlAoQyXVx*i0S;CV^khmAA?}bYDWwT^j>tc=ott!)2A>G`xMS}%qq z*g%xA9`UNqSm?bP+Ec(s%nefIS-I3G+(?X&aZa7h-@06tQIb2uO;4A0c%x8WR{*7y zgxIUrHxY?UTx#U{u zgvx5AQ!6Wq^q{;n81Fn=)=RuTR)~w9%D_IEmoZ!z-$eI#1L|5=P}kl5!LnB&e5^xm zOtL0hory7UTj}W%gEzVn>!};j;_qi;&BoqdnlTkApldFwexoAc5 ztVG#Yj=Xr6L7ECi|4OI3LFHLjR359U`iCZa?j8(PAvv{W>4KrxfTt8~;3?~B`c?71*P=vxUd3C|J;x~b2am{S zJB{8LJHaeViZUCzRh1J%nVuSw^5&8?n2$EmtF>?*7T0$k{#5hSF{-p-PTkXN%wD^v zppkf%aHXZuU2pE2dWN%5EJi&F6PWh4@U6pCem|f7L zzmV*Y9&CCVlr%K1E+4C?9>0u^dw&ljau<+AL%J7&yLBTF7BSnjyy*= z=V>)uvK+0O>0CIR)00kjQgs72T~D~Fjfu2d#tXteU&k2lrPidMHbdz$ z#~!R-Rur}tYfv?PH6b>#U|_JuvwfoZL5#38FXVjhmvh5gbL-%I&6E=TwB;0A`a zo-j1QC=uNTyq4lgq)TnZnDk;P8l8M1SNt(aU-*N_((++Sby8|{V+O&)A6>f?HI#4jt!YRsJb#AHkb>;L5MZ&V7)71_YyTZOZKc{qR zIUOCdf>l>6vQrY#@12vWYn^O!a!%d!DI`uZ;D>NLHZ0Bb3d5 zFzgW_#c^O*QyKC=zz3y->Zvo>*&F|=Wm3e@W2_=Oh zTxx%wnj-du6Aq3q48@Dm5KTnb&NxwQrXkXXq$(eP3{Ote<78ofp396hBC(g+3o0i; zOl5yFk3L%G)0csk>0;kxZbh`kOI>jE`@y^Kq4MtKX}tS{6_0_2x*~*!~W30ICWPp~oO+U(7O7pZ!-Qog-CAhlm77|1Ryxph|mKB|1RUvV?W@ zpc4NNY4?<(Y8<~$kUQ+f*Z7(`t@Al03Blr4MotZ3VJ zr4^d0v~tg*Dy_zs7a~-RPvXS+ls#FL{BmZ;Bz!iXY5l*Z0b3RG9{jc}w zdc4B1n}2Tp>^6b#G;pDg+XOkIrZ4%Gz_m5tJpfyD_+{o!5>!zJZWRD-Ov>B)8@!*k zVd>AWbvOQA4+CBkm9Tqsb=hEAUB<+~aG>mE?=q7jw+UQbqZ?=-drYaafyUO1gbg&e zt(sI`76OkDfK??^+}_o@Pf+okq~clC@^}9K-~YjEAQy<-z2-Yl1F}jYfy>8$$96Wn z1uh)Od~!+LK^3e)XDR=CFNjhQ`A@HwFA~VOl#0}%1FJ-+ftbD|7`VY5bRRmXfq`TK xL>5dghdL6dXcUZwz-S1J@DOOY`Tx)P|LpH$TYKsy^P@qb;pyt_eo+&%yR literal 0 HcmV?d00001 diff --git a/docs/security/data/benchres.csv b/docs/security/data/benchres.csv new file mode 100644 index 000000000..98bec8734 --- /dev/null +++ b/docs/security/data/benchres.csv @@ -0,0 +1,1281 @@ +, +barnes, mi, 02.90, 66712, 2.88, 0.01, 0, 2461 +barnes, sn-0.6.0-full-checks, 02.91, 65716, 2.88, 0.02, 0, 2863 +barnes, sn-0.6.0, 02.87, 65508, 2.85, 0.01, 0, 2838 +barnes, sn-0.5.3, 02.87, 70068, 2.84, 0.02, 0, 2518 +barnes, sn-0.6.0-memcpy-checks, 02.89, 65608, 2.88, 0.01, 0, 2837 +barnes, je, 02.90, 76652, 2.87, 0.02, 0, 2550 +barnes, scudo, 02.99, 61892, 2.94, 0.04, 0, 4270 +barnes, smi, 03.02, 66728, 3.00, 0.01, 0, 2657 +espresso, mi, 05.27, 8220, 5.25, 0.02, 0, 174 +espresso, sn-0.6.0-full-checks, 05.35, 12640, 5.30, 0.04, 0, 744 +espresso, sn-0.6.0, 05.19, 6216, 5.16, 0.03, 0, 654 +espresso, sn-0.5.3, 05.21, 10244, 5.19, 0.01, 0, 410 +espresso, sn-0.6.0-memcpy-checks, 05.20, 6256, 5.19, 0.00, 0, 657 +espresso, je, 05.54, 12184, 5.50, 0.03, 0, 322 +espresso, scudo, 06.07, 4940, 6.05, 0.02, 0, 645 +espresso, smi, 05.50, 6620, 5.48, 0.02, 0, 288 +z3, mi, 01.19, 71272, 1.18, 0.01, 0, 458 +z3, sn-0.6.0-full-checks, 01.18, 72428, 1.17, 0.01, 0, 773 +z3, sn-0.6.0, 01.17, 66184, 1.16, 0.01, 0, 734 +z3, sn-0.5.3, 01.18, 73508, 1.16, 0.01, 0, 563 +z3, sn-0.6.0-memcpy-checks, 01.19, 66128, 1.17, 0.02, 0, 738 +z3, je, 01.20, 65792, 1.18, 0.02, 0, 2770 +z3, scudo, 01.27, 56116, 1.24, 0.03, 0, 8681 +z3, smi, 01.26, 66104, 1.23, 0.02, 0, 3836 +gs, mi, 01.17, 57476, 1.13, 0.03, 0, 1660 +gs, sn-0.6.0-full-checks, 01.21, 54304, 1.18, 0.02, 0, 2018 +gs, sn-0.6.0, 01.18, 48280, 1.17, 0.01, 0, 1999 +gs, sn-0.5.3, 01.17, 56084, 1.16, 0.01, 0, 1873 +gs, sn-0.6.0-memcpy-checks, 01.17, 48512, 1.14, 0.03, 0, 2000 +gs, je, 01.20, 53216, 1.16, 0.04, 0, 3724 +gs, scudo, 01.21, 41248, 1.16, 0.05, 0, 17117 +gs, smi, 01.20, 56372, 1.16, 0.04, 0, 4550 +redis, mi, 4.357, 35112, 1.87, 0.32, 0, 8007 +redis, sn-0.6.0-full-checks, 4.435, 33628, 1.91, 0.32, 0, 8964 +redis, sn-0.6.0, 4.216, 30440, 1.74, 0.38, 0, 8556 +redis, sn-0.5.3, 4.348, 37168, 1.90, 0.28, 0, 7518 +redis, sn-0.6.0-memcpy-checks, 4.274, 30408, 1.75, 0.40, 0, 8544 +redis, je, 5.060, 36836, 2.21, 0.32, 1, 6765 +redis, scudo, 5.252, 37900, 2.22, 0.42, 0, 9863 +redis, smi, 4.622, 35372, 1.97, 0.35, 0, 8035 +cfrac, mi, 06.37, 4508, 6.37, 0.00, 0, 184 +cfrac, sn-0.6.0-full-checks, 06.63, 3584, 6.63, 0.00, 0, 496 +cfrac, sn-0.6.0, 06.27, 3332, 6.27, 0.00, 0, 432 +cfrac, sn-0.5.3, 06.33, 8244, 6.32, 0.00, 0, 446 +cfrac, sn-0.6.0-memcpy-checks, 06.27, 3320, 6.27, 0.00, 0, 431 +cfrac, je, 06.64, 10056, 6.64, 0.00, 0, 273 +cfrac, scudo, 08.45, 4684, 8.45, 0.00, 0, 616 +cfrac, smi, 07.09, 4464, 7.09, 0.00, 0, 183 +leanN, mi, 25.62, 591256, 96.05, 1.07, 0, 200920 +leanN, sn-0.6.0-full-checks, 26.78, 681948, 102.76, 1.04, 0, 16260 +leanN, sn-0.6.0, 24.85, 545020, 95.77, 0.89, 0, 12529 +leanN, sn-0.5.3, 25.46, 546652, 96.19, 0.86, 0, 3333 +leanN, sn-0.6.0-memcpy-checks, 25.01, 545020, 96.33, 0.92, 0, 13126 +leanN, je, 26.06, 503092, 97.26, 1.13, 0, 171614 +leanN, scudo, 33.01, 593072, 120.15, 1.81, 0, 598360 +leanN, smi, 27.04, 624072, 96.26, 1.92, 0, 354731 +sed, mi, 01.74, 324400, 1.66, 0.07, 0, 402 +sed, sn-0.6.0-full-checks, 01.74, 349752, 1.65, 0.08, 0, 1670 +sed, sn-0.6.0, 01.73, 342816, 1.64, 0.08, 0, 1449 +sed, sn-0.5.3, 01.73, 310148, 1.65, 0.08, 0, 682 +sed, sn-0.6.0-memcpy-checks, 01.72, 342792, 1.62, 0.10, 0, 1473 +sed, je, 01.74, 300292, 1.65, 0.08, 0, 8858 +sed, scudo, 01.81, 245512, 1.69, 0.11, 0, 60801 +sed, smi, 01.82, 317312, 1.71, 0.11, 0, 34437 +barnes, mi, 02.85, 66836, 2.84, 0.01, 0, 2464 +barnes, sn-0.6.0-full-checks, 02.91, 65620, 2.90, 0.01, 0, 2854 +barnes, sn-0.6.0, 02.86, 65508, 2.83, 0.02, 0, 2839 +barnes, sn-0.5.3, 02.84, 70132, 2.82, 0.01, 0, 2530 +barnes, sn-0.6.0-memcpy-checks, 02.91, 65628, 2.89, 0.02, 0, 2840 +barnes, je, 02.86, 76748, 2.84, 0.01, 0, 2547 +barnes, scudo, 02.92, 61480, 2.90, 0.02, 0, 4252 +barnes, smi, 02.91, 66940, 2.90, 0.01, 0, 2670 +espresso, mi, 05.19, 8316, 5.16, 0.03, 0, 174 +espresso, sn-0.6.0-full-checks, 05.27, 12760, 5.23, 0.04, 0, 739 +espresso, sn-0.6.0, 05.12, 6536, 5.09, 0.02, 0, 658 +espresso, sn-0.5.3, 05.13, 10240, 5.11, 0.02, 0, 411 +espresso, sn-0.6.0-memcpy-checks, 05.12, 6440, 5.09, 0.02, 0, 659 +espresso, je, 05.44, 9956, 5.44, 0.00, 0, 355 +espresso, scudo, 06.04, 4900, 6.01, 0.02, 0, 642 +espresso, smi, 05.48, 6740, 5.47, 0.01, 0, 288 +z3, mi, 01.20, 71076, 1.20, 0.00, 0, 455 +z3, sn-0.6.0-full-checks, 01.17, 70204, 1.16, 0.01, 0, 758 +z3, sn-0.6.0, 01.16, 66108, 1.14, 0.01, 0, 735 +z3, sn-0.5.3, 01.17, 73488, 1.15, 0.02, 0, 565 +z3, sn-0.6.0-memcpy-checks, 01.17, 65872, 1.16, 0.00, 0, 735 +z3, je, 01.17, 66016, 1.15, 0.02, 0, 2773 +z3, scudo, 01.26, 56052, 1.23, 0.02, 0, 8668 +z3, smi, 01.22, 65448, 1.20, 0.01, 0, 3699 +gs, mi, 01.16, 57396, 1.12, 0.03, 0, 1657 +gs, sn-0.6.0-full-checks, 01.19, 56340, 1.15, 0.03, 0, 1955 +gs, sn-0.6.0, 01.17, 48452, 1.15, 0.02, 0, 1998 +gs, sn-0.5.3, 01.19, 56076, 1.16, 0.02, 0, 1875 +gs, sn-0.6.0-memcpy-checks, 01.17, 48300, 1.14, 0.02, 0, 1997 +gs, je, 01.20, 53824, 1.16, 0.04, 0, 3729 +gs, scudo, 01.21, 41532, 1.17, 0.03, 0, 17160 +gs, smi, 01.19, 57232, 1.14, 0.05, 0, 4648 +redis, mi, 4.395, 35208, 1.88, 0.33, 0, 8001 +redis, sn-0.6.0-full-checks, 4.504, 33404, 1.72, 0.54, 0, 8904 +redis, sn-0.6.0, 4.241, 30464, 1.80, 0.34, 0, 8558 +redis, sn-0.5.3, 4.314, 37128, 1.78, 0.38, 0, 7346 +redis, sn-0.6.0-memcpy-checks, 4.317, 30544, 1.78, 0.39, 0, 8538 +redis, je, 5.109, 36816, 2.16, 0.41, 0, 6769 +redis, scudo, 5.209, 37820, 2.27, 0.34, 0, 9874 +redis, smi, 4.631, 35436, 1.95, 0.38, 0, 8036 +cfrac, mi, 06.33, 4460, 6.33, 0.00, 0, 188 +cfrac, sn-0.6.0-full-checks, 06.62, 3560, 6.62, 0.00, 0, 499 +cfrac, sn-0.6.0, 06.27, 3244, 6.27, 0.00, 0, 434 +cfrac, sn-0.5.3, 06.31, 8424, 6.30, 0.01, 0, 444 +cfrac, sn-0.6.0-memcpy-checks, 06.27, 3320, 6.27, 0.00, 0, 432 +cfrac, je, 06.64, 10072, 6.63, 0.00, 0, 273 +cfrac, scudo, 08.53, 4616, 8.53, 0.00, 0, 615 +cfrac, smi, 07.14, 4580, 7.14, 0.00, 0, 180 +leanN, mi, 25.83, 587052, 98.11, 1.28, 0, 163240 +leanN, sn-0.6.0-full-checks, 26.59, 650568, 102.61, 1.04, 0, 12623 +leanN, sn-0.6.0, 25.01, 517504, 97.71, 0.77, 0, 11406 +leanN, sn-0.5.3, 26.53, 537680, 101.52, 0.99, 0, 4183 +leanN, sn-0.6.0-memcpy-checks, 27.40, 529564, 110.40, 0.89, 0, 12793 +leanN, je, 25.99, 529828, 98.94, 1.02, 0, 106383 +leanN, scudo, 32.58, 602392, 117.53, 1.81, 0, 532564 +leanN, smi, 27.65, 631068, 99.97, 1.98, 0, 422694 +sed, mi, 01.74, 326500, 1.65, 0.08, 0, 401 +sed, sn-0.6.0-full-checks, 01.74, 347584, 1.63, 0.10, 0, 1694 +sed, sn-0.6.0, 01.73, 342784, 1.65, 0.07, 0, 1477 +sed, sn-0.5.3, 01.73, 310248, 1.64, 0.08, 0, 684 +sed, sn-0.6.0-memcpy-checks, 01.72, 342716, 1.61, 0.10, 0, 1452 +sed, je, 01.73, 295420, 1.64, 0.08, 0, 9180 +sed, scudo, 01.82, 245464, 1.71, 0.10, 0, 60799 +sed, smi, 01.81, 315944, 1.67, 0.13, 0, 34063 +barnes, mi, 02.88, 66932, 2.86, 0.02, 0, 2462 +barnes, sn-0.6.0-full-checks, 02.86, 65748, 2.82, 0.03, 0, 2853 +barnes, sn-0.6.0, 02.90, 65604, 2.88, 0.02, 0, 2837 +barnes, sn-0.5.3, 02.85, 70108, 2.83, 0.02, 0, 2526 +barnes, sn-0.6.0-memcpy-checks, 02.86, 65628, 2.84, 0.01, 0, 2840 +barnes, je, 02.85, 76568, 2.83, 0.02, 0, 2545 +barnes, scudo, 02.94, 61888, 2.94, 0.00, 0, 4270 +barnes, smi, 02.94, 66856, 2.92, 0.02, 0, 2657 +espresso, mi, 05.17, 8328, 5.15, 0.01, 0, 176 +espresso, sn-0.6.0-full-checks, 05.28, 12604, 5.25, 0.03, 0, 730 +espresso, sn-0.6.0, 05.10, 6248, 5.06, 0.04, 0, 659 +espresso, sn-0.5.3, 05.14, 10212, 5.11, 0.02, 0, 411 +espresso, sn-0.6.0-memcpy-checks, 05.11, 6256, 5.08, 0.03, 0, 655 +espresso, je, 05.44, 9880, 5.41, 0.03, 0, 297 +espresso, scudo, 06.04, 5000, 6.02, 0.01, 0, 648 +espresso, smi, 05.48, 6640, 5.46, 0.02, 0, 288 +z3, mi, 01.21, 71316, 1.19, 0.01, 0, 462 +z3, sn-0.6.0-full-checks, 01.19, 72360, 1.18, 0.01, 0, 783 +z3, sn-0.6.0, 01.18, 66188, 1.16, 0.01, 0, 740 +z3, sn-0.5.3, 01.17, 73712, 1.16, 0.01, 0, 566 +z3, sn-0.6.0-memcpy-checks, 01.17, 66060, 1.15, 0.01, 0, 738 +z3, je, 01.19, 65880, 1.16, 0.03, 0, 2762 +z3, scudo, 01.26, 56160, 1.23, 0.02, 0, 8672 +z3, smi, 01.23, 65868, 1.21, 0.02, 0, 3826 +gs, mi, 01.17, 57272, 1.11, 0.05, 0, 1656 +gs, sn-0.6.0-full-checks, 01.18, 56336, 1.15, 0.02, 0, 2378 +gs, sn-0.6.0, 01.18, 48188, 1.15, 0.02, 0, 1998 +gs, sn-0.5.3, 01.15, 56044, 1.14, 0.01, 0, 1871 +gs, sn-0.6.0-memcpy-checks, 01.19, 48188, 1.16, 0.03, 0, 2000 +gs, je, 01.19, 53716, 1.16, 0.03, 0, 3723 +gs, scudo, 01.20, 41456, 1.16, 0.04, 0, 17153 +gs, smi, 01.20, 57060, 1.16, 0.03, 0, 4647 +redis, mi, 4.294, 35128, 1.84, 0.31, 0, 8032 +redis, sn-0.6.0-full-checks, 4.406, 33360, 1.83, 0.39, 0, 8930 +redis, sn-0.6.0, 4.254, 30504, 1.80, 0.34, 0, 8552 +redis, sn-0.5.3, 4.244, 37064, 1.74, 0.39, 0, 7619 +redis, sn-0.6.0-memcpy-checks, 4.257, 30400, 1.75, 0.39, 0, 8555 +redis, je, 5.021, 36832, 2.09, 0.43, 0, 6775 +redis, scudo, 5.238, 37884, 2.18, 0.45, 0, 9872 +redis, smi, 4.599, 35428, 1.93, 0.39, 0, 8050 +cfrac, mi, 06.33, 4568, 6.32, 0.00, 0, 186 +cfrac, sn-0.6.0-full-checks, 06.60, 3628, 6.60, 0.00, 0, 496 +cfrac, sn-0.6.0, 06.26, 3332, 6.26, 0.00, 0, 434 +cfrac, sn-0.5.3, 06.31, 8408, 6.31, 0.00, 0, 447 +cfrac, sn-0.6.0-memcpy-checks, 06.31, 3320, 6.31, 0.00, 0, 432 +cfrac, je, 06.72, 10136, 6.72, 0.00, 0, 270 +cfrac, scudo, 08.44, 4696, 8.44, 0.00, 0, 610 +cfrac, smi, 07.03, 4572, 7.02, 0.00, 0, 183 +leanN, mi, 25.89, 588932, 98.21, 1.16, 0, 165165 +leanN, sn-0.6.0-full-checks, 26.99, 655148, 105.01, 1.00, 0, 13196 +leanN, sn-0.6.0, 27.54, 544660, 111.08, 1.01, 0, 13274 +leanN, sn-0.5.3, 25.44, 547264, 96.18, 0.87, 0, 3596 +leanN, sn-0.6.0-memcpy-checks, 26.73, 542128, 106.45, 0.90, 0, 11923 +leanN, je, 26.89, 529516, 103.02, 1.26, 0, 157578 +leanN, scudo, 33.36, 589768, 121.28, 2.00, 0, 601688 +leanN, smi, 26.96, 620428, 96.80, 1.63, 0, 326027 +sed, mi, 01.73, 330532, 1.65, 0.08, 0, 405 +sed, sn-0.6.0-full-checks, 01.73, 349380, 1.65, 0.08, 0, 1545 +sed, sn-0.6.0, 01.72, 342796, 1.60, 0.11, 0, 1474 +sed, sn-0.5.3, 01.72, 310392, 1.65, 0.07, 0, 683 +sed, sn-0.6.0-memcpy-checks, 01.72, 342820, 1.65, 0.07, 0, 1451 +sed, je, 01.73, 293900, 1.64, 0.08, 0, 8280 +sed, scudo, 01.81, 245412, 1.70, 0.10, 0, 60796 +sed, smi, 01.82, 316752, 1.69, 0.12, 0, 34265 +barnes, mi, 02.85, 66912, 2.84, 0.00, 0, 2466 +barnes, sn-0.6.0-full-checks, 02.87, 65740, 2.85, 0.02, 0, 2853 +barnes, sn-0.6.0, 02.86, 65672, 2.84, 0.02, 0, 2839 +barnes, sn-0.5.3, 02.86, 69948, 2.84, 0.01, 0, 2523 +barnes, sn-0.6.0-memcpy-checks, 02.86, 65628, 2.84, 0.01, 0, 2837 +barnes, je, 02.86, 76616, 2.84, 0.01, 0, 2548 +barnes, scudo, 02.92, 61692, 2.91, 0.01, 0, 4279 +barnes, smi, 02.95, 66820, 2.93, 0.02, 0, 2655 +espresso, mi, 05.17, 8276, 5.14, 0.02, 0, 177 +espresso, sn-0.6.0-full-checks, 05.27, 12648, 5.20, 0.06, 0, 740 +espresso, sn-0.6.0, 05.11, 6232, 5.09, 0.02, 0, 654 +espresso, sn-0.5.3, 05.11, 10280, 5.09, 0.02, 0, 406 +espresso, sn-0.6.0-memcpy-checks, 05.15, 6216, 5.10, 0.04, 0, 656 +espresso, je, 05.55, 9940, 5.52, 0.02, 0, 388 +espresso, scudo, 06.08, 4876, 6.03, 0.05, 0, 635 +espresso, smi, 05.49, 6684, 5.45, 0.03, 0, 285 +z3, mi, 01.19, 71344, 1.16, 0.02, 0, 460 +z3, sn-0.6.0-full-checks, 01.17, 70140, 1.14, 0.02, 0, 761 +z3, sn-0.6.0, 01.17, 66068, 1.15, 0.01, 0, 741 +z3, sn-0.5.3, 01.19, 73804, 1.17, 0.02, 0, 564 +z3, sn-0.6.0-memcpy-checks, 01.17, 66232, 1.15, 0.01, 0, 741 +z3, je, 01.18, 67784, 1.16, 0.02, 0, 2757 +z3, scudo, 01.26, 56312, 1.24, 0.01, 0, 8163 +z3, smi, 01.23, 66204, 1.21, 0.01, 0, 3834 +gs, mi, 01.16, 57488, 1.12, 0.04, 0, 1652 +gs, sn-0.6.0-full-checks, 01.18, 54004, 1.16, 0.02, 0, 1961 +gs, sn-0.6.0, 01.18, 48352, 1.14, 0.03, 0, 2001 +gs, sn-0.5.3, 01.16, 56260, 1.11, 0.04, 0, 1875 +gs, sn-0.6.0-memcpy-checks, 01.18, 48336, 1.15, 0.02, 0, 2000 +gs, je, 01.19, 55668, 1.15, 0.03, 0, 3715 +gs, scudo, 01.20, 41428, 1.15, 0.05, 0, 17159 +gs, smi, 01.20, 56748, 1.18, 0.01, 0, 4635 +redis, mi, 4.675, 35124, 1.81, 0.54, 0, 7996 +redis, sn-0.6.0-full-checks, 4.357, 33356, 1.85, 0.34, 0, 8977 +redis, sn-0.6.0, 4.199, 30520, 1.74, 0.36, 0, 8579 +redis, sn-0.5.3, 4.254, 37024, 1.80, 0.33, 0, 7116 +redis, sn-0.6.0-memcpy-checks, 4.236, 30356, 1.68, 0.45, 0, 8560 +redis, je, 5.026, 36816, 2.20, 0.32, 0, 6771 +redis, scudo, 5.156, 38016, 2.25, 0.34, 0, 9884 +redis, smi, 4.564, 35472, 1.90, 0.40, 0, 8051 +cfrac, mi, 06.32, 4460, 6.31, 0.00, 0, 179 +cfrac, sn-0.6.0-full-checks, 06.61, 3584, 6.61, 0.00, 0, 495 +cfrac, sn-0.6.0, 06.28, 3300, 6.28, 0.00, 0, 433 +cfrac, sn-0.5.3, 06.38, 8424, 6.37, 0.00, 0, 446 +cfrac, sn-0.6.0-memcpy-checks, 06.24, 3336, 6.24, 0.00, 0, 434 +cfrac, je, 06.62, 10056, 6.62, 0.00, 0, 271 +cfrac, scudo, 08.40, 4592, 8.39, 0.00, 0, 611 +cfrac, smi, 07.04, 4596, 7.03, 0.00, 0, 182 +leanN, mi, 26.06, 592944, 99.54, 1.21, 0, 241980 +leanN, sn-0.6.0-full-checks, 26.66, 675748, 101.94, 1.01, 0, 13980 +leanN, sn-0.6.0, 25.55, 535144, 99.90, 0.83, 0, 12792 +leanN, sn-0.5.3, 25.91, 536216, 98.51, 0.89, 0, 3475 +leanN, sn-0.6.0-memcpy-checks, 25.17, 533080, 97.46, 0.95, 0, 13412 +leanN, je, 26.95, 514740, 103.52, 1.21, 0, 204418 +leanN, scudo, 32.10, 628432, 114.89, 1.64, 0, 568793 +leanN, smi, 27.59, 635276, 100.36, 1.84, 2, 343670 +sed, mi, 01.72, 326452, 1.63, 0.09, 0, 404 +sed, sn-0.6.0-full-checks, 01.73, 347264, 1.64, 0.09, 0, 1571 +sed, sn-0.6.0, 01.73, 342908, 1.63, 0.09, 0, 1480 +sed, sn-0.5.3, 01.71, 310356, 1.63, 0.08, 0, 684 +sed, sn-0.6.0-memcpy-checks, 01.72, 342824, 1.62, 0.09, 0, 1453 +sed, je, 01.73, 295488, 1.67, 0.06, 0, 9192 +sed, scudo, 01.81, 245284, 1.67, 0.13, 0, 60797 +sed, smi, 01.80, 316116, 1.69, 0.11, 0, 34107 +barnes, mi, 02.85, 66800, 2.84, 0.01, 0, 2464 +barnes, sn-0.6.0-full-checks, 02.87, 65632, 2.86, 0.01, 0, 2853 +barnes, sn-0.6.0, 02.86, 65552, 2.83, 0.02, 0, 2838 +barnes, sn-0.5.3, 02.84, 70108, 2.83, 0.00, 0, 2520 +barnes, sn-0.6.0-memcpy-checks, 02.86, 65608, 2.82, 0.03, 0, 2834 +barnes, je, 02.85, 76636, 2.83, 0.02, 0, 2547 +barnes, scudo, 02.89, 64112, 2.86, 0.02, 0, 3516 +barnes, smi, 02.95, 66948, 2.93, 0.01, 0, 2662 +espresso, mi, 05.16, 8228, 5.13, 0.02, 0, 176 +espresso, sn-0.6.0-full-checks, 05.24, 12608, 5.22, 0.02, 0, 748 +espresso, sn-0.6.0, 05.11, 6328, 5.07, 0.03, 0, 661 +espresso, sn-0.5.3, 05.14, 10392, 5.10, 0.03, 0, 414 +espresso, sn-0.6.0-memcpy-checks, 05.14, 6412, 5.11, 0.02, 0, 659 +espresso, je, 05.53, 11876, 5.49, 0.04, 0, 334 +espresso, scudo, 06.10, 5000, 6.07, 0.03, 0, 648 +espresso, smi, 05.48, 6748, 5.46, 0.02, 0, 289 +z3, mi, 01.18, 71260, 1.16, 0.02, 0, 458 +z3, sn-0.6.0-full-checks, 01.18, 72220, 1.17, 0.01, 0, 772 +z3, sn-0.6.0, 01.17, 66052, 1.17, 0.00, 0, 734 +z3, sn-0.5.3, 01.16, 73748, 1.15, 0.01, 0, 566 +z3, sn-0.6.0-memcpy-checks, 01.17, 66104, 1.15, 0.01, 0, 738 +z3, je, 01.17, 65920, 1.16, 0.01, 0, 2771 +z3, scudo, 01.27, 55960, 1.24, 0.02, 0, 8678 +z3, smi, 01.23, 65724, 1.21, 0.02, 0, 3728 +gs, mi, 01.15, 57376, 1.12, 0.03, 0, 1655 +gs, sn-0.6.0-full-checks, 01.19, 56208, 1.15, 0.03, 0, 1961 +gs, sn-0.6.0, 01.17, 48216, 1.14, 0.01, 0, 1999 +gs, sn-0.5.3, 01.16, 56052, 1.15, 0.01, 0, 1871 +gs, sn-0.6.0-memcpy-checks, 01.17, 48108, 1.13, 0.04, 0, 1997 +gs, je, 01.18, 53792, 1.16, 0.01, 0, 3727 +gs, scudo, 01.20, 41180, 1.17, 0.03, 0, 17107 +gs, smi, 01.18, 56612, 1.15, 0.03, 0, 4610 +redis, mi, 4.314, 35176, 1.85, 0.31, 0, 8029 +redis, sn-0.6.0-full-checks, 4.369, 33168, 1.79, 0.40, 0, 8840 +redis, sn-0.6.0, 4.248, 30392, 1.74, 0.40, 0, 8556 +redis, sn-0.5.3, 4.271, 37064, 1.74, 0.41, 0, 6995 +redis, sn-0.6.0-memcpy-checks, 4.265, 30460, 1.72, 0.42, 0, 8543 +redis, je, 4.991, 36876, 2.10, 0.40, 0, 6772 +redis, scudo, 5.188, 38000, 2.16, 0.45, 0, 9878 +redis, smi, 4.662, 35452, 2.03, 0.31, 0, 8040 +cfrac, mi, 06.52, 4588, 6.52, 0.00, 0, 185 +cfrac, sn-0.6.0-full-checks, 06.76, 3580, 6.76, 0.00, 0, 497 +cfrac, sn-0.6.0, 06.39, 3272, 6.39, 0.00, 0, 429 +cfrac, sn-0.5.3, 06.46, 8348, 6.45, 0.00, 0, 445 +cfrac, sn-0.6.0-memcpy-checks, 06.28, 3336, 6.27, 0.00, 0, 432 +cfrac, je, 06.63, 10056, 6.63, 0.00, 0, 271 +cfrac, scudo, 08.39, 4564, 8.39, 0.00, 0, 608 +cfrac, smi, 07.07, 4604, 7.07, 0.00, 0, 182 +leanN, mi, 25.77, 579000, 97.39, 1.21, 0, 226057 +leanN, sn-0.6.0-full-checks, 26.67, 671232, 102.70, 1.00, 0, 13239 +leanN, sn-0.6.0, 26.24, 529888, 103.74, 0.90, 0, 10606 +leanN, sn-0.5.3, 25.31, 555048, 95.87, 0.90, 0, 3509 +leanN, sn-0.6.0-memcpy-checks, 25.95, 534752, 101.70, 0.95, 0, 11941 +leanN, je, 25.51, 520756, 95.50, 1.06, 0, 164558 +leanN, scudo, 32.49, 589876, 116.58, 1.64, 0, 607090 +leanN, smi, 27.65, 627320, 100.04, 2.02, 0, 363186 +sed, mi, 01.73, 326428, 1.67, 0.06, 0, 403 +sed, sn-0.6.0-full-checks, 01.75, 351632, 1.63, 0.11, 0, 1713 +sed, sn-0.6.0, 01.73, 342732, 1.66, 0.06, 0, 1477 +sed, sn-0.5.3, 01.72, 310300, 1.65, 0.07, 0, 685 +sed, sn-0.6.0-memcpy-checks, 01.73, 342804, 1.64, 0.08, 0, 1476 +sed, je, 01.73, 295712, 1.65, 0.08, 0, 9184 +sed, scudo, 01.82, 245648, 1.69, 0.11, 0, 60799 +sed, smi, 01.82, 316576, 1.71, 0.11, 0, 34234 +barnes, mi, 02.84, 66972, 2.83, 0.01, 0, 2465 +barnes, sn-0.6.0-full-checks, 02.88, 65700, 2.86, 0.02, 0, 2854 +barnes, sn-0.6.0, 02.86, 65660, 2.85, 0.01, 0, 2839 +barnes, sn-0.5.3, 02.94, 70008, 2.93, 0.00, 0, 2522 +barnes, sn-0.6.0-memcpy-checks, 02.86, 65604, 2.84, 0.01, 0, 2841 +barnes, je, 02.86, 76672, 2.84, 0.01, 0, 2550 +barnes, scudo, 02.90, 64096, 2.88, 0.01, 0, 3531 +barnes, smi, 02.92, 66776, 2.90, 0.01, 0, 2661 +espresso, mi, 05.19, 8220, 5.17, 0.02, 0, 172 +espresso, sn-0.6.0-full-checks, 05.33, 12760, 5.29, 0.04, 0, 726 +espresso, sn-0.6.0, 05.21, 6272, 5.18, 0.03, 0, 660 +espresso, sn-0.5.3, 05.15, 10252, 5.13, 0.01, 0, 413 +espresso, sn-0.6.0-memcpy-checks, 05.14, 6440, 5.12, 0.02, 0, 661 +espresso, je, 05.43, 9908, 5.39, 0.04, 0, 332 +espresso, scudo, 06.03, 4884, 6.00, 0.02, 0, 642 +espresso, smi, 05.48, 6680, 5.44, 0.03, 0, 287 +z3, mi, 01.18, 71236, 1.17, 0.01, 0, 458 +z3, sn-0.6.0-full-checks, 01.17, 70276, 1.16, 0.00, 0, 767 +z3, sn-0.6.0, 01.17, 65988, 1.14, 0.02, 0, 733 +z3, sn-0.5.3, 01.17, 73808, 1.15, 0.01, 0, 571 +z3, sn-0.6.0-memcpy-checks, 01.16, 66264, 1.14, 0.02, 0, 740 +z3, je, 01.18, 65916, 1.17, 0.01, 0, 2774 +z3, scudo, 01.25, 56064, 1.23, 0.02, 0, 8664 +z3, smi, 01.23, 66032, 1.21, 0.01, 0, 3809 +gs, mi, 01.17, 57300, 1.15, 0.01, 0, 1658 +gs, sn-0.6.0-full-checks, 01.19, 54364, 1.15, 0.03, 0, 2014 +gs, sn-0.6.0, 01.16, 48528, 1.13, 0.03, 0, 1997 +gs, sn-0.5.3, 01.16, 56216, 1.14, 0.02, 0, 1872 +gs, sn-0.6.0-memcpy-checks, 01.18, 48464, 1.14, 0.04, 0, 1999 +gs, je, 01.18, 53628, 1.16, 0.02, 0, 3712 +gs, scudo, 01.20, 41440, 1.14, 0.06, 0, 17166 +gs, smi, 01.18, 57068, 1.15, 0.03, 0, 4675 +redis, mi, 4.288, 35248, 1.74, 0.42, 0, 8036 +redis, sn-0.6.0-full-checks, 4.343, 33512, 1.90, 0.29, 0, 9002 +redis, sn-0.6.0, 4.203, 30428, 1.78, 0.34, 0, 8564 +redis, sn-0.5.3, 4.224, 37128, 1.74, 0.38, 0, 7520 +redis, sn-0.6.0-memcpy-checks, 4.229, 30404, 1.71, 0.42, 0, 8564 +redis, je, 5.033, 36940, 2.16, 0.36, 0, 6761 +redis, scudo, 5.361, 37964, 2.31, 0.38, 0, 9853 +redis, smi, 4.665, 35436, 1.91, 0.43, 0, 8032 +cfrac, mi, 06.31, 4460, 6.30, 0.00, 0, 180 +cfrac, sn-0.6.0-full-checks, 06.60, 3596, 6.60, 0.00, 0, 507 +cfrac, sn-0.6.0, 06.23, 3276, 6.23, 0.00, 0, 431 +cfrac, sn-0.5.3, 06.30, 8292, 6.30, 0.00, 0, 445 +cfrac, sn-0.6.0-memcpy-checks, 06.24, 3336, 6.24, 0.00, 0, 430 +cfrac, je, 06.67, 9996, 6.66, 0.00, 0, 270 +cfrac, scudo, 08.64, 4756, 8.63, 0.00, 0, 613 +cfrac, smi, 07.18, 4572, 7.18, 0.00, 0, 184 +leanN, mi, 26.23, 583088, 100.11, 1.16, 2, 161724 +leanN, sn-0.6.0-full-checks, 26.23, 673484, 100.36, 0.96, 0, 12775 +leanN, sn-0.6.0, 27.12, 530604, 108.73, 1.00, 0, 12559 +leanN, sn-0.5.3, 27.30, 526460, 106.32, 0.96, 0, 2879 +leanN, sn-0.6.0-memcpy-checks, 26.90, 518652, 106.80, 0.83, 0, 11945 +leanN, je, 26.70, 513676, 103.21, 0.97, 0, 78470 +leanN, scudo, 33.24, 625584, 121.33, 1.88, 0, 656737 +leanN, smi, 27.25, 617944, 97.77, 2.02, 0, 449513 +sed, mi, 01.75, 324396, 1.69, 0.05, 0, 402 +sed, sn-0.6.0-full-checks, 01.74, 349496, 1.65, 0.09, 0, 1693 +sed, sn-0.6.0, 01.73, 342848, 1.64, 0.09, 0, 1474 +sed, sn-0.5.3, 01.72, 310416, 1.64, 0.07, 0, 686 +sed, sn-0.6.0-memcpy-checks, 01.74, 342748, 1.66, 0.07, 0, 1482 +sed, je, 01.74, 295408, 1.65, 0.09, 0, 9194 +sed, scudo, 01.81, 245432, 1.69, 0.12, 0, 60798 +sed, smi, 01.80, 316564, 1.69, 0.11, 0, 34217 +barnes, mi, 02.90, 66824, 2.87, 0.02, 0, 2463 +barnes, sn-0.6.0-full-checks, 02.94, 65752, 2.90, 0.03, 0, 2856 +barnes, sn-0.6.0, 02.93, 65796, 2.92, 0.01, 0, 2844 +barnes, sn-0.5.3, 02.87, 70156, 2.85, 0.02, 0, 2519 +barnes, sn-0.6.0-memcpy-checks, 02.85, 65648, 2.82, 0.02, 0, 2844 +barnes, je, 02.85, 76616, 2.83, 0.01, 0, 2551 +barnes, scudo, 02.92, 64456, 2.89, 0.03, 0, 3607 +barnes, smi, 02.91, 66984, 2.89, 0.02, 0, 2659 +espresso, mi, 05.17, 8224, 5.15, 0.01, 0, 176 +espresso, sn-0.6.0-full-checks, 05.25, 12740, 5.22, 0.02, 0, 734 +espresso, sn-0.6.0, 05.11, 6256, 5.11, 0.00, 0, 658 +espresso, sn-0.5.3, 05.13, 10420, 5.09, 0.03, 0, 414 +espresso, sn-0.6.0-memcpy-checks, 05.12, 6548, 5.08, 0.04, 0, 663 +espresso, je, 05.44, 10148, 5.42, 0.02, 0, 307 +espresso, scudo, 06.05, 4992, 6.03, 0.02, 0, 642 +espresso, smi, 05.45, 6684, 5.43, 0.02, 0, 289 +z3, mi, 01.18, 71336, 1.16, 0.02, 0, 456 +z3, sn-0.6.0-full-checks, 01.18, 70424, 1.16, 0.01, 0, 759 +z3, sn-0.6.0, 01.16, 66164, 1.15, 0.01, 0, 739 +z3, sn-0.5.3, 01.16, 73580, 1.14, 0.02, 0, 565 +z3, sn-0.6.0-memcpy-checks, 01.17, 66016, 1.15, 0.01, 0, 742 +z3, je, 01.19, 66028, 1.16, 0.02, 0, 2773 +z3, scudo, 01.24, 56172, 1.22, 0.01, 0, 8145 +z3, smi, 01.23, 65560, 1.20, 0.03, 0, 3753 +gs, mi, 01.15, 57368, 1.13, 0.02, 0, 1657 +gs, sn-0.6.0-full-checks, 01.19, 53812, 1.17, 0.01, 0, 1939 +gs, sn-0.6.0, 01.17, 48532, 1.15, 0.01, 0, 1993 +gs, sn-0.5.3, 01.15, 56056, 1.12, 0.02, 0, 1876 +gs, sn-0.6.0-memcpy-checks, 01.17, 48452, 1.13, 0.03, 0, 1998 +gs, je, 01.18, 53568, 1.13, 0.05, 0, 3723 +gs, scudo, 01.20, 41540, 1.16, 0.03, 0, 17162 +gs, smi, 01.19, 57140, 1.16, 0.02, 0, 4643 +redis, mi, 4.308, 35204, 1.77, 0.39, 0, 8019 +redis, sn-0.6.0-full-checks, 4.392, 33984, 1.86, 0.35, 0, 9164 +redis, sn-0.6.0, 4.196, 30348, 1.74, 0.37, 0, 8562 +redis, sn-0.5.3, 4.389, 37100, 1.73, 0.47, 0, 7100 +redis, sn-0.6.0-memcpy-checks, 4.328, 30368, 1.83, 0.34, 0, 8531 +redis, je, 4.989, 36816, 2.17, 0.33, 0, 6756 +redis, scudo, 5.182, 37880, 2.22, 0.39, 0, 9882 +redis, smi, 4.593, 35444, 1.88, 0.43, 0, 8057 +cfrac, mi, 06.29, 4480, 6.29, 0.00, 0, 183 +cfrac, sn-0.6.0-full-checks, 06.61, 3664, 6.61, 0.00, 0, 512 +cfrac, sn-0.6.0, 06.25, 3328, 6.25, 0.00, 0, 431 +cfrac, sn-0.5.3, 06.36, 8348, 6.36, 0.00, 0, 445 +cfrac, sn-0.6.0-memcpy-checks, 06.25, 3340, 6.25, 0.00, 0, 430 +cfrac, je, 06.63, 10000, 6.62, 0.00, 0, 270 +cfrac, scudo, 08.40, 4564, 8.39, 0.00, 0, 610 +cfrac, smi, 07.00, 4604, 7.00, 0.00, 0, 183 +leanN, mi, 26.14, 585036, 100.89, 1.35, 0, 185556 +leanN, sn-0.6.0-full-checks, 27.06, 681168, 105.03, 1.04, 0, 11898 +leanN, sn-0.6.0, 26.39, 518364, 104.78, 0.84, 0, 11801 +leanN, sn-0.5.3, 26.22, 540668, 102.18, 0.90, 0, 3315 +leanN, sn-0.6.0-memcpy-checks, 25.73, 544584, 100.10, 0.90, 0, 13151 +leanN, je, 26.63, 489512, 103.36, 1.12, 0, 178461 +leanN, scudo, 32.24, 586964, 116.42, 1.71, 4, 656609 +leanN, smi, 27.36, 612228, 98.39, 1.86, 3, 411598 +sed, mi, 01.75, 326444, 1.66, 0.08, 0, 398 +sed, sn-0.6.0-full-checks, 01.79, 347472, 1.72, 0.06, 0, 1697 +sed, sn-0.6.0, 01.76, 342848, 1.68, 0.07, 0, 1478 +sed, sn-0.5.3, 01.78, 310088, 1.68, 0.09, 0, 679 +sed, sn-0.6.0-memcpy-checks, 01.74, 342872, 1.63, 0.10, 0, 1479 +sed, je, 01.76, 295512, 1.69, 0.07, 0, 9198 +sed, scudo, 01.83, 245376, 1.73, 0.09, 0, 60796 +sed, smi, 01.81, 315884, 1.69, 0.11, 0, 34061 +barnes, mi, 02.88, 66716, 2.86, 0.02, 0, 2461 +barnes, sn-0.6.0-full-checks, 02.91, 65776, 2.88, 0.02, 0, 2856 +barnes, sn-0.6.0, 02.86, 65636, 2.85, 0.01, 0, 2847 +barnes, sn-0.5.3, 02.88, 69948, 2.85, 0.02, 0, 2519 +barnes, sn-0.6.0-memcpy-checks, 02.86, 65648, 2.84, 0.02, 0, 2848 +barnes, je, 02.89, 76676, 2.87, 0.01, 0, 2546 +barnes, scudo, 02.91, 61900, 2.89, 0.01, 0, 3644 +barnes, smi, 02.95, 66896, 2.94, 0.00, 0, 2657 +espresso, mi, 05.20, 8260, 5.18, 0.01, 0, 174 +espresso, sn-0.6.0-full-checks, 05.28, 12608, 5.27, 0.01, 0, 736 +espresso, sn-0.6.0, 05.12, 6268, 5.09, 0.03, 0, 659 +espresso, sn-0.5.3, 05.16, 10328, 5.14, 0.01, 0, 413 +espresso, sn-0.6.0-memcpy-checks, 05.12, 6428, 5.10, 0.02, 0, 657 +espresso, je, 05.49, 9904, 5.47, 0.01, 0, 346 +espresso, scudo, 06.05, 4884, 6.02, 0.02, 0, 632 +espresso, smi, 05.50, 6684, 5.46, 0.03, 0, 284 +z3, mi, 01.19, 71380, 1.17, 0.02, 0, 459 +z3, sn-0.6.0-full-checks, 01.18, 70388, 1.15, 0.02, 0, 768 +z3, sn-0.6.0, 01.18, 66116, 1.17, 0.00, 0, 738 +z3, sn-0.5.3, 01.16, 73572, 1.14, 0.01, 0, 562 +z3, sn-0.6.0-memcpy-checks, 01.17, 66184, 1.15, 0.01, 0, 741 +z3, je, 01.19, 65840, 1.16, 0.02, 0, 2772 +z3, scudo, 01.26, 56180, 1.25, 0.01, 0, 8670 +z3, smi, 01.25, 65808, 1.23, 0.01, 0, 3798 +gs, mi, 01.16, 57188, 1.14, 0.02, 0, 1655 +gs, sn-0.6.0-full-checks, 01.18, 54236, 1.16, 0.02, 0, 1958 +gs, sn-0.6.0, 01.18, 48488, 1.16, 0.02, 0, 2000 +gs, sn-0.5.3, 01.16, 56172, 1.13, 0.03, 0, 1872 +gs, sn-0.6.0-memcpy-checks, 01.19, 48188, 1.15, 0.04, 0, 2000 +gs, je, 01.20, 53220, 1.15, 0.04, 0, 3711 +gs, scudo, 01.21, 41180, 1.15, 0.06, 0, 17153 +gs, smi, 01.20, 57036, 1.16, 0.04, 0, 4612 +redis, mi, 4.340, 35184, 1.79, 0.39, 0, 8010 +redis, sn-0.6.0-full-checks, 4.403, 33604, 1.88, 0.33, 0, 9014 +redis, sn-0.6.0, 4.320, 30360, 1.80, 0.37, 0, 8539 +redis, sn-0.5.3, 4.348, 36980, 1.82, 0.36, 0, 7483 +redis, sn-0.6.0-memcpy-checks, 4.306, 30440, 1.81, 0.35, 0, 8542 +redis, je, 5.087, 36972, 2.19, 0.36, 0, 6784 +redis, scudo, 5.234, 37924, 2.22, 0.41, 0, 9880 +redis, smi, 4.690, 35696, 1.92, 0.44, 0, 8085 +cfrac, mi, 06.33, 4496, 6.33, 0.00, 0, 186 +cfrac, sn-0.6.0-full-checks, 06.62, 3576, 6.62, 0.00, 0, 495 +cfrac, sn-0.6.0, 06.27, 3336, 6.27, 0.00, 0, 435 +cfrac, sn-0.5.3, 06.32, 8408, 6.32, 0.00, 0, 448 +cfrac, sn-0.6.0-memcpy-checks, 06.27, 3376, 6.27, 0.00, 0, 434 +cfrac, je, 06.66, 10104, 6.65, 0.00, 0, 271 +cfrac, scudo, 08.43, 4656, 8.42, 0.00, 0, 611 +cfrac, smi, 07.06, 4636, 7.06, 0.00, 0, 183 +leanN, mi, 26.72, 595404, 103.20, 1.25, 3, 232568 +leanN, sn-0.6.0-full-checks, 26.55, 655872, 101.70, 1.10, 0, 13236 +leanN, sn-0.6.0, 25.36, 534340, 98.64, 0.81, 0, 12637 +leanN, sn-0.5.3, 25.75, 549084, 98.06, 0.79, 0, 3423 +leanN, sn-0.6.0-memcpy-checks, 26.59, 534420, 106.42, 0.76, 0, 12479 +leanN, je, 26.02, 524788, 98.90, 1.07, 0, 140108 +leanN, scudo, 32.32, 617108, 115.04, 1.95, 3, 560579 +leanN, smi, 27.69, 626448, 100.56, 2.05, 0, 453730 +sed, mi, 01.75, 326400, 1.67, 0.07, 0, 404 +sed, sn-0.6.0-full-checks, 01.79, 347480, 1.70, 0.09, 0, 1697 +sed, sn-0.6.0, 01.76, 342736, 1.66, 0.09, 0, 1478 +sed, sn-0.5.3, 01.76, 310376, 1.70, 0.05, 0, 681 +sed, sn-0.6.0-memcpy-checks, 01.79, 342792, 1.69, 0.10, 0, 1477 +sed, je, 01.75, 294172, 1.65, 0.09, 0, 8283 +sed, scudo, 01.81, 245416, 1.69, 0.11, 0, 60794 +sed, smi, 01.82, 312476, 1.68, 0.13, 0, 33703 +barnes, mi, 02.86, 66764, 2.85, 0.01, 0, 2461 +barnes, sn-0.6.0-full-checks, 02.87, 65632, 2.85, 0.02, 0, 2855 +barnes, sn-0.6.0, 02.87, 65668, 2.84, 0.03, 0, 2837 +barnes, sn-0.5.3, 02.90, 70064, 2.87, 0.02, 0, 2525 +barnes, sn-0.6.0-memcpy-checks, 02.88, 65616, 2.87, 0.01, 0, 2841 +barnes, je, 02.85, 76744, 2.85, 0.00, 0, 2549 +barnes, scudo, 02.90, 63144, 2.88, 0.02, 0, 3502 +barnes, smi, 02.94, 66808, 2.90, 0.03, 0, 2660 +espresso, mi, 05.19, 8308, 5.17, 0.02, 0, 176 +espresso, sn-0.6.0-full-checks, 05.28, 12608, 5.25, 0.02, 0, 758 +espresso, sn-0.6.0, 05.13, 6236, 5.10, 0.02, 0, 656 +espresso, sn-0.5.3, 05.14, 10240, 5.10, 0.04, 0, 411 +espresso, sn-0.6.0-memcpy-checks, 05.12, 6240, 5.10, 0.02, 0, 655 +espresso, je, 05.45, 9860, 5.45, 0.00, 0, 336 +espresso, scudo, 06.02, 4804, 6.02, 0.00, 0, 640 +espresso, smi, 05.48, 6756, 5.45, 0.03, 0, 290 +z3, mi, 01.20, 71272, 1.18, 0.01, 0, 456 +z3, sn-0.6.0-full-checks, 01.19, 70156, 1.17, 0.01, 0, 765 +z3, sn-0.6.0, 01.17, 66192, 1.16, 0.00, 0, 737 +z3, sn-0.5.3, 01.17, 73556, 1.15, 0.01, 0, 562 +z3, sn-0.6.0-memcpy-checks, 01.18, 66120, 1.16, 0.02, 0, 731 +z3, je, 01.18, 65900, 1.17, 0.01, 0, 2771 +z3, scudo, 01.27, 56036, 1.26, 0.01, 0, 8678 +z3, smi, 01.23, 65884, 1.20, 0.02, 0, 3786 +gs, mi, 01.17, 57512, 1.15, 0.01, 0, 1655 +gs, sn-0.6.0-full-checks, 01.19, 54220, 1.15, 0.04, 0, 1945 +gs, sn-0.6.0, 01.17, 48364, 1.16, 0.01, 0, 1998 +gs, sn-0.5.3, 01.16, 56032, 1.14, 0.02, 0, 1876 +gs, sn-0.6.0-memcpy-checks, 01.19, 47984, 1.15, 0.03, 0, 1996 +gs, je, 01.20, 53552, 1.17, 0.03, 0, 3733 +gs, scudo, 01.22, 41524, 1.15, 0.06, 0, 17151 +gs, smi, 01.22, 56648, 1.18, 0.03, 0, 4597 +redis, mi, 4.380, 35048, 1.87, 0.33, 0, 8004 +redis, sn-0.6.0-full-checks, 4.406, 33720, 1.87, 0.35, 0, 9048 +redis, sn-0.6.0, 4.182, 30384, 1.72, 0.39, 0, 8574 +redis, sn-0.5.3, 4.314, 36988, 1.81, 0.36, 0, 7349 +redis, sn-0.6.0-memcpy-checks, 4.492, 30384, 1.88, 0.38, 0, 8479 +redis, je, 5.061, 36840, 2.14, 0.40, 0, 6776 +redis, scudo, 5.222, 37976, 2.24, 0.39, 0, 9876 +redis, smi, 4.618, 35384, 1.88, 0.44, 0, 8039 +cfrac, mi, 06.36, 4496, 6.35, 0.00, 0, 183 +cfrac, sn-0.6.0-full-checks, 06.62, 3672, 6.62, 0.00, 0, 512 +cfrac, sn-0.6.0, 06.27, 3332, 6.27, 0.00, 0, 436 +cfrac, sn-0.5.3, 06.34, 8424, 6.33, 0.00, 0, 446 +cfrac, sn-0.6.0-memcpy-checks, 06.28, 3376, 6.28, 0.00, 0, 434 +cfrac, je, 06.64, 10108, 6.64, 0.00, 0, 271 +cfrac, scudo, 08.56, 4644, 8.56, 0.00, 0, 610 +cfrac, smi, 07.06, 4496, 7.06, 0.00, 0, 184 +leanN, mi, 25.91, 597148, 98.60, 1.04, 0, 190255 +leanN, sn-0.6.0-full-checks, 27.77, 662424, 108.83, 0.93, 0, 13224 +leanN, sn-0.6.0, 24.73, 534340, 94.63, 0.75, 0, 13335 +leanN, sn-0.5.3, 26.49, 539092, 101.33, 0.99, 3, 3592 +leanN, sn-0.6.0-memcpy-checks, 26.56, 530728, 106.76, 0.88, 0, 12442 +leanN, je, 27.05, 504264, 103.22, 1.17, 0, 156329 +leanN, scudo, 32.94, 612364, 117.66, 2.10, 5, 805633 +leanN, smi, 27.56, 625856, 99.23, 1.87, 1, 381154 +sed, mi, 01.74, 326504, 1.66, 0.08, 0, 403 +sed, sn-0.6.0-full-checks, 01.74, 349428, 1.65, 0.08, 0, 1596 +sed, sn-0.6.0, 01.73, 342912, 1.64, 0.08, 0, 1475 +sed, sn-0.5.3, 01.72, 310300, 1.65, 0.07, 0, 679 +sed, sn-0.6.0-memcpy-checks, 01.73, 342796, 1.64, 0.09, 0, 1474 +sed, je, 01.74, 296120, 1.65, 0.09, 0, 8271 +sed, scudo, 01.82, 245408, 1.70, 0.12, 0, 60794 +sed, smi, 01.82, 312548, 1.70, 0.11, 0, 33757 +barnes, mi, 02.86, 66808, 2.84, 0.01, 0, 2465 +barnes, sn-0.6.0-full-checks, 02.86, 65732, 2.84, 0.02, 0, 2854 +barnes, sn-0.6.0, 02.86, 65640, 2.84, 0.02, 0, 2848 +barnes, sn-0.5.3, 02.86, 70068, 2.85, 0.01, 0, 2523 +barnes, sn-0.6.0-memcpy-checks, 02.92, 65608, 2.90, 0.02, 0, 2837 +barnes, je, 02.91, 76640, 2.88, 0.02, 0, 2547 +barnes, scudo, 02.92, 63716, 2.91, 0.00, 0, 3472 +barnes, smi, 02.93, 66852, 2.89, 0.03, 0, 2658 +espresso, mi, 05.19, 8316, 5.15, 0.03, 0, 172 +espresso, sn-0.6.0-full-checks, 05.27, 12808, 5.26, 0.00, 0, 733 +espresso, sn-0.6.0, 05.14, 6296, 5.10, 0.03, 0, 658 +espresso, sn-0.5.3, 05.15, 10352, 5.14, 0.01, 0, 415 +espresso, sn-0.6.0-memcpy-checks, 05.13, 6548, 5.11, 0.01, 0, 660 +espresso, je, 05.47, 9932, 5.44, 0.02, 0, 355 +espresso, scudo, 06.13, 4804, 6.12, 0.01, 0, 633 +espresso, smi, 05.50, 6692, 5.46, 0.04, 0, 288 +z3, mi, 01.20, 71152, 1.18, 0.01, 0, 456 +z3, sn-0.6.0-full-checks, 01.17, 70276, 1.15, 0.02, 0, 767 +z3, sn-0.6.0, 01.17, 66000, 1.14, 0.02, 0, 732 +z3, sn-0.5.3, 01.17, 73528, 1.15, 0.01, 0, 564 +z3, sn-0.6.0-memcpy-checks, 01.16, 65980, 1.16, 0.00, 0, 736 +z3, je, 01.19, 65772, 1.17, 0.01, 0, 2774 +z3, scudo, 01.26, 56280, 1.24, 0.01, 0, 8661 +z3, smi, 01.24, 65928, 1.22, 0.02, 0, 3817 +gs, mi, 01.17, 57516, 1.17, 0.00, 0, 1654 +gs, sn-0.6.0-full-checks, 01.19, 54336, 1.15, 0.03, 0, 1947 +gs, sn-0.6.0, 01.18, 48316, 1.14, 0.03, 0, 1996 +gs, sn-0.5.3, 01.17, 56208, 1.15, 0.02, 0, 1872 +gs, sn-0.6.0-memcpy-checks, 01.18, 48384, 1.16, 0.02, 0, 1997 +gs, je, 01.20, 53652, 1.18, 0.02, 0, 3718 +gs, scudo, 01.21, 41432, 1.18, 0.02, 0, 17155 +gs, smi, 01.19, 56816, 1.12, 0.06, 0, 4649 +redis, mi, 4.314, 35128, 1.80, 0.36, 0, 8009 +redis, sn-0.6.0-full-checks, 4.415, 33776, 1.86, 0.36, 0, 9147 +redis, sn-0.6.0, 4.239, 30492, 1.74, 0.39, 0, 8547 +redis, sn-0.5.3, 4.277, 36992, 1.79, 0.36, 0, 7345 +redis, sn-0.6.0-memcpy-checks, 4.320, 30388, 1.80, 0.37, 0, 8544 +redis, je, 5.188, 36876, 2.28, 0.32, 0, 6771 +redis, scudo, 5.231, 37984, 2.27, 0.36, 0, 9869 +redis, smi, 4.599, 35488, 1.92, 0.39, 0, 8062 +cfrac, mi, 06.39, 4496, 6.38, 0.00, 0, 182 +cfrac, sn-0.6.0-full-checks, 06.73, 3572, 6.73, 0.00, 0, 494 +cfrac, sn-0.6.0, 06.27, 3264, 6.26, 0.00, 0, 430 +cfrac, sn-0.5.3, 06.32, 8368, 6.32, 0.00, 0, 447 +cfrac, sn-0.6.0-memcpy-checks, 06.27, 3308, 6.27, 0.00, 0, 433 +cfrac, je, 06.68, 10028, 6.68, 0.00, 0, 271 +cfrac, scudo, 08.43, 4624, 8.43, 0.00, 0, 611 +cfrac, smi, 07.06, 4468, 7.06, 0.00, 0, 181 +leanN, mi, 25.73, 599228, 96.25, 1.11, 0, 222805 +leanN, sn-0.6.0-full-checks, 27.00, 657728, 104.05, 0.79, 0, 14663 +leanN, sn-0.6.0, 25.04, 538884, 96.63, 0.76, 0, 13205 +leanN, sn-0.5.3, 25.42, 542936, 96.17, 0.85, 0, 3418 +leanN, sn-0.6.0-memcpy-checks, 25.50, 536352, 99.48, 0.93, 0, 13333 +leanN, je, 26.47, 507228, 100.33, 0.96, 0, 126259 +leanN, scudo, 33.81, 608676, 124.38, 1.69, 0, 497025 +leanN, smi, 27.40, 612612, 100.31, 1.90, 0, 403445 +sed, mi, 01.74, 330548, 1.67, 0.07, 0, 406 +sed, sn-0.6.0-full-checks, 01.76, 347356, 1.66, 0.09, 0, 1590 +sed, sn-0.6.0, 01.73, 342812, 1.65, 0.08, 0, 1454 +sed, sn-0.5.3, 01.73, 310336, 1.66, 0.07, 0, 680 +sed, sn-0.6.0-memcpy-checks, 01.73, 342988, 1.64, 0.09, 0, 1481 +sed, je, 01.73, 295408, 1.63, 0.09, 0, 9187 +sed, scudo, 01.81, 245452, 1.70, 0.10, 0, 60797 +sed, smi, 01.81, 316720, 1.69, 0.11, 0, 34280 +barnes, mi, 02.85, 66840, 2.83, 0.01, 0, 2463 +barnes, sn-0.6.0-full-checks, 02.86, 65688, 2.84, 0.02, 0, 2855 +barnes, sn-0.6.0, 02.90, 65508, 2.89, 0.00, 0, 2839 +barnes, sn-0.5.3, 02.88, 70236, 2.87, 0.01, 0, 2521 +barnes, sn-0.6.0-memcpy-checks, 02.87, 65556, 2.85, 0.02, 0, 2842 +barnes, je, 02.87, 76584, 2.87, 0.00, 0, 2549 +barnes, scudo, 02.94, 61456, 2.92, 0.02, 0, 4264 +barnes, smi, 02.94, 66808, 2.92, 0.02, 0, 2657 +espresso, mi, 05.24, 8276, 5.20, 0.03, 0, 177 +espresso, sn-0.6.0-full-checks, 05.28, 12940, 5.26, 0.02, 0, 736 +espresso, sn-0.6.0, 05.13, 6432, 5.12, 0.01, 0, 659 +espresso, sn-0.5.3, 05.27, 10284, 5.25, 0.01, 0, 413 +espresso, sn-0.6.0-memcpy-checks, 05.26, 6440, 5.23, 0.02, 0, 659 +espresso, je, 05.47, 10276, 5.46, 0.01, 0, 295 +espresso, scudo, 06.03, 4980, 6.00, 0.03, 0, 639 +espresso, smi, 05.49, 6728, 5.47, 0.02, 0, 292 +z3, mi, 01.20, 71076, 1.18, 0.01, 0, 453 +z3, sn-0.6.0-full-checks, 01.18, 70316, 1.17, 0.01, 0, 764 +z3, sn-0.6.0, 01.17, 66128, 1.16, 0.01, 0, 734 +z3, sn-0.5.3, 01.16, 73644, 1.14, 0.02, 0, 564 +z3, sn-0.6.0-memcpy-checks, 01.18, 66176, 1.17, 0.01, 0, 738 +z3, je, 01.18, 65908, 1.17, 0.01, 0, 2770 +z3, scudo, 01.25, 56216, 1.23, 0.02, 0, 8662 +z3, smi, 01.23, 66256, 1.21, 0.02, 0, 3864 +gs, mi, 01.16, 57492, 1.13, 0.02, 0, 1655 +gs, sn-0.6.0-full-checks, 01.20, 54284, 1.17, 0.02, 0, 1957 +gs, sn-0.6.0, 01.18, 48284, 1.15, 0.02, 0, 1997 +gs, sn-0.5.3, 01.17, 55884, 1.13, 0.03, 0, 1869 +gs, sn-0.6.0-memcpy-checks, 01.18, 48476, 1.15, 0.02, 0, 2006 +gs, je, 01.19, 53308, 1.16, 0.02, 0, 3724 +gs, scudo, 01.20, 41576, 1.13, 0.06, 0, 17165 +gs, smi, 01.20, 57036, 1.17, 0.03, 0, 4615 +redis, mi, 4.314, 35236, 1.84, 0.33, 0, 8029 +redis, sn-0.6.0-full-checks, 4.645, 33244, 1.93, 0.41, 0, 8835 +redis, sn-0.6.0, 4.231, 30396, 1.68, 0.44, 0, 8561 +redis, sn-0.5.3, 4.289, 37040, 1.77, 0.38, 0, 6881 +redis, sn-0.6.0-memcpy-checks, 4.328, 30440, 1.79, 0.38, 0, 8532 +redis, je, 5.107, 36840, 2.19, 0.37, 0, 6749 +redis, scudo, 5.297, 37972, 2.29, 0.37, 0, 9852 +redis, smi, 4.717, 35384, 1.97, 0.40, 0, 8022 +cfrac, mi, 06.34, 4460, 6.34, 0.00, 0, 184 +cfrac, sn-0.6.0-full-checks, 06.62, 3600, 6.61, 0.00, 0, 494 +cfrac, sn-0.6.0, 06.26, 3312, 6.26, 0.00, 0, 432 +cfrac, sn-0.5.3, 06.36, 8408, 6.36, 0.00, 0, 447 +cfrac, sn-0.6.0-memcpy-checks, 06.27, 3336, 6.27, 0.00, 0, 429 +cfrac, je, 06.64, 10028, 6.64, 0.00, 0, 269 +cfrac, scudo, 08.45, 4648, 8.45, 0.00, 0, 611 +cfrac, smi, 07.02, 4488, 7.02, 0.00, 0, 177 +leanN, mi, 25.88, 585348, 97.10, 1.24, 4, 281872 +leanN, sn-0.6.0-full-checks, 26.70, 667524, 101.43, 0.87, 0, 12013 +leanN, sn-0.6.0, 24.98, 530168, 95.53, 0.83, 0, 12350 +leanN, sn-0.5.3, 25.63, 538804, 97.85, 0.82, 0, 3326 +leanN, sn-0.6.0-memcpy-checks, 25.23, 533980, 96.62, 0.83, 0, 12236 +leanN, je, 25.87, 521736, 96.91, 1.09, 0, 178770 +leanN, scudo, 32.13, 589304, 114.72, 1.66, 0, 541686 +leanN, smi, 27.63, 611432, 101.04, 1.79, 0, 402887 +sed, mi, 01.74, 324444, 1.63, 0.11, 0, 402 +sed, sn-0.6.0-full-checks, 01.74, 347576, 1.67, 0.07, 0, 1696 +sed, sn-0.6.0, 01.73, 342840, 1.63, 0.10, 0, 1475 +sed, sn-0.5.3, 01.74, 310300, 1.67, 0.06, 0, 679 +sed, sn-0.6.0-memcpy-checks, 01.73, 342752, 1.64, 0.09, 0, 1452 +sed, je, 01.74, 293788, 1.67, 0.07, 0, 8269 +sed, scudo, 01.83, 245484, 1.70, 0.12, 0, 60795 +sed, smi, 01.81, 316640, 1.71, 0.09, 0, 34248 +barnes, mi, 02.90, 66716, 2.88, 0.01, 0, 2461 +barnes, sn-0.6.0-full-checks, 02.87, 65668, 2.84, 0.03, 0, 2853 +barnes, sn-0.6.0, 02.86, 65600, 2.86, 0.00, 0, 2837 +barnes, sn-0.5.3, 02.85, 70236, 2.82, 0.02, 0, 2516 +barnes, sn-0.6.0-memcpy-checks, 02.87, 65572, 2.84, 0.02, 0, 2839 +barnes, je, 02.88, 76748, 2.86, 0.01, 0, 2547 +barnes, scudo, 02.95, 61908, 2.93, 0.01, 0, 4305 +barnes, smi, 02.94, 66940, 2.91, 0.02, 0, 2660 +espresso, mi, 05.18, 8312, 5.15, 0.03, 0, 174 +espresso, sn-0.6.0-full-checks, 05.28, 12644, 5.25, 0.02, 0, 736 +espresso, sn-0.6.0, 05.22, 6232, 5.19, 0.02, 0, 655 +espresso, sn-0.5.3, 05.26, 10352, 5.21, 0.04, 0, 414 +espresso, sn-0.6.0-memcpy-checks, 05.20, 6260, 5.17, 0.02, 0, 656 +espresso, je, 05.47, 9972, 5.45, 0.02, 0, 307 +espresso, scudo, 06.04, 4980, 5.99, 0.04, 0, 644 +espresso, smi, 05.49, 6640, 5.47, 0.02, 0, 286 +z3, mi, 01.20, 71316, 1.19, 0.00, 0, 460 +z3, sn-0.6.0-full-checks, 01.18, 72472, 1.16, 0.02, 0, 780 +z3, sn-0.6.0, 01.16, 66120, 1.14, 0.02, 0, 733 +z3, sn-0.5.3, 01.17, 73692, 1.16, 0.01, 0, 563 +z3, sn-0.6.0-memcpy-checks, 01.17, 66212, 1.15, 0.01, 0, 737 +z3, je, 01.19, 65768, 1.17, 0.01, 0, 2772 +z3, scudo, 01.25, 56092, 1.23, 0.02, 0, 8666 +z3, smi, 01.24, 65416, 1.21, 0.02, 0, 3713 +gs, mi, 01.18, 57324, 1.14, 0.03, 0, 1650 +gs, sn-0.6.0-full-checks, 01.18, 54196, 1.17, 0.01, 0, 1955 +gs, sn-0.6.0, 01.17, 48504, 1.15, 0.02, 0, 1996 +gs, sn-0.5.3, 01.17, 56216, 1.14, 0.02, 0, 1869 +gs, sn-0.6.0-memcpy-checks, 01.17, 48144, 1.15, 0.02, 0, 1997 +gs, je, 01.20, 53332, 1.17, 0.02, 0, 3721 +gs, scudo, 01.21, 41008, 1.17, 0.04, 0, 17108 +gs, smi, 01.19, 57232, 1.16, 0.03, 0, 4622 +redis, mi, 4.669, 35204, 1.98, 0.37, 0, 7916 +redis, sn-0.6.0-full-checks, 4.429, 33292, 1.83, 0.40, 0, 8884 +redis, sn-0.6.0, 4.189, 30384, 1.69, 0.41, 0, 8580 +redis, sn-0.5.3, 4.263, 36996, 1.75, 0.39, 0, 7261 +redis, sn-0.6.0-memcpy-checks, 4.289, 30472, 1.78, 0.37, 0, 8542 +redis, je, 5.073, 36820, 2.24, 0.31, 0, 6761 +redis, scudo, 5.271, 37924, 2.22, 0.42, 0, 9862 +redis, smi, 4.612, 35620, 1.94, 0.38, 0, 8048 +cfrac, mi, 06.36, 4408, 6.36, 0.00, 0, 181 +cfrac, sn-0.6.0-full-checks, 06.71, 3668, 6.70, 0.00, 0, 511 +cfrac, sn-0.6.0, 06.43, 3328, 6.43, 0.00, 0, 434 +cfrac, sn-0.5.3, 06.38, 8252, 6.37, 0.00, 0, 442 +cfrac, sn-0.6.0-memcpy-checks, 06.27, 3420, 6.26, 0.00, 0, 435 +cfrac, je, 06.65, 10024, 6.65, 0.00, 0, 271 +cfrac, scudo, 08.45, 4568, 8.44, 0.00, 0, 610 +cfrac, smi, 07.07, 4580, 7.07, 0.00, 0, 180 +leanN, mi, 25.42, 592852, 95.62, 1.14, 0, 239072 +leanN, sn-0.6.0-full-checks, 26.04, 680596, 99.47, 0.93, 0, 14036 +leanN, sn-0.6.0, 25.93, 546076, 101.21, 0.89, 0, 12098 +leanN, sn-0.5.3, 25.65, 530420, 97.33, 0.95, 0, 3349 +leanN, sn-0.6.0-memcpy-checks, 24.51, 522388, 93.95, 0.87, 0, 11928 +leanN, je, 25.38, 538536, 95.43, 0.90, 0, 84230 +leanN, scudo, 32.97, 591284, 118.01, 1.87, 0, 721248 +leanN, smi, 28.42, 620424, 105.04, 1.68, 0, 232111 +sed, mi, 01.73, 326448, 1.65, 0.08, 0, 404 +sed, sn-0.6.0-full-checks, 01.74, 347564, 1.63, 0.10, 0, 1649 +sed, sn-0.6.0, 01.73, 342820, 1.59, 0.13, 0, 1454 +sed, sn-0.5.3, 01.73, 310348, 1.65, 0.07, 0, 685 +sed, sn-0.6.0-memcpy-checks, 01.72, 342740, 1.64, 0.08, 0, 1472 +sed, je, 01.75, 300848, 1.66, 0.08, 0, 7482 +sed, scudo, 01.80, 245376, 1.68, 0.11, 0, 60799 +sed, smi, 01.81, 316680, 1.66, 0.14, 0, 34248 +barnes, mi, 02.87, 66904, 2.84, 0.03, 0, 2463 +barnes, sn-0.6.0-full-checks, 02.87, 65620, 2.83, 0.03, 0, 2851 +barnes, sn-0.6.0, 02.84, 65664, 2.83, 0.01, 0, 2834 +barnes, sn-0.5.3, 02.92, 70236, 2.89, 0.02, 0, 2525 +barnes, sn-0.6.0-memcpy-checks, 02.86, 65616, 2.84, 0.02, 0, 2829 +barnes, je, 02.85, 76700, 2.83, 0.01, 0, 2550 +barnes, scudo, 02.95, 61764, 2.93, 0.01, 0, 4295 +barnes, smi, 02.98, 66920, 2.95, 0.02, 0, 2657 +espresso, mi, 05.20, 8316, 5.16, 0.04, 0, 175 +espresso, sn-0.6.0-full-checks, 05.29, 12660, 5.27, 0.01, 0, 729 +espresso, sn-0.6.0, 05.24, 6204, 5.21, 0.03, 0, 658 +espresso, sn-0.5.3, 05.26, 10276, 5.23, 0.03, 0, 408 +espresso, sn-0.6.0-memcpy-checks, 05.16, 6336, 5.14, 0.01, 0, 658 +espresso, je, 05.45, 9988, 5.41, 0.03, 0, 280 +espresso, scudo, 06.04, 4780, 6.01, 0.03, 0, 641 +espresso, smi, 05.53, 6620, 5.52, 0.00, 0, 290 +z3, mi, 01.21, 71204, 1.19, 0.02, 0, 461 +z3, sn-0.6.0-full-checks, 01.19, 70084, 1.17, 0.01, 0, 759 +z3, sn-0.6.0, 01.16, 66120, 1.14, 0.01, 0, 732 +z3, sn-0.5.3, 01.17, 73648, 1.16, 0.01, 0, 569 +z3, sn-0.6.0-memcpy-checks, 01.18, 66124, 1.17, 0.01, 0, 734 +z3, je, 01.19, 65748, 1.17, 0.01, 0, 2768 +z3, scudo, 01.25, 56248, 1.23, 0.01, 0, 8657 +z3, smi, 01.23, 65568, 1.19, 0.03, 0, 3746 +gs, mi, 01.17, 57032, 1.13, 0.03, 0, 1650 +gs, sn-0.6.0-full-checks, 01.18, 54320, 1.17, 0.01, 0, 1964 +gs, sn-0.6.0, 01.18, 48436, 1.16, 0.02, 0, 1994 +gs, sn-0.5.3, 01.18, 56084, 1.14, 0.03, 0, 1874 +gs, sn-0.6.0-memcpy-checks, 01.17, 48504, 1.16, 0.01, 0, 1998 +gs, je, 01.19, 53616, 1.16, 0.03, 0, 3722 +gs, scudo, 01.20, 41332, 1.13, 0.07, 0, 17114 +gs, smi, 01.19, 56812, 1.14, 0.05, 0, 4576 +redis, mi, 4.366, 35140, 1.87, 0.32, 0, 7994 +redis, sn-0.6.0-full-checks, 4.443, 33348, 1.77, 0.46, 0, 8872 +redis, sn-0.6.0, 4.205, 30532, 1.72, 0.39, 0, 8568 +redis, sn-0.5.3, 4.324, 36948, 1.82, 0.36, 0, 7594 +redis, sn-0.6.0-memcpy-checks, 4.270, 30452, 1.82, 0.32, 0, 8553 +redis, je, 5.081, 36804, 2.16, 0.39, 0, 6746 +redis, scudo, 5.268, 38072, 2.26, 0.38, 0, 9861 +redis, smi, 4.587, 35500, 1.88, 0.43, 0, 8049 +cfrac, mi, 06.32, 4572, 6.32, 0.00, 0, 183 +cfrac, sn-0.6.0-full-checks, 06.62, 3640, 6.62, 0.00, 0, 512 +cfrac, sn-0.6.0, 06.26, 3300, 6.26, 0.00, 0, 429 +cfrac, sn-0.5.3, 06.33, 8424, 6.33, 0.00, 0, 445 +cfrac, sn-0.6.0-memcpy-checks, 06.26, 3308, 6.26, 0.00, 0, 434 +cfrac, je, 06.65, 10136, 6.65, 0.00, 0, 270 +cfrac, scudo, 08.42, 4664, 8.41, 0.00, 0, 613 +cfrac, smi, 07.06, 4640, 7.05, 0.00, 0, 182 +leanN, mi, 25.47, 597420, 94.71, 1.27, 0, 246037 +leanN, sn-0.6.0-full-checks, 27.41, 674896, 106.67, 0.85, 0, 14026 +leanN, sn-0.6.0, 24.78, 528436, 95.18, 0.90, 0, 12185 +leanN, sn-0.5.3, 27.13, 532548, 104.80, 0.79, 0, 3409 +leanN, sn-0.6.0-memcpy-checks, 24.86, 533536, 96.02, 0.79, 0, 13301 +leanN, je, 25.86, 541932, 97.58, 0.99, 0, 98387 +leanN, scudo, 33.37, 615476, 121.70, 1.84, 0, 548507 +leanN, smi, 27.35, 620144, 98.41, 1.95, 1, 396107 +sed, mi, 01.74, 326364, 1.66, 0.07, 0, 401 +sed, sn-0.6.0-full-checks, 01.79, 349524, 1.69, 0.10, 0, 1683 +sed, sn-0.6.0, 01.73, 342816, 1.63, 0.09, 0, 1453 +sed, sn-0.5.3, 01.73, 310464, 1.66, 0.07, 0, 683 +sed, sn-0.6.0-memcpy-checks, 01.73, 342864, 1.64, 0.08, 0, 1477 +sed, je, 01.74, 295660, 1.64, 0.10, 0, 9190 +sed, scudo, 01.81, 245396, 1.70, 0.11, 0, 60801 +sed, smi, 01.80, 316024, 1.65, 0.14, 0, 34081 +barnes, mi, 02.93, 66764, 2.91, 0.01, 0, 2464 +barnes, sn-0.6.0-full-checks, 02.90, 65716, 2.89, 0.01, 0, 2870 +barnes, sn-0.6.0, 02.90, 65672, 2.88, 0.01, 0, 2838 +barnes, sn-0.5.3, 02.86, 70068, 2.84, 0.01, 0, 2525 +barnes, sn-0.6.0-memcpy-checks, 02.90, 65628, 2.88, 0.01, 0, 2840 +barnes, je, 02.93, 74592, 2.91, 0.01, 0, 2546 +barnes, scudo, 02.93, 62684, 2.90, 0.02, 0, 3959 +barnes, smi, 03.04, 66940, 3.02, 0.02, 0, 2661 +espresso, mi, 05.25, 8276, 5.23, 0.02, 0, 177 +espresso, sn-0.6.0-full-checks, 05.28, 10580, 5.26, 0.02, 0, 1111 +espresso, sn-0.6.0, 05.11, 6332, 5.08, 0.03, 0, 659 +espresso, sn-0.5.3, 05.14, 10388, 5.12, 0.02, 0, 411 +espresso, sn-0.6.0-memcpy-checks, 05.11, 6336, 5.10, 0.01, 0, 658 +espresso, je, 05.46, 9816, 5.42, 0.04, 0, 300 +espresso, scudo, 06.02, 4792, 5.99, 0.02, 0, 640 +espresso, smi, 05.50, 6648, 5.46, 0.03, 0, 287 +z3, mi, 01.19, 71124, 1.16, 0.02, 0, 458 +z3, sn-0.6.0-full-checks, 01.18, 70424, 1.17, 0.01, 0, 766 +z3, sn-0.6.0, 01.17, 66156, 1.14, 0.02, 0, 737 +z3, sn-0.5.3, 01.18, 73764, 1.16, 0.01, 0, 568 +z3, sn-0.6.0-memcpy-checks, 01.18, 66104, 1.16, 0.01, 0, 736 +z3, je, 01.20, 69616, 1.19, 0.00, 0, 2198 +z3, scudo, 01.24, 56236, 1.21, 0.02, 0, 8669 +z3, smi, 01.23, 66260, 1.20, 0.02, 0, 3863 +gs, mi, 01.16, 57296, 1.15, 0.01, 0, 1655 +gs, sn-0.6.0-full-checks, 01.18, 54272, 1.16, 0.02, 0, 1944 +gs, sn-0.6.0, 01.17, 48076, 1.14, 0.03, 0, 1993 +gs, sn-0.5.3, 01.16, 56212, 1.14, 0.01, 0, 1874 +gs, sn-0.6.0-memcpy-checks, 01.18, 47980, 1.14, 0.03, 0, 1995 +gs, je, 01.19, 53280, 1.16, 0.03, 0, 3722 +gs, scudo, 01.21, 41568, 1.16, 0.05, 0, 17154 +gs, smi, 01.18, 57304, 1.14, 0.04, 0, 4728 +redis, mi, 4.340, 35180, 1.79, 0.38, 0, 8014 +redis, sn-0.6.0-full-checks, 4.441, 33556, 1.88, 0.35, 0, 8974 +redis, sn-0.6.0, 4.241, 30492, 1.70, 0.43, 0, 8554 +redis, sn-0.5.3, 4.310, 37092, 1.80, 0.37, 0, 7027 +redis, sn-0.6.0-memcpy-checks, 4.441, 30420, 1.90, 0.33, 0, 8490 +redis, je, 5.076, 36892, 2.13, 0.41, 0, 6769 +redis, scudo, 5.196, 37996, 2.24, 0.37, 0, 9886 +redis, smi, 4.613, 35436, 1.88, 0.44, 0, 8042 +cfrac, mi, 06.32, 4596, 6.32, 0.00, 0, 183 +cfrac, sn-0.6.0-full-checks, 06.61, 3632, 6.61, 0.00, 0, 504 +cfrac, sn-0.6.0, 06.26, 3272, 6.25, 0.00, 0, 431 +cfrac, sn-0.5.3, 06.32, 8468, 6.31, 0.00, 0, 444 +cfrac, sn-0.6.0-memcpy-checks, 06.26, 3312, 6.26, 0.00, 0, 432 +cfrac, je, 06.66, 9992, 6.66, 0.00, 0, 271 +cfrac, scudo, 08.43, 4656, 8.43, 0.00, 0, 614 +cfrac, smi, 07.04, 4640, 7.03, 0.00, 0, 182 +leanN, mi, 25.58, 577132, 96.07, 1.15, 0, 224607 +leanN, sn-0.6.0-full-checks, 26.09, 670788, 99.44, 0.85, 0, 13183 +leanN, sn-0.6.0, 25.24, 540816, 98.88, 0.86, 0, 13196 +leanN, sn-0.5.3, 26.48, 553936, 101.92, 0.96, 1, 3961 +leanN, sn-0.6.0-memcpy-checks, 24.50, 526484, 93.75, 0.78, 0, 11139 +leanN, je, 26.93, 532264, 104.61, 1.12, 0, 99886 +leanN, scudo, 32.45, 594880, 117.90, 1.73, 0, 569976 +leanN, smi, 27.77, 626188, 101.15, 1.75, 0, 326530 +sed, mi, 01.74, 326444, 1.65, 0.09, 0, 401 +sed, sn-0.6.0-full-checks, 01.74, 347336, 1.64, 0.09, 0, 1539 +sed, sn-0.6.0, 01.72, 342792, 1.62, 0.10, 0, 1473 +sed, sn-0.5.3, 01.74, 310380, 1.66, 0.07, 0, 682 +sed, sn-0.6.0-memcpy-checks, 01.74, 342876, 1.65, 0.09, 0, 1479 +sed, je, 01.74, 301688, 1.65, 0.08, 0, 5053 +sed, scudo, 01.81, 245436, 1.71, 0.09, 0, 60800 +sed, smi, 01.81, 316808, 1.72, 0.08, 0, 34267 +barnes, mi, 02.84, 66772, 2.83, 0.01, 0, 2465 +barnes, sn-0.6.0-full-checks, 02.85, 65668, 2.84, 0.01, 0, 2850 +barnes, sn-0.6.0, 02.88, 65672, 2.86, 0.01, 0, 2846 +barnes, sn-0.5.3, 02.91, 69964, 2.89, 0.02, 0, 2519 +barnes, sn-0.6.0-memcpy-checks, 02.93, 65692, 2.91, 0.01, 0, 2840 +barnes, je, 02.93, 74488, 2.90, 0.03, 0, 2544 +barnes, scudo, 02.92, 65468, 2.89, 0.02, 0, 3574 +barnes, smi, 02.98, 66852, 2.97, 0.01, 0, 2657 +espresso, mi, 05.25, 8288, 5.23, 0.02, 0, 176 +espresso, sn-0.6.0-full-checks, 05.33, 12672, 5.29, 0.04, 0, 744 +espresso, sn-0.6.0, 05.16, 6204, 5.14, 0.02, 0, 657 +espresso, sn-0.5.3, 05.22, 10328, 5.20, 0.02, 0, 413 +espresso, sn-0.6.0-memcpy-checks, 05.16, 6204, 5.13, 0.03, 0, 655 +espresso, je, 05.50, 9992, 5.46, 0.04, 0, 331 +espresso, scudo, 06.08, 4932, 6.07, 0.01, 0, 640 +espresso, smi, 05.55, 6648, 5.51, 0.04, 0, 287 +z3, mi, 01.21, 71384, 1.19, 0.01, 0, 461 +z3, sn-0.6.0-full-checks, 01.20, 70352, 1.19, 0.01, 0, 763 +z3, sn-0.6.0, 01.19, 66132, 1.16, 0.02, 0, 740 +z3, sn-0.5.3, 01.18, 73564, 1.17, 0.01, 0, 565 +z3, sn-0.6.0-memcpy-checks, 01.22, 66008, 1.20, 0.02, 0, 732 +z3, je, 01.20, 65816, 1.18, 0.02, 0, 2768 +z3, scudo, 01.27, 56472, 1.25, 0.01, 0, 8671 +z3, smi, 01.26, 65784, 1.24, 0.02, 0, 3787 +gs, mi, 01.18, 57240, 1.14, 0.03, 0, 1653 +gs, sn-0.6.0-full-checks, 01.19, 54336, 1.15, 0.03, 0, 2006 +gs, sn-0.6.0, 01.19, 48452, 1.14, 0.05, 0, 1998 +gs, sn-0.5.3, 01.18, 56072, 1.15, 0.02, 0, 1875 +gs, sn-0.6.0-memcpy-checks, 01.19, 48452, 1.15, 0.04, 0, 1998 +gs, je, 01.21, 53448, 1.19, 0.01, 0, 3717 +gs, scudo, 01.23, 41300, 1.18, 0.04, 0, 17115 +gs, smi, 01.19, 57044, 1.14, 0.05, 0, 4617 +redis, mi, 4.328, 35216, 1.77, 0.40, 0, 8017 +redis, sn-0.6.0-full-checks, 4.438, 33844, 1.91, 0.31, 0, 9062 +redis, sn-0.6.0, 4.162, 30416, 1.72, 0.37, 0, 8595 +redis, sn-0.5.3, 4.342, 36944, 1.80, 0.38, 0, 7593 +redis, sn-0.6.0-memcpy-checks, 4.367, 30356, 1.80, 0.39, 0, 8516 +redis, je, 5.139, 36888, 2.16, 0.42, 0, 6762 +redis, scudo, 5.432, 37996, 2.35, 0.38, 0, 9824 +redis, smi, 4.675, 35420, 1.96, 0.39, 0, 8044 +cfrac, mi, 06.35, 4492, 6.35, 0.00, 0, 184 +cfrac, sn-0.6.0-full-checks, 06.64, 3604, 6.64, 0.00, 0, 498 +cfrac, sn-0.6.0, 06.27, 3276, 6.27, 0.00, 0, 429 +cfrac, sn-0.5.3, 06.35, 8424, 6.34, 0.00, 0, 444 +cfrac, sn-0.6.0-memcpy-checks, 06.29, 3336, 6.29, 0.00, 0, 430 +cfrac, je, 06.66, 10020, 6.66, 0.00, 0, 271 +cfrac, scudo, 10.05, 4680, 10.05, 0.00, 0, 612 +cfrac, smi, 07.09, 4600, 7.09, 0.00, 0, 182 +leanN, mi, 25.94, 591264, 98.57, 1.13, 0, 184057 +leanN, sn-0.6.0-full-checks, 27.52, 671932, 105.43, 0.99, 0, 15234 +leanN, sn-0.6.0, 25.06, 516652, 97.97, 0.97, 0, 11333 +leanN, sn-0.5.3, 26.71, 540632, 103.48, 0.85, 5, 3013 +leanN, sn-0.6.0-memcpy-checks, 26.06, 528052, 101.95, 0.89, 0, 11474 +leanN, je, 25.78, 527208, 96.93, 1.01, 0, 119894 +leanN, scudo, 34.30, 593504, 123.18, 2.01, 0, 558728 +leanN, smi, 27.56, 625864, 101.01, 1.84, 2, 356616 +sed, mi, 01.73, 330580, 1.65, 0.07, 0, 406 +sed, sn-0.6.0-full-checks, 01.73, 345648, 1.65, 0.07, 0, 1696 +sed, sn-0.6.0, 01.73, 342792, 1.64, 0.08, 0, 1478 +sed, sn-0.5.3, 01.73, 310388, 1.67, 0.06, 0, 683 +sed, sn-0.6.0-memcpy-checks, 01.73, 342804, 1.64, 0.08, 0, 1455 +sed, je, 01.74, 301020, 1.67, 0.06, 0, 6483 +sed, scudo, 01.82, 245464, 1.68, 0.13, 0, 60798 +sed, smi, 01.81, 315672, 1.69, 0.11, 0, 34008 +barnes, mi, 02.89, 66772, 2.87, 0.02, 0, 2459 +barnes, sn-0.6.0-full-checks, 02.93, 65632, 2.92, 0.01, 0, 2854 +barnes, sn-0.6.0, 02.92, 65664, 2.91, 0.01, 0, 2840 +barnes, sn-0.5.3, 02.90, 69944, 2.87, 0.02, 0, 2518 +barnes, sn-0.6.0-memcpy-checks, 02.85, 65772, 2.83, 0.02, 0, 2825 +barnes, je, 02.91, 76676, 2.89, 0.01, 0, 2547 +barnes, scudo, 02.90, 62304, 2.87, 0.02, 0, 3184 +barnes, smi, 02.94, 66860, 2.92, 0.02, 0, 2656 +espresso, mi, 05.18, 8196, 5.14, 0.03, 0, 172 +espresso, sn-0.6.0-full-checks, 05.26, 12632, 5.22, 0.03, 0, 727 +espresso, sn-0.6.0, 05.11, 6296, 5.09, 0.02, 0, 656 +espresso, sn-0.5.3, 05.16, 10312, 5.14, 0.02, 0, 411 +espresso, sn-0.6.0-memcpy-checks, 05.11, 6300, 5.07, 0.04, 0, 655 +espresso, je, 05.44, 9796, 5.42, 0.02, 0, 317 +espresso, scudo, 06.07, 4884, 6.05, 0.02, 0, 642 +espresso, smi, 05.48, 6700, 5.45, 0.02, 0, 287 +z3, mi, 01.20, 71148, 1.18, 0.01, 0, 457 +z3, sn-0.6.0-full-checks, 01.19, 70152, 1.15, 0.03, 0, 756 +z3, sn-0.6.0, 01.17, 66088, 1.15, 0.01, 0, 741 +z3, sn-0.5.3, 01.17, 73680, 1.16, 0.01, 0, 565 +z3, sn-0.6.0-memcpy-checks, 01.17, 66100, 1.16, 0.00, 0, 735 +z3, je, 01.18, 65736, 1.17, 0.01, 0, 2769 +z3, scudo, 01.26, 56144, 1.23, 0.02, 0, 8153 +z3, smi, 01.23, 65912, 1.21, 0.01, 0, 3802 +gs, mi, 01.17, 57300, 1.13, 0.03, 0, 1657 +gs, sn-0.6.0-full-checks, 01.19, 54004, 1.17, 0.01, 0, 1957 +gs, sn-0.6.0, 01.17, 48348, 1.15, 0.02, 0, 1996 +gs, sn-0.5.3, 01.17, 56240, 1.14, 0.02, 0, 1875 +gs, sn-0.6.0-memcpy-checks, 01.18, 48300, 1.16, 0.01, 0, 1997 +gs, je, 01.19, 53460, 1.15, 0.03, 0, 3726 +gs, scudo, 01.21, 41428, 1.18, 0.03, 0, 17120 +gs, smi, 01.19, 57004, 1.14, 0.04, 0, 4609 +redis, mi, 4.329, 35124, 1.79, 0.38, 0, 8018 +redis, sn-0.6.0-full-checks, 4.395, 33452, 1.84, 0.37, 0, 8967 +redis, sn-0.6.0, 4.202, 30528, 1.74, 0.37, 0, 8568 +redis, sn-0.5.3, 4.314, 37140, 1.73, 0.43, 0, 7378 +redis, sn-0.6.0-memcpy-checks, 4.367, 30380, 1.81, 0.38, 0, 8527 +redis, je, 5.193, 36928, 2.21, 0.40, 0, 6766 +redis, scudo, 5.236, 37972, 2.31, 0.32, 0, 9876 +redis, smi, 4.599, 35436, 1.85, 0.46, 0, 8052 +cfrac, mi, 06.33, 4496, 6.33, 0.00, 0, 182 +cfrac, sn-0.6.0-full-checks, 06.62, 3584, 6.62, 0.00, 0, 495 +cfrac, sn-0.6.0, 06.28, 3312, 6.28, 0.00, 0, 435 +cfrac, sn-0.5.3, 06.32, 8420, 6.32, 0.00, 0, 447 +cfrac, sn-0.6.0-memcpy-checks, 06.26, 3336, 6.26, 0.00, 0, 431 +cfrac, je, 06.68, 10084, 6.68, 0.00, 0, 273 +cfrac, scudo, 08.43, 4672, 8.43, 0.00, 0, 615 +cfrac, smi, 07.02, 4504, 7.01, 0.00, 0, 182 +leanN, mi, 25.94, 578956, 98.00, 1.17, 0, 216122 +leanN, sn-0.6.0-full-checks, 25.72, 685952, 96.81, 0.91, 0, 12094 +leanN, sn-0.6.0, 24.59, 547572, 94.33, 0.82, 0, 11927 +leanN, sn-0.5.3, 26.72, 553304, 103.27, 0.74, 0, 3848 +leanN, sn-0.6.0-memcpy-checks, 25.88, 525276, 101.33, 0.80, 0, 12426 +leanN, je, 25.71, 531268, 96.34, 1.03, 1, 177553 +leanN, scudo, 32.33, 597856, 116.09, 1.80, 0, 515053 +leanN, smi, 27.63, 626168, 101.20, 1.82, 0, 328983 +sed, mi, 01.77, 326436, 1.71, 0.05, 0, 402 +sed, sn-0.6.0-full-checks, 01.79, 347652, 1.70, 0.09, 0, 1639 +sed, sn-0.6.0, 01.76, 342800, 1.69, 0.07, 0, 1456 +sed, sn-0.5.3, 01.77, 310252, 1.70, 0.07, 0, 685 +sed, sn-0.6.0-memcpy-checks, 01.78, 342860, 1.67, 0.10, 0, 1480 +sed, je, 01.79, 293892, 1.71, 0.07, 0, 8287 +sed, scudo, 01.83, 245420, 1.71, 0.12, 0, 60795 +sed, smi, 01.83, 316888, 1.69, 0.14, 0, 34282 +barnes, mi, 02.85, 66972, 2.82, 0.03, 0, 2465 +barnes, sn-0.6.0-full-checks, 02.87, 65580, 2.85, 0.01, 0, 2854 +barnes, sn-0.6.0, 02.89, 65644, 2.87, 0.01, 0, 2849 +barnes, sn-0.5.3, 02.86, 70168, 2.84, 0.02, 0, 2518 +barnes, sn-0.6.0-memcpy-checks, 02.90, 65808, 2.87, 0.02, 0, 2847 +barnes, je, 02.86, 76616, 2.84, 0.01, 0, 2550 +barnes, scudo, 02.93, 61688, 2.90, 0.02, 0, 4275 +barnes, smi, 02.94, 66828, 2.92, 0.01, 0, 2659 +espresso, mi, 05.17, 8216, 5.15, 0.02, 0, 177 +espresso, sn-0.6.0-full-checks, 05.27, 12940, 5.25, 0.02, 0, 757 +espresso, sn-0.6.0, 05.13, 6248, 5.10, 0.02, 0, 657 +espresso, sn-0.5.3, 05.12, 10276, 5.09, 0.02, 0, 408 +espresso, sn-0.6.0-memcpy-checks, 05.14, 6252, 5.09, 0.04, 0, 657 +espresso, je, 05.46, 9956, 5.45, 0.01, 0, 300 +espresso, scudo, 06.02, 4996, 5.99, 0.03, 0, 642 +espresso, smi, 05.48, 6644, 5.45, 0.02, 0, 287 +z3, mi, 01.19, 71072, 1.18, 0.01, 0, 454 +z3, sn-0.6.0-full-checks, 01.18, 70108, 1.16, 0.01, 0, 766 +z3, sn-0.6.0, 01.17, 66224, 1.15, 0.01, 0, 738 +z3, sn-0.5.3, 01.18, 73504, 1.16, 0.01, 0, 562 +z3, sn-0.6.0-memcpy-checks, 01.18, 66176, 1.15, 0.02, 0, 737 +z3, je, 01.19, 70012, 1.17, 0.02, 0, 2781 +z3, scudo, 01.26, 56208, 1.21, 0.04, 0, 8662 +z3, smi, 01.24, 65296, 1.22, 0.02, 0, 3679 +gs, mi, 01.17, 57476, 1.13, 0.03, 0, 1656 +gs, sn-0.6.0-full-checks, 01.18, 54372, 1.15, 0.02, 0, 1953 +gs, sn-0.6.0, 01.17, 48396, 1.14, 0.02, 0, 1999 +gs, sn-0.5.3, 01.16, 56036, 1.12, 0.03, 0, 1875 +gs, sn-0.6.0-memcpy-checks, 01.17, 48444, 1.14, 0.02, 0, 1997 +gs, je, 01.19, 53260, 1.16, 0.02, 0, 3722 +gs, scudo, 01.21, 41520, 1.17, 0.03, 0, 17159 +gs, smi, 01.19, 56596, 1.17, 0.02, 0, 4579 +redis, mi, 4.335, 35180, 1.80, 0.38, 0, 8010 +redis, sn-0.6.0-full-checks, 4.486, 33236, 1.84, 0.42, 0, 8862 +redis, sn-0.6.0, 4.217, 30432, 1.76, 0.36, 0, 8574 +redis, sn-0.5.3, 4.326, 37000, 1.80, 0.37, 0, 7342 +redis, sn-0.6.0-memcpy-checks, 4.263, 30432, 1.80, 0.34, 0, 8561 +redis, je, 5.044, 36848, 2.17, 0.36, 0, 6771 +redis, scudo, 5.209, 38000, 2.16, 0.45, 0, 9888 +redis, smi, 4.618, 35860, 2.01, 0.31, 0, 8127 +cfrac, mi, 06.33, 4548, 6.32, 0.00, 0, 185 +cfrac, sn-0.6.0-full-checks, 06.63, 3632, 6.63, 0.00, 0, 510 +cfrac, sn-0.6.0, 06.26, 3336, 6.26, 0.00, 0, 435 +cfrac, sn-0.5.3, 06.33, 8404, 6.32, 0.00, 0, 446 +cfrac, sn-0.6.0-memcpy-checks, 06.27, 3336, 6.27, 0.00, 0, 432 +cfrac, je, 06.68, 10080, 6.68, 0.00, 0, 272 +cfrac, scudo, 08.44, 4644, 8.43, 0.00, 0, 610 +cfrac, smi, 07.05, 4572, 7.05, 0.00, 0, 178 +leanN, mi, 26.35, 601364, 100.71, 1.07, 0, 172904 +leanN, sn-0.6.0-full-checks, 26.27, 673524, 100.89, 0.85, 0, 13522 +leanN, sn-0.6.0, 25.81, 534344, 100.53, 0.99, 0, 12584 +leanN, sn-0.5.3, 25.97, 547068, 99.19, 0.88, 0, 3336 +leanN, sn-0.6.0-memcpy-checks, 24.65, 529484, 93.79, 0.87, 0, 12943 +leanN, je, 26.09, 530264, 98.26, 1.12, 0, 153838 +leanN, scudo, 32.48, 621548, 116.91, 1.89, 2, 604914 +leanN, smi, 26.99, 617668, 97.22, 1.82, 0, 395341 +sed, mi, 01.74, 326300, 1.65, 0.08, 0, 398 +sed, sn-0.6.0-full-checks, 01.76, 349592, 1.69, 0.07, 0, 1642 +sed, sn-0.6.0, 01.76, 342784, 1.64, 0.12, 0, 1476 +sed, sn-0.5.3, 01.77, 310252, 1.68, 0.08, 0, 683 +sed, sn-0.6.0-memcpy-checks, 01.76, 342828, 1.65, 0.10, 0, 1454 +sed, je, 01.73, 301076, 1.62, 0.10, 0, 5991 +sed, scudo, 01.81, 245472, 1.69, 0.12, 0, 60797 +sed, smi, 01.81, 317340, 1.70, 0.10, 0, 34402 +barnes, mi, 02.88, 66928, 2.85, 0.02, 0, 2466 +barnes, sn-0.6.0-full-checks, 02.88, 65744, 2.87, 0.01, 0, 2863 +barnes, sn-0.6.0, 02.88, 65636, 2.86, 0.02, 0, 2848 +barnes, sn-0.5.3, 02.87, 70164, 2.84, 0.02, 0, 2523 +barnes, sn-0.6.0-memcpy-checks, 02.87, 65672, 2.86, 0.01, 0, 2840 +barnes, je, 02.87, 76688, 2.84, 0.02, 0, 2551 +barnes, scudo, 02.93, 63700, 2.92, 0.00, 0, 3472 +barnes, smi, 02.95, 66776, 2.93, 0.01, 0, 2660 +espresso, mi, 05.18, 8220, 5.15, 0.02, 0, 173 +espresso, sn-0.6.0-full-checks, 05.27, 12588, 5.26, 0.01, 0, 725 +espresso, sn-0.6.0, 05.13, 6432, 5.11, 0.01, 0, 656 +espresso, sn-0.5.3, 05.17, 10280, 5.15, 0.02, 0, 410 +espresso, sn-0.6.0-memcpy-checks, 05.13, 6268, 5.10, 0.03, 0, 658 +espresso, je, 05.48, 9980, 5.45, 0.03, 0, 317 +espresso, scudo, 06.13, 4980, 6.10, 0.02, 0, 604 +espresso, smi, 05.57, 6644, 5.55, 0.02, 0, 285 +z3, mi, 01.21, 71192, 1.19, 0.01, 0, 459 +z3, sn-0.6.0-full-checks, 01.19, 70400, 1.19, 0.00, 0, 772 +z3, sn-0.6.0, 01.18, 65980, 1.17, 0.01, 0, 736 +z3, sn-0.5.3, 01.18, 73772, 1.16, 0.02, 0, 568 +z3, sn-0.6.0-memcpy-checks, 01.20, 65928, 1.18, 0.01, 0, 731 +z3, je, 01.21, 65892, 1.19, 0.01, 0, 2772 +z3, scudo, 01.26, 56160, 1.23, 0.03, 0, 8667 +z3, smi, 01.25, 65920, 1.22, 0.02, 0, 3847 +gs, mi, 01.18, 57096, 1.15, 0.03, 0, 1652 +gs, sn-0.6.0-full-checks, 01.22, 56300, 1.18, 0.03, 0, 1960 +gs, sn-0.6.0, 01.19, 48464, 1.15, 0.03, 0, 1999 +gs, sn-0.5.3, 01.19, 56172, 1.16, 0.02, 0, 1872 +gs, sn-0.6.0-memcpy-checks, 01.19, 47956, 1.17, 0.02, 0, 1990 +gs, je, 01.21, 53468, 1.18, 0.02, 0, 3719 +gs, scudo, 01.23, 41448, 1.15, 0.07, 0, 17161 +gs, smi, 01.21, 57184, 1.18, 0.02, 0, 4672 +redis, mi, 4.357, 35104, 1.80, 0.39, 0, 7991 +redis, sn-0.6.0-full-checks, 4.432, 33844, 1.81, 0.42, 0, 9037 +redis, sn-0.6.0, 4.178, 30424, 1.69, 0.40, 0, 8592 +redis, sn-0.5.3, 4.310, 37016, 1.79, 0.37, 0, 7484 +redis, sn-0.6.0-memcpy-checks, 4.422, 30536, 1.94, 0.28, 0, 8505 +redis, je, 5.238, 36860, 2.25, 0.37, 0, 6765 +redis, scudo, 5.234, 37960, 2.21, 0.42, 0, 9891 +redis, smi, 4.669, 35336, 1.94, 0.41, 0, 8028 +cfrac, mi, 06.35, 4408, 6.35, 0.00, 0, 181 +cfrac, sn-0.6.0-full-checks, 06.61, 3708, 6.61, 0.00, 0, 516 +cfrac, sn-0.6.0, 06.29, 3304, 6.29, 0.00, 0, 436 +cfrac, sn-0.5.3, 06.33, 8260, 6.33, 0.00, 0, 448 +cfrac, sn-0.6.0-memcpy-checks, 06.27, 3336, 6.27, 0.00, 0, 432 +cfrac, je, 06.69, 10076, 6.68, 0.00, 0, 269 +cfrac, scudo, 08.43, 4708, 8.43, 0.00, 0, 610 +cfrac, smi, 07.14, 4596, 7.13, 0.00, 0, 184 +leanN, mi, 26.06, 591428, 99.05, 1.13, 3, 226227 +leanN, sn-0.6.0-full-checks, 26.06, 653060, 99.23, 1.01, 0, 12978 +leanN, sn-0.6.0, 25.81, 539980, 100.77, 0.90, 0, 13684 +leanN, sn-0.5.3, 25.73, 533320, 97.81, 0.66, 0, 3598 +leanN, sn-0.6.0-memcpy-checks, 25.97, 551676, 101.63, 0.87, 0, 14181 +leanN, je, 26.63, 496704, 101.91, 1.14, 1, 152314 +leanN, scudo, 32.46, 581268, 116.25, 1.70, 0, 531142 +leanN, smi, 27.36, 621904, 99.41, 2.03, 4, 392238 +sed, mi, 01.76, 326452, 1.68, 0.07, 0, 401 +sed, sn-0.6.0-full-checks, 01.75, 347260, 1.66, 0.08, 0, 1585 +sed, sn-0.6.0, 01.73, 342852, 1.63, 0.09, 0, 1479 +sed, sn-0.5.3, 01.73, 310284, 1.66, 0.07, 0, 682 +sed, sn-0.6.0-memcpy-checks, 01.73, 342800, 1.65, 0.08, 0, 1478 +sed, je, 01.74, 295456, 1.68, 0.06, 0, 9177 +sed, scudo, 01.81, 245652, 1.68, 0.13, 0, 60802 +sed, smi, 01.81, 317072, 1.70, 0.11, 0, 34381 +barnes, mi, 02.86, 66820, 2.84, 0.02, 0, 2464 +barnes, sn-0.6.0-full-checks, 02.87, 65616, 2.85, 0.01, 0, 2855 +barnes, sn-0.6.0, 02.88, 65684, 2.85, 0.02, 0, 2840 +barnes, sn-0.5.3, 02.85, 70132, 2.83, 0.01, 0, 2524 +barnes, sn-0.6.0-memcpy-checks, 02.87, 65672, 2.85, 0.02, 0, 2839 +barnes, je, 02.88, 78712, 2.87, 0.00, 0, 2546 +barnes, scudo, 02.95, 61940, 2.94, 0.01, 0, 4267 +barnes, smi, 02.98, 66836, 2.96, 0.01, 0, 2659 +espresso, mi, 05.19, 8196, 5.17, 0.01, 0, 169 +espresso, sn-0.6.0-full-checks, 05.28, 12652, 5.26, 0.02, 0, 733 +espresso, sn-0.6.0, 05.16, 6364, 5.15, 0.01, 0, 660 +espresso, sn-0.5.3, 05.13, 10292, 5.11, 0.02, 0, 411 +espresso, sn-0.6.0-memcpy-checks, 05.12, 6280, 5.10, 0.02, 0, 659 +espresso, je, 05.48, 9832, 5.46, 0.01, 0, 345 +espresso, scudo, 06.03, 4896, 6.01, 0.02, 0, 647 +espresso, smi, 05.48, 6700, 5.46, 0.02, 0, 286 +z3, mi, 01.20, 71152, 1.18, 0.02, 0, 457 +z3, sn-0.6.0-full-checks, 01.19, 70324, 1.16, 0.02, 0, 763 +z3, sn-0.6.0, 01.17, 65920, 1.15, 0.02, 0, 731 +z3, sn-0.5.3, 01.16, 73628, 1.14, 0.02, 0, 564 +z3, sn-0.6.0-memcpy-checks, 01.18, 66240, 1.15, 0.02, 0, 736 +z3, je, 01.18, 68604, 1.17, 0.01, 0, 2431 +z3, scudo, 01.26, 56284, 1.25, 0.01, 0, 8161 +z3, smi, 01.23, 65772, 1.21, 0.01, 0, 3727 +gs, mi, 01.17, 57540, 1.15, 0.02, 0, 1657 +gs, sn-0.6.0-full-checks, 01.19, 54320, 1.15, 0.03, 0, 1961 +gs, sn-0.6.0, 01.17, 48424, 1.14, 0.02, 0, 1998 +gs, sn-0.5.3, 01.18, 55808, 1.13, 0.03, 0, 1871 +gs, sn-0.6.0-memcpy-checks, 01.18, 48492, 1.17, 0.01, 0, 2005 +gs, je, 01.19, 53572, 1.15, 0.04, 0, 3728 +gs, scudo, 01.21, 41320, 1.15, 0.05, 0, 17111 +gs, smi, 01.22, 56964, 1.17, 0.04, 0, 4612 +redis, mi, 4.489, 35192, 1.81, 0.44, 0, 7969 +redis, sn-0.6.0-full-checks, 4.408, 33576, 1.89, 0.32, 0, 9014 +redis, sn-0.6.0, 4.193, 30440, 1.71, 0.40, 0, 8578 +redis, sn-0.5.3, 4.289, 37084, 1.80, 0.35, 0, 7347 +redis, sn-0.6.0-memcpy-checks, 4.294, 30464, 1.79, 0.37, 0, 8546 +redis, je, 5.052, 38772, 2.20, 0.34, 0, 6761 +redis, scudo, 5.244, 38000, 2.19, 0.44, 0, 9869 +redis, smi, 4.599, 35432, 1.87, 0.44, 0, 8057 +cfrac, mi, 06.34, 4596, 6.34, 0.00, 0, 185 +cfrac, sn-0.6.0-full-checks, 06.64, 3660, 6.64, 0.00, 0, 508 +cfrac, sn-0.6.0, 06.26, 3328, 6.26, 0.00, 0, 428 +cfrac, sn-0.5.3, 06.31, 8308, 6.30, 0.00, 0, 448 +cfrac, sn-0.6.0-memcpy-checks, 06.28, 3336, 6.28, 0.00, 0, 432 +cfrac, je, 06.72, 10056, 6.71, 0.00, 0, 271 +cfrac, scudo, 08.45, 4568, 8.44, 0.00, 0, 615 +cfrac, smi, 07.03, 4584, 7.03, 0.00, 0, 181 +leanN, mi, 26.55, 593452, 101.72, 1.13, 0, 230978 +leanN, sn-0.6.0-full-checks, 26.62, 676740, 102.76, 0.92, 0, 13834 +leanN, sn-0.6.0, 24.98, 545136, 96.54, 0.80, 0, 12708 +leanN, sn-0.5.3, 25.65, 551156, 97.81, 0.78, 4, 4057 +leanN, sn-0.6.0-memcpy-checks, 25.64, 536612, 100.11, 0.84, 0, 13271 +leanN, je, 26.16, 515292, 100.02, 0.99, 3, 110034 +leanN, scudo, 31.78, 599964, 114.04, 1.60, 4, 654494 +leanN, smi, 27.40, 619364, 99.87, 1.82, 0, 362050 +sed, mi, 01.75, 330536, 1.64, 0.11, 0, 404 +sed, sn-0.6.0-full-checks, 01.74, 347536, 1.62, 0.11, 0, 1703 +sed, sn-0.6.0, 01.74, 342788, 1.64, 0.10, 0, 1477 +sed, sn-0.5.3, 01.72, 310240, 1.64, 0.08, 0, 682 +sed, sn-0.6.0-memcpy-checks, 01.72, 342916, 1.64, 0.08, 0, 1477 +sed, je, 01.74, 295444, 1.66, 0.07, 0, 9184 +sed, scudo, 01.80, 245660, 1.70, 0.10, 0, 60801 +sed, smi, 01.82, 316496, 1.70, 0.12, 0, 34208 +barnes, mi, 02.86, 66908, 2.83, 0.01, 0, 2465 +barnes, sn-0.6.0-full-checks, 02.85, 65632, 2.83, 0.01, 0, 2853 +barnes, sn-0.6.0, 02.85, 65604, 2.84, 0.01, 0, 2829 +barnes, sn-0.5.3, 02.86, 69948, 2.84, 0.01, 0, 2525 +barnes, sn-0.6.0-memcpy-checks, 02.86, 65692, 2.85, 0.00, 0, 2843 +barnes, je, 02.90, 74568, 2.88, 0.01, 0, 2547 +barnes, scudo, 02.86, 63084, 2.84, 0.01, 0, 3492 +barnes, smi, 02.94, 66784, 2.92, 0.02, 0, 2661 +espresso, mi, 05.19, 8288, 5.15, 0.04, 0, 177 +espresso, sn-0.6.0-full-checks, 05.27, 12648, 5.24, 0.03, 0, 737 +espresso, sn-0.6.0, 05.09, 6296, 5.07, 0.02, 0, 656 +espresso, sn-0.5.3, 05.14, 10328, 5.11, 0.02, 0, 412 +espresso, sn-0.6.0-memcpy-checks, 05.12, 6304, 5.09, 0.02, 0, 655 +espresso, je, 05.44, 9864, 5.41, 0.03, 0, 304 +espresso, scudo, 06.03, 4796, 6.01, 0.02, 0, 642 +espresso, smi, 05.48, 6708, 5.46, 0.02, 0, 291 +z3, mi, 01.18, 71348, 1.15, 0.03, 0, 459 +z3, sn-0.6.0-full-checks, 01.17, 70352, 1.15, 0.01, 0, 769 +z3, sn-0.6.0, 01.17, 66064, 1.15, 0.01, 0, 732 +z3, sn-0.5.3, 01.17, 73444, 1.16, 0.01, 0, 564 +z3, sn-0.6.0-memcpy-checks, 01.18, 66208, 1.17, 0.01, 0, 736 +z3, je, 01.18, 65876, 1.16, 0.01, 0, 2782 +z3, scudo, 01.25, 56096, 1.22, 0.02, 0, 8667 +z3, smi, 01.24, 65752, 1.21, 0.02, 0, 3822 +gs, mi, 01.17, 57476, 1.14, 0.02, 0, 1655 +gs, sn-0.6.0-full-checks, 01.19, 53988, 1.16, 0.02, 0, 2022 +gs, sn-0.6.0, 01.19, 48128, 1.17, 0.01, 0, 1997 +gs, sn-0.5.3, 01.17, 55792, 1.13, 0.03, 0, 1869 +gs, sn-0.6.0-memcpy-checks, 01.19, 48212, 1.16, 0.02, 0, 2001 +gs, je, 01.21, 53748, 1.18, 0.02, 0, 3721 +gs, scudo, 01.20, 41096, 1.16, 0.04, 0, 17116 +gs, smi, 01.19, 57064, 1.15, 0.04, 0, 4646 +redis, mi, 4.386, 35048, 1.86, 0.34, 0, 7998 +redis, sn-0.6.0-full-checks, 4.478, 33468, 1.91, 0.34, 0, 8968 +redis, sn-0.6.0, 4.199, 30328, 1.78, 0.33, 0, 8578 +redis, sn-0.5.3, 4.262, 37176, 1.76, 0.38, 0, 6883 +redis, sn-0.6.0-memcpy-checks, 4.283, 30304, 1.82, 0.34, 0, 8541 +redis, je, 5.047, 36796, 2.15, 0.39, 0, 6770 +redis, scudo, 5.271, 38052, 2.23, 0.41, 0, 9871 +redis, smi, 4.596, 35492, 1.89, 0.42, 0, 8051 +cfrac, mi, 06.34, 4460, 6.34, 0.00, 0, 182 +cfrac, sn-0.6.0-full-checks, 06.61, 3604, 6.60, 0.00, 0, 494 +cfrac, sn-0.6.0, 06.27, 3312, 6.27, 0.00, 0, 430 +cfrac, sn-0.5.3, 06.33, 8404, 6.32, 0.00, 0, 442 +cfrac, sn-0.6.0-memcpy-checks, 06.26, 3320, 6.26, 0.00, 0, 428 +cfrac, je, 06.65, 10052, 6.65, 0.00, 0, 271 +cfrac, scudo, 08.43, 4660, 8.42, 0.00, 0, 613 +cfrac, smi, 07.14, 4500, 7.13, 0.00, 0, 181 +leanN, mi, 27.12, 578536, 105.64, 1.15, 2, 163365 +leanN, sn-0.6.0-full-checks, 26.12, 671828, 98.39, 0.91, 0, 13146 +leanN, sn-0.6.0, 24.71, 546284, 94.03, 0.79, 0, 12227 +leanN, sn-0.5.3, 25.67, 549420, 97.34, 0.76, 0, 3992 +leanN, sn-0.6.0-memcpy-checks, 24.79, 534008, 95.51, 0.88, 0, 12152 +leanN, je, 25.97, 501392, 97.88, 1.12, 4, 154360 +leanN, scudo, 34.44, 587232, 125.79, 1.89, 1, 484336 +leanN, smi, 29.18, 626364, 109.19, 1.87, 0, 255797 +sed, mi, 01.74, 326500, 1.67, 0.07, 0, 402 +sed, sn-0.6.0-full-checks, 01.74, 349584, 1.65, 0.09, 0, 1713 +sed, sn-0.6.0, 01.73, 342852, 1.65, 0.08, 0, 1476 +sed, sn-0.5.3, 01.73, 310336, 1.67, 0.06, 0, 677 +sed, sn-0.6.0-memcpy-checks, 01.73, 342800, 1.65, 0.08, 0, 1477 +sed, je, 01.72, 301980, 1.65, 0.07, 0, 3141 +sed, scudo, 01.82, 245416, 1.72, 0.10, 0, 60795 +sed, smi, 01.81, 317056, 1.69, 0.11, 0, 34329 diff --git a/docs/security/data/doublefreeprotection.gif b/docs/security/data/doublefreeprotection.gif new file mode 100644 index 0000000000000000000000000000000000000000..1ab646a993ab017d147de0eaa63c2cde191a2b31 GIT binary patch literal 1552092 zcmeF(XH?Vcy722iAwcK}ARPqhs5FtTMmh+HfYK2~Iw}IvmC$={fdmM>_uh*Lp;zf3 zA|0fObmhb~_nv#Mxz;}K+2_3De0uXGjEsE9U!-o&Kxw*l? z!AeR>NF;K8e%{*JdV6~t27~G9>bkqTi;9X?S6AB?D}DR1zPvl%P?Wt}8fx{PpYC z?~n2S?Lim`N7yUhV2s3OIrP{`r48pfsTLcQP8B)#eRFSJEOS*8Kn<-|xevT*{JdA^ zsZlz+d(!lNX!raQ;qBw==N}Ll6dV#779J596&({B7oU)bOiD(fQ&Q6~=^2?>**Up+ z`2~eV#U-U>19S z=jInaFMe72y1cTww!X2swf${pcW?jT`{9qHUlWY&vTsmvKbNjPrK z^;G5#rNMnkSX6IT<&9)2$MPHZR^^Z7YG$gYs8$zzD%7jAneVMGoGdl{5W@1Hrf9m- zYOKJxkFTb9w$}b@Z_0z(lKDp0@0;^|wWXh15%{F6YIS8_J_J!;H|eh{|JohJ$?+gn zt-fNV4|&UWp})RzZ3rV5%BtQ_wK0~bU1&1UP`x!-#%9)+s@_=hZMM#FYhj?VcK36e zFX>f{rn>#F*x2h&2bU0&OsT3=>3kvf>4mJy_L~h%1)h;pG z^43pm+CFMlr79|GcQUvP=0*PhEKh{O9LW;+_;nKkWWX!b z28cfO7U_|pth&W|55KsA8a&qzhbeq6IvgOh`qBna%E&kk>Aa??V$n6bbDNr!3;~CU zd2QQ+$;pP{gSwpdmE(_Ve`-xQwXqyeJSa1gpCo6#BR~4c3|8~uG4|Wh;vR%G@JrxW ziu@=7zpZV6h(21VTc2~zWif=h3&rzGv{UHGybr_2Qt?RCwg?*KB@W$6i$qH&a* zY~(gpe)+xMLHHzvBy5n5{qBC!)!oY$__fxZy=|+?g}YZOYsuiFXR+_?M{k|akCljn z!LG%S8EM(knhii*J7znoXJpiC!0Z7WuAd>A)$dTa_DZPIZC2y2jRw zN_iQ}aAWp}Uc&Eo$&jamjpX+G&!#aBRGdhl@Iv}j})4X z-&-r2r!XA7X`4x9bK1c`ao3_vfrHyzD&0UZ<$tb4AS1qpo*Eos?d77 zsI}#YMQ*6_sZ}QLZTn+sPozyKs3^okv(^X~rOzLy+mWv4!U%cm!%sEbvgAQ`q$Onm zv{tfdSgw3EyNHKWob|EO2jB-`+e@wc9uMRI5Y=L1Ajy3!W${FS+Zt zxVhf;e(LyAMa*dTY!qEj7|PY39Z{KgFW>&jFrTS2h2~d4r>=JLU09Z)>z9!ZgFy<= z>q2AlZK(x@@lt*tonGp^?mWv5e$F6Gl1ej_DNi9qM^1Z1;*UfXfA}D~a@qTb*k?2{ zbSiCHM^yn!ZQBewjRszPuaQHZEb8C5{Y$FLh`nNO*tjrJOeD0|w%(!FsdX_7C#I@Z zjO}2s$UI3)QP7i4YQI z_0%T1r3a-+C<}Pw_RLp*s*Q=FTt=EKhzwPubxmVmK#gpK@~YlPp%#MBrz%TSlPAxG z2lsDSY!5PhJ^3g_@uVX3Gx)jGsUB8&0zWdd>;?B#NpqD5kq~b+bB+h6y@-&QgxdZU z0gux@Q;JGteb|b~J==aOjmqRmPCVkG(*gU4@38}FmL}C118)bv8?T~?WVTL+Bs?1` z!GQXD?H&W=dx5HqF^6?&PWO>0?gyFQ_STgR&#uRo$Yl!-SQo!Yrat z=ZOZW=C9aq>NK7i`@F9%N~_qEsB)jEqg5^bV!vf{-oVrNUar(jecSqD!xZ*NrEFek z`-Qal`OEJ2HC471+hU>~GgD$J)zkLh>^!77XHDd4Th(7h_!*La6{u|xbl4Hi@|fRC zP;MOA+i{&0ZDjv`SFU+?;I-fKEZO-^ZQDEAZ{AGI*DeL*+9(I@-*KYJNMF}=Ztd?0 z%X==-3mUcG8hpR+rDH@!w_ArD+TR!Q^IYbA_^nHO(7`Q^nT+RkeShWtfncZSszk2h zK=ZH#i5dd$F45VLuf}0IlkvK;p!;ZK$zjf|pBoP%8^*FJe&icAZ|dbXOq90#NXNgk zhX2`(Zbt2>0`+s->S5!|?FBt-TA~{|>5~4nf2Uiz^+yYgRC#eHrdE9A_H(y>OV$O{kn-BStGmNyFAlSw z)@}&;oQ)@X{GhdN-V$mhKM%paKbkJNqwp&7++nuw)lW*F%L_sGOTrPaU%FO+9uy>jKs-T!-yk4)mLw8TvMI#dBLoQM zO=V?4Get>P=1n(jLEq-hVDAmzN~C3sa;9bT;RvE-h;pVw_+%t_|J3xcaPbj|p7$gE;yE447QWvrc zRBsD((N<<~qbGZh1^ecK{Z9g;U_ov6K`L!QN{c~3So)9?(kLr%yfQeEJ~&7?xFsrB zZZQ}DLXwr~(Gt&J^73X&gm`oXCj&47et z)P=qu39Z`;t>O*ax*sMY7Frw>wvrxJV;VNF9`+S4yqYb%UOD_zP^kPw_)M@e>x44t z)Oz@5Ue+&lX8l%#59M^9?}@%pFk3kZn_CZf0Tsh=5vt(pXwXJ3N<=m&6B<+ND&YR@ zmp#N^1Qrzt%;mKAu#v0lkzrbdSWz&EY19Kp!;?&vE2~jo-b7RgMLcJ_Pw)vK%L0UB zqaLvt=ygUfeu*vzVxB>xwsr2BI~v)UYh3S%xm@@XledT$iHtBSQ{3={i#tYYn=10; z#7?!x=I-HLwFfBxv08w<>PO}J+h9idCx?FE8DFUcIdF7*oT@d8nl;JQD2}U8P#Ppo z8_=2^yF(dgG{mVvX33zRIczbc_*n~$I89XOtcZR#Y zPFmD_y7YX~;+gm4-YPmz2O>5S=SHb~@;R8mi6bQd2Q%Yg3HNB8ug``pk&)9B$1^tfco z1YgP&L&}VP%5+c4tXs;E3i6&T$Y*RFobnhh(WA%_C%yP7SrsZqYARoml={vZHQf$y zF5+<_(oX8rex0UWsDJ<|%&+M*LRAc&6o$wE19HcZCSxdOFjQw4h(S7ydpcDwh9o&1 zDwWP`lYTWMok=PKW{}P?%pu`})I`L|M(OZ7rOHN0&#a^_tf(C7s}jY<9xe2|PGbOg zO+kT@$?5SA1FAI8@tLVtk|fR|@EMd<;F;w0nKH>7_ZkvpH^C~mRTY3t0Df0V2<5V& zqNuN?wh3ya(?i^)3`w&Bt>FG#ZLeXVI7{BjfQ z=>|p-WXqN*pCPkIzN!moCp|ySO%~5n2Q)(&6Q8WyaZ1w3_6MUIj2-I5RU0B>-0ihC za%MTSTyH0S8&|=%S0G%JOp{N#$jS$gKPqs`uhK8bQO#G2%2th6Hc$X3fu&dE9`|O- zDky+&%sg+3Wpr*a5Kohfw!b2ytO^#SR;FarEF}ikofK^zi^ZOTEzgQ%oC?&V^y(ao zg%k)PuMp_wXRb=+>-K_C*{MiLfNUDfA6y`&Qt;jV^?EP3lt7}y_ffxll#*nTrhRfW zW7&dOnUQup8IzO(acOu`48wUDH1s7^A9x~L6dGl0B&qvo&zN*Jci8~Na{cDjM$?-& z4BbaaDuoN(Z}nZjVOZIEP$5xRx!G0OwN$xGSY^pwCFfCj-?pkguu7q@O5=fHX_jV* z#)I4PsGHhb9%{DLy3(Hd*99aSt0T0kO-O5uNUQT@Yp`?xjX}CyW6i7c^p~WyubFC{ zLenHgYF)3_x*OJdplUs5YqUaZuH*Rz_!-s(T<7!&wYw|D5jI;F@gO^bsXk`*TCib# zf=4|PRiE5gkDjehr8}?3Fg0XIH)I($S3H`h zP|Y)q&2zKO3+K&?Of5^&Ez2~9Evp_a>!_B^#+L2bmYwsKJ*HOb1L@X7!`35@))Q3g zS!3(PZ0j$Y^HzYl4J6YBHfkgEY=fZNNSfNH$>!Q9F4~~X?Nl=DG)C>Tp6&Eh=yt}Y zcILTu){AyF=8kJJ9h_7~9o(KByyy=8rj8qP9Re2}LNv@DL}WgQ8hsG+{2+n;Al39i zX6^&k-HQ)!=1y6ePI;qFMbA!UbSG4`sZ(vPQ{$pjllh~T%tsw6qmNHKKkA`B8Z>=0 zn)_&S@zIoux$C)1*9)UA3(qbqbeBz2mmO@b>(xb8mp8tMTlsL-#=&n);gN`dTjf z+L-%0WcoWPjrzMh`?2W$-lqQkx&A@eMgK7Kz^Kf?xY59b=fD(tfO4j3U~X<;;bLHs zd2mT)kaF2*aMg2g9X+_&G`KxCNV#({xW_znATxAmG<4)SggQYFoiztM_5@#+3t*9 zH6Fe8a+LGkD1XZ6jqp){=25}`WSYnB7LUQ_$L{|elQkZf ze>X0dGOk!WuG~DXF+Z+3Fs}7;T$|;S&bv=fQa(Ly{sbKW-~WTaCjN=QZ~%pKE}U@T zE0KmKUH-SCKM&OW?3~cHP06x6LU_hd+7Dr%iq(Ewusk#y1n@Mw*?6-iK zUXna{gD7Vt^v*TX0DcRYCf+UAF#x{87h?J0H-HU+5$cuoH;gKcKjQ$*%MGx{6VJob z$tOkpIM}+Js(c-Gm!S-w{mmL}-uz;H6qxi^?^{hY768Q(lL;*?o<#Mn1{)2kN!1Hz`_Lhku4$r4n)>w0FVYr_(jUO>dpJpQaxe{N+TWrWQC}c(gY(tLV z1$#2@=*mYrdat2FDmIR}V0-f)my@h=rpZITrROohyyg|U0Yx8vLq4Kw^3?U&mOdRW1fx7vQL;DBprsGqlM+lM}fA4I09ZS%WD>*zPu;oZ;Op zG6V5*Q9*h)Y@-!jdEw7CS@&PHkcn>#F|NpNJcZ%$S{Pd32@S(*H`x*U9?Ax`ogu`m z-k(44<`z}ZogeWO^kgf#rn+Bo;0C9t4BM?h$G7NPsa^bI!ok!XT*C+&zB&0K@nsDi>or^XXEv6h)6@3dH*0C+waN4X zlxQq|%y}NqA6qC6TOUG&k2a3K1UVYNNwf!*)k5F$p-z_LbYGpWAT3KySJ}`mJEv=y zK!LOMti)Gm8+o}UXPZT}J7-&E-2&&^Rg>(m&cD^Im7MQ19q*j)wh;+l>~+$;zSzg| zlwKV4i|<~1XC77({CPO8`}*gPDa+EIM{_Q_KaUp!1%I6^C%$I+b-JEg`s-}FcK6r$ zUbo=o#o^@Z%bzD}rI)`hj#+nc4oijsh?LtvjC%-(8U{p%Z6lQ0^M;0Dz&t|A?IZ?! zKJ@(8FWbgj$T*Gm7<^Q8FX&=p0XE~E3>9D z!lwKkUC6aS(X7Q?9-@ev1n$U2VEfk)wW~n-sJad>MF_5p}(mR%V zc9;t%FVK8fp!HaFF;7{&Kr69#JV)wBzGirVPOj>w0)rm~dIJSdYI{GGSXAdd%t9md zx>YACl7AFgsTUed_D<9^{3x~$FEmwT6Z3Yfy zDSSoGID+-Cyw2spcH3sGq|R2u@ukU9` zcl-pJUk|Wvu+A@edoRei%F3ltl4W~GzlC&xJY@35^3B(TLxu^51;;+yT%jJH4ckv5 z%v%mqFz-J4e7+DHjsm*An()K%Kv(122#(}SA^3L6-FldOI?cVe3^x{=fviq|;KFhX{=)6W)W+~&T1I^6#hxN4hW3eW>eTc~u_nEbfWzjzh*s zo4k&c-dsZQK+5v>J3)>Cx+>YRY*Lz8P*?*&W!lV znK+5Vnn_f34Mt)XRx`EBtu>US#%8LV_~q}|sQEIYQtm1gJpQ?%{WNww7xA`8{oAHZ zNdrx7*V{r{b#?v3j>*;>`P-%0Nn4L-W~UliokhLxkXVQtPW8TDEteC1X~QEuGw9^Z zTh;v1$_$nC>6d)n$9Lae3&zf-!Ce}fU+%n!Xr$f^aH+SQ4Oe^X(y?%CA>JI$u&aL9 zNb*Hk?itv^-)q+}@nk{1qY$VQ5mkDuuXob!Xl<{iBST^!BKI=miGS$BrUa^Pd2A*k z*)J)588&&`?YptBS~W*p&3;0^`qn?OlsSQ;R(>!=v-VaeGpY1tt0dnYs1L`K5tWjz ztkXE6I(f)&9aNEl9My}Z3zO{Sn*2YPUm=de68QTi6+m8i)GRL&00ec8aZ-R~ey zL+5kn?~hw|@9YJ>KJ9yI41TC|bmdo!N-|x`(-EqX`hb2=8Pv;!s_z3? zE;lqx<@fHp4T%R3xCFgJG82wD!5#&?Y6}d3I#kwz8SA)a9N0+Bzz@LhM#OJx0U=52 zw49m#3MYgnbr8B^7Z4oK#RW`p^4%-&k07p)P>nTx=phmW@jH7qWLCq#8dWw`wpEjKb+^~ zVfZt-xAL-x8Knr2tT%x~keyIO0Mvoi;hij$NAbD;BOxz-;3{E{V97~HS$AY}yVw=) z2wO}6O2^dwPt==%4O$7t>gdlwgxSlO38hHOEWKKCy z8OrL*0p6BXnTv7Y&u8&*j26r%1O_9WC6O>4q$Pm#29l!rkYUqEr6j^&Nk~$DQmB3s#WK=@Ey;^5 zIa4Jm^)xBb8ky2VjN*e7yCv`9C!52Q-@#GU*2&&w_3#Q6!qQWs<_%O~5(-j!%D@6OPmqec?+Tn@$0}NO^`y zalxc+^`xx2rGC~Yd_n7ZDCybZlJ+Aw?I@zaX@F{)>tq_ojy68u_2v#CY|*xosBW$T8PW`DMsN-DxP<$sXb_YGDCSa znrig)s3`y2qsVqf5xHp5x@J*V103s4(c4h;jCj0gb+2gH9X@J6F>X*y z-~_rlT)c2ryvSIxBvrC(P_pV?vYuSB*-)}QQ?hebvJe7-6%=!TK}YVTC&{H}4W$<| zxG#>S08<%Ax(sYsM(9xnL6x$?OUY)-C~)p|g{hoMx}3(aoYte99#ziRSk63K&PsD$ z&c;-6O}c{9u!7s8f)`c6-&jF)W41!zyh4bnQbf8^)UZ;_qmo(zRVme2DKlGn_q-C$ zR3$52B~N2mrRY(mjH*&?tWuk;(m1cuWU8jplCIV zc`jY^!m!4|qs9tVLuJ!gV>gTQuo`=&S_kP`C&OAQXOCJ}RIPg>PQz;7pVuOo>Zp9A z>wbfnM_mxAE~K#z2V&5O^SUUUh)LJS{UKtoKS}IAMU3iiB!&|)=-){U*4X$r5`+F> zV*f>Aus==gKS_-8pG?fK1^RargE6)K&BQ2wqu5_ejPg$u`@_U2{|Uv=?a;rY80Ftk z>@iNoa42SUo#x3uq1eBg*xyO)Uq$Q>5ZnIvC&%#orAWeY431iG?1FQaKSCs&&io-S zIEBHv4324VSc8j_{*W3R)BKfVa1evz7+i~la~Yh);98`Ih-@6h;3_1X$lwSDXEA@| zNH~J|LudZCC=Bjn|M4JWLr9P>Zs7&TmxFtPnr_d_Pc3wBm9kujcvV_$N=@S2xx^<7y=3h^OSt&IngIFcR%U zWAnr+T^yec0Pqh#xq!UqjDP1yV%l@W#G-2kh|~DrIno(b;_F^m`utnrRa}i!y#3ZV zTwZ96J@R*r#Fdpov%J~owLe*qfc?)Ak;MH ze*S8*JzTldVxXLF4ebqGUHHoi-#5Q?RE+jDB zpvWgS8`(&Uf(;zVuy1bUVgmR#^Rg0bHuLjx3N{OhYBo0u%ewftimE1Twu-OTtrl#R zG#zbjm9`OH-!AKeXGTZ%ud~M zg6&TII#W*JPQ!N1)=uMI*Y(||!-?M{vs$>@a&ff9yxR&8-Pmgb)7kB{LwJhzI&dbl z_W`POW51JzQ`c_)BfVwOeiyUL_I@{8;Ee<9+cWd|Hxe;L4$a(|VQJpiyQR@PF$ep6 z-v?G@=@*TJzHZ;DTe=OM^u2Rw=TzHkjZGg=5^pYccp~!caD3`4o+n(#^5-|;?txL& zgsG_Xg9T#MfR|!51g3oJ({}efGAEzb-gGImcI57S3GFX-sesNoA3t;3y?Ih@OZlp< zESj~1#kJI5aL2VI_@01darndkWSRd^8M^<5WoWA={~gQxv&;M!%Uti}h!;vy^vqJ+m1RTug```1g!TR|yqyb-bERG1?q8ZHkK!^inA^RkTEQ z(jbM?efwX4?2?A8ah(Lgz!zo5@~{B<8lcTUvh7Ln@_C zT^1?Zaf#B;@P}MGY*S0~bMMUiKm4>V>6o^?X=(F|So1A0&28{a-UbTyWIujU%QueM zVAkY;ph1TWnjbj|Z-Pg{YGS3>zEKkL3XSqa#AV5`hSMp_^kzr{L+p z@P*?yR4JFW_&P*$cat|1{mr!bmT2Vy?ab5rI#2Fdmf! zsTVz`>znG5Ix3F}FM7fAV7k}fsN9Ntd!KNNWU!*6GWn(np-5lINKHpo`b}0V-SQco zDtWQ$Ho2G9DupD|9b&Gf3JR~(%V%eI25YLbU-BciW*2_c&~&pXIE30tFJ*Mp=O6?e z-_#mC(xIc7OnJ4PWk|9U(a|V3lVa?APk2EiPQ0lIzm#{Jl;kL5^6lD+yytD<&&R2Q z&1r3~c@AwqkAD)UzHF99oL`sv<WfN5jTAxilm}+G_hZ!K@*ca?dYKTW#q?G z{)O9W4R}7Rfmb*Dh0KiNLjqiwW~l-Y7RAW$^!kLT7u8eAb#$o=Q{ZEVYun86x<%j5 zh4Bc#;Fc%Ed%>oTU!}Rs_K*S_j&Oh}Q=17dE0StCSA0n(BNpKJqG+)`8)Sat(&%O= z1(;JpHsMOS1^ZzanYe)n?Tnp8raYm!v;ZlQ;+{7JO?zJi{0%-7N*9-{3|V0ZV98E? zE#6ZQm%AFNL|k+tlWsSo??&iL!1eB)>qDIGb|QMdR$PxHy7yX2?Pbs~TMsk)Se-D) z;XT^>L92q5yGSSziO0mLpLwv@$bsuV-DK^s;119D@M3nr>^T*{It;dB zVt;AYBHLJehh~&niicgI5Hi)US1o2#OsOmJfv4n2SHfvMU7$}zARNOcx*{!`M=M)M znF{v3insWQ`^4s?BNmE)lcN>4i6FDn28LhMSaWM5BQ#mBb|(?vl9ipex-34ZI84-)C{D-(SX@uzbDutMRfM{Mbii zfE%2t3&vZriKDeYt@XMWNPt~-@sfIE#aroh8}Ge6+j}3VDc*r6IR$2uGq>km0CM%~UfhffH*vI=d z#@icex|L|h&I5LHc$b^wrLbqA31w5}u`jpuqaX9SMUO}~^<}C63M~NfrT1Mm?BO|X zz8!%66gZ-WKmqH2=;QYaf!I+e%0( z5K=HDkclPCuOq5O64gtDE_DRivxWKqVFN>vq8=-vKE1H{j*wPJXgO~< zkz)8DuTXa$;eZ~|tOU`#YxvYs=xcbG7d+yfX@u7zy}&e*@G~#bH>9^AGJ+I8{Hs+s z3K8Lfh&-6OxfMjX3yCb=i#&(iTv{hQ%OfHRCL)%Mni`F?WQ%g4k8I+KCes%>w2C@H znlc0vlj=u*T8gp|(pwRWSyzl%b&laqis7n{;q8gxpN?5j)Dx7970{0rQi&C}j+Nkh zCHiPBnnXM5rMpzBg*q zG#dI;bkp(1Q`DM#2^8PsIq~8w^o6Y?sm$vWz6ZI6+#pJ%9ZR^jpWqZM2lavUp zOl0#;d~Y4?Ac=gdlHh2K^qQuAmydi??<^D{DU3LUJ!43Utf%&|PWmO1#5#gZwiZH9 z!;(&u&Z&}_A0=fGCkG`Zlbar0|oL$ z(LX|UFbH`%)Xn*>oI`zLH85*3)NM&#uWN^D@@Fr*QH)PzH$)FZE%MfDB6p_jlHOLfm z&y+~cl%i_Ll$puAdzJ}j%#xMLk~hdwq;bzuPR>$o$Woih(m2b~WX#5a43&;S_7nGP zz2t0zhHRY3n4D!(nKI@)m&$ozkb@H$tK=M;h8!xpnVeT=IXIGWkjiy3$aQwlrE*Qq z#hHxfOz!)$Tm)mDk5nEN4rKz|^MaD|LK^bIX7X?*1C3(LkCDocGssVH&&Q!mGF3x9 zdL}>hEFZ&IfK!<)gMu8GdqG}uK|w}z5CX8j&bdUnrjzs-p3(;SL&omK>z6c{sY%8<+uMK05}%F z5kYbL3NFsT;Xp#!!ha0&|2v-k4+#9fT2b=+b4BS`1pq);Apk(&#WugIh4QiirQQv= z6{S^ogdhWf=M^DuGI&J-*z=YkZbRwTSQmgLTa-rhSOM9UniTdv2=k#bv%U%9vp35< zwUwfS(!Up!1Z^WFQw)gT+$6-=(_mA*V^qgS&G#uU8WIS4(`+w458h$N?{FoB25E2t z&Ml-MC@*LVYrG&EL@96sgLrzaJ239CZLVz=f8SxZ9iOastUD0z5 zcuFxfIqy$zaS{f?v)~4N(_!s^7ij5i8Z?gK{(=qH!P_O`3G((fzngb4@`0e7u1Vvh zn>_LkH@Sf?P|8IKOaN{>>4hM8Sa-%1v?-nrY^JmURup7z$srNsZx~G|i*z{weRq`^ zs5&SLVNQlP`3##nRrW&=knJJDdG=fpPr+e)x$&O2?bAR=Vxi@jAnpPye#}pCI6_|3 z037MsF=d{Wn7X$T8b<*$OMSuW4ss^H0n3FY@#&SzKP&pOwge?hD7ab}`kMp%gfMMD z<5BNcX(P9{K1dNz>GbPl6i!DQ(w0Aeo_6JkT&do+a&l z{*v{aLID=hnyDp}+@Ak8=yHz-*mutxRb2Qf8O0JZ++diIf2k;43dInUAqt+7T&y+V zC_tE=li%o79Z?!|-a3AV>1htc@@xY-XJa5>O2X6lvVC8`P9Bd|m%9^R_^R0cz@@}m zQ$X{eiK>40;Tc&)SJJAIe2*|Fp`4nl7A+j4wg)L4)X{zUgHFa{&R{}bXdS3Fva!wY z|0UufJgF&q6Fegy8s2F=ZhZ52&Pl1SE|L0)QG!)qiv03#SI_GKyHSunAC*e!prVg!CKlz|U!(XscH5qZs zlp@aAJWhm=J|`aY?Izf(=zhIqThg>L5b5>47mn~CX@>(8RShEBc4@#eD4|}F9u_CL z*O=4e_Uim}ooBNSLA<924%`u7+sx#8a#aG@@;J#DL6DGD!HONHVL}>{8Kp3GFD{X% zXSCQhQgT3^!gbhWk%I$&Z%ytploHGpM5p_%Se-_s?c$d+2*QkzC(Wa#HszLKAKrXP z4vE4ay(LY&uYmiChX65Qc|Ne7?FGWgGN6IVo!1!mgOt@Ws1mWA+*14hL;zg>)&Q*3 zG8reaU1AOU;r3yf%xl+_yQOCKBV7A4S&y;ZchB}C5oB3xL@HQW#)GIJwXAD&J?vOT zse|aKuq;jD5JgZ3{L@S_w#USvx20A~TT{~iOb9919(iY&zYrU%`R zDlz26R!R?MMro3ds`Au}ZFKu)CK`^a%fgH8EFWB%otZhRsT(MM<MN%9eK_&>HNd-dGn2hhQ~32$qxmwyRWJf?*ZFUpsY2IcV0wwA=iRyBoxG!grE> zB#T-C@i2TuC(CcoLbh?bCtdQ8icOOr=C}h^D{;Ja9loJzcs7b`xdr1z;K^uOy!Bf* znbcrw(&EAl2fY7YLDLNGc@bd}7V9J`4Bs-CJ)5X|BAdGfM-aD9K?`K7eSV2ra?~+^ zWS#?svBS1nSMT2!OQ;r6!FcmoxkI?xZ8^X1Q(q(7t^9L@UKzxGSdn;*!ksV-OFm6`hn6A}Ol zej0-zb?(606A-vX(b89m=}VXAamQLS(J*Iyzl{_Ek8Tgp2|XWITz4!;0={v|F)jD& z)X_)`+uw<6><;9;GE8nF+r+^CHR@|QRn8S_G@bTxR9i$Q)3eRkH5GS-#i?>*ll>Jn z7Q~XXyU5!eNVhj5%z_(K?i6&@nL1tQrQ+F2$9o^fG~>P)>hrNL--5nZSs8?`ON6af zncEm|KIZBVt&yT|Euu^_+3n`^;DG35ba7mp=NbJp3D#WK`blhVEPH`^Y~glIucfQf z*Hb86fWqMptEV6L=wcL`J%oH7BKYLp@+oV?c-!?UD-w0WmG^JoisQ*LjQ&EWS}fpz_UaYQN?g0N1X0Es~PBglgENly^S6$F)@K8#Jz zJ6^$?o>!l)&O0XCoApGGh2AHUz=sp7$KmP|X6_@#kJRHsLK-K0?tJtKgxiWtX$uBH zVD>M?UA0B6AktI5((%55P(N9u_B}m_qN|^{f*%~~E2_q;t)dKhK<_WQ;-?Dne{!Ow zEdhC4=P&H>)R&**FOeWtp_++X}y6!9Fc*% z=7A1Df$t@>JWhz-Nd$2b1bOlX1wpj@*NFnBg05u;1z>|jW2PQOP7y`B247VOjKA9!FXbqbq=!TdM8v02=bcfIjVK!HhcIFy>ZItC7tvI~(W0!951CF0StMhA zfMb~XVt9!ka%~XuOvfB##c=n;h)in;P7?~d#qP?-3R%a>h}&L{l_JLR zfQDQ>p+bG!hF_d~ew?O?hI&5XL;d(w&Ug)#cmoCv-6TT&)A+BQ@p>EaqAxbop9K?| zB_%ArNO&Hc@Ty+j){XF`Wa2zH(T*?CHCX+vHKEIN;!IYeb5Ej=zPgt_p|`dha#9|N zuttXQsRyeNhB73LFC#;UlM+tVVkHUV>yt)k@YoU7S^CL2ZpnE`$p!k!9GE0bk|P%s zRUwJ0(nr;}q3V)Q4fUvM6+A9eP$>f%ipw)P_0e5!XlxR?w;tV3HH{uTMGrHij7p~9 z`iu#;lqpzJ%1nLA+;qyHGC=jOGywf=1CnWn`e{dQ|F8krzu*A$PaPne!BCuGpp^gM zf&bD0*k6l{f9L@8uRQQu2Ppr>1C)Q`0T|8){+S0T|L}o7d4TdyANYd@DF5mMI3A$< zn-5U_%?HSFK=5BaVECUvFi`gPpM2n7dEj4k;O{u_R~zUp_;Wgj{nv51A8ta$4ZOIa z7&jyTvA}>ElX1u8xQY0WdpmGrGHx!$&C7ob%edM3e>Bt*1e$V2ys%ESDk!nrYO%jD z~Rs`r3+in;RlNW#v^1>~<8Z&@N4qaHR@^D+bevAKG zqbry$r^%0Xj@Pqok_1WcEzOhw3OA=u^g2YAca}~At*D1AhX+s{N73!bPYv&XGWtYR z6X)=zJu|L7{eC}x=N-#}-^%vxYtd4sA9lWj<7ffE5xW5bbWMn@Ey?0em+>275=>r}<*R^6M?5ynh8cl9gjd%~(S9loj~&;!?e$==d0BRS63$uK!%G`0N{^emVq)x3b% zoPXFFRN^KM3gRaV0BlwGh4LYNOITAMsEqqWF|jHWGjY{m;|^rvPFV{6ZWrHP8)Z1b z6n^^5@rFm!?W9DX(&)vTZy{tB)&Vei0&C)93Z=!5vElGG(^2P!j!Mk!hJ6T93}Hju zvvj>f|5|MJmu`#a*15!&5@(Fc&gEP8HPrz+8$o)9eh@jpI2og~zMqu)40c+=Sj_TYI`3 zL#sT+TS8Ug{nwb=v3Bt{+wkRXMbzp{C6rCz5izYhT8%~5JlDyQM8I_@X)vx}0lFxg z{%tB=cc3UmA$}w_z+JBk&vU+Qoml9u1-K#i)46GD_mLE+u|Pu-0MXAhnx!gTVU6fu zGR_>~l2GZ;U&`3~;b41HNj9b+UC0?k+!SK=48prv*bo)UCfOYy3->U93CF%k2wS{#_>P3K$h2NrZe@3r`r|XE_V7+R zu6p|bes+OzkEPFYXn*EPVioXt~B5O22;=ZTW`DJA)a+y=1oqW`3dnQ{KM#054MP z{KHgThjJTkq-8C3?@tn<@s^5i6RD8R_za5TWLyml)nQKyIUE=6js9j(#}sN!(Z&Fl zX#*1bIl0B%dM-G5{Gw^_NT02C?A7xE_^P26$;!8k2-LT6)noGh8a6I3=O{$V{hNG&N)y}&G<~Tiye4XzyVaH zXVCN=@hr+?yw#nMajBo^fblZ<%XHFMW_{3-PPb#tlE&>%UHpyHP@z{0T*P`rSnUNB z|LZf{eQKOy?^?R#t!~}djibB5zY}4_E4O0idArB^`G~u*nFVp!5f=@_hQR1SCgT-@ zB$_9&c0?a@gw;Q2y~eE>EVy#*Rq(Xt^9kv0UrVV;HP2`@wzwA|Wk^)a_u$nRN9p1! zy41To@hD+a`|D2Wbde|hoT(yCkdG|Pav=T&}8Kz*GFHQ(3AN4D8t|cY!oU*kr=hS4o^8{^s z8!q4>nc#*p(-_;Jx^;-2F%QE}8SCY87NE(qT_LK|F;|m&HZWy6GNm(=cg3P!=Kin6 zjcYxS2amF2ti(Q5=zh&G4e2x!P|cP!6@zs*E5CqF0yngYIT9OrHH3vuBoI2;v4Q0> zHer>wke{_Ih0KcII5zy?fd@=D%w-CC$!uhZn3G*1>p1R@Zhl;3=eZU0pnr=bYp#mQ zjv}%6CbYn)Dv_E};saORSp4+|$WmCWCll=rr<=F4Uzrbu74v#p!3BE&oN|{tY~_ zlIZ_p?=6F>?EkgzMW=KuO6itv5a|>|xY9F%s($X1aE{d}7=v4QqakKchb4dI{ zk9BPcbwPR+7F-((Z`ZQMiG3UcdCEN@pc8MVuX0}Rc{Z=r#R@j63xD)HuvpHeR0E+q z_qXd3y2fIUv6P@ROmy~r)`z;!t|BH3CD-0=aMhxd%4iZ?!tHziBB zE1Y1imtf(ZV3m|$Q=ecrmGF@KFu{Q=(NQ?jSufGWJ<%;G(W5@mYbx={VIl>D4DKfk z572`Lxx=AJa9I7fdO^e?Jc=v{93z|*r9wwKQrBs6dJYk!rQd$mETFFw|NiUtSKQ37CuM_sq3zp>H&28dK7wp9WOZsp2 zHgLcaf4g8W9k30V#J?&BzuDWw7ykAodz2BlyRzbLM zw{d^C+ute(fV+)%xq^Uu$=}BP-a+_faO3{yApGNSj zz5$W`r7v6n5&)Iq!gmLh1wduEu-gIM0ZaT0O$)q>>e=D0Ye?29RQv>5V-#@ z6bF>xd4~Vh7yc`<=TChB{*CNuji_*gmLD_*kUeROY_yOk9JC69_;>j?G@n_$wz+QW zciq7LUT7y(+vn}@dW)T+xq%5QBcg|&8S%qlJGf1c&sWMfJIgV9yy=v?lMZC&t5m3C zUzJGP>vahU#dcv!?=QVB&|>_c|DMN|F;B9q*4K^W9(h&ZQcJmL&Bu3?ItCG*6p9~J zdhU%q+2$8ES`~GXMmq~#Y@D2` zQYTZ-r>-nMzcMB6^?B*^`BT@)X3IHWbn$(yC-5s^YR`BKp#&d-m^c1zTaPEE{nur6 zX`G0p>aGTN+*vbqU6a&(=g2l0yZ}Q8I=CCW$zZh@j)re(7R-7kf6wiy3Uf|iI*PDS zh|xOwysb~YMSLt@yAZK2mb&nAgu1x!e2}h)#Zt`m1~&$uuNA$^a0zX9`UtyP_lOwk z_Nm2G14Ox{h&_bNYw3%hR^JAZDUmU!ndc>CrFrwXTPEsavg-N-l9OAc#&rubMLx7B zD!eg`akT2rfU_Z$7Ln)&%L!%K!dS^y$m_R?DXSD=_9kz)DhBWnHq<%sD@WiY%d>`! zyiSsVh@=1kFk2pH)Dex@U6SW)Ph)6VOG57U+pv0F7fP{wHX+boT=u*n86bgX*={v# z`PCF;^(WM`Kf5VnP!?=y5n9l^9#%aa3hksxY?I4`)@#F0lL4gSu^9LQ2943ww)Er4#ekdCltGrgYeDx^&^z0;5%JxHV@tp(o+QxYNa%=v{ z6t9A=roySMd)PLqhtBu88d)3^(q8Mo+YoF$O#Nv1c=PsCba}TMqjNa->&YlFbb8xK zEaZIBr88#*R@=#^cY<`GYMwd%tz`?}U31-ewE7K2mG7=OXSHC9iVZ5O;UB?yhQE{Z zmtAwfG-vp4T=NUloaVo9%`Z%In*ZpU|1iz@ei03Rcg>r!Kpb2kIy?{v1#ZR%@C=ZR zV?lBfKp)2e@-Vqie zgzZgFYBLnpF!$5Hzz|JsHEi)T3!U58rMU<6)Pzk_%Vj~{8EkXm+OEiy z<3&yrDTL^1I;;%dg%1{3bsAds(i-Oq?2iL=C;ei@KQJ(PWgd-T6(lyJ=PCLuhP-K? zu6;%amN2NdEq;Hr7l4H$p-xpYi&Y}P+(sP~V&v2C#-^w`14V%aL?8h|->Yq07naaA z6VN!mBhaiupu~aP`lOU- zQ}p#2_&uoab!!1Y@ zl8V>VIhWX;Zf|7Uzcd-xMMY2!-&FZ5irDr>c2K8T;t|zcrsnO~>mMmg3WMjOKW4pY zP`)8>-)V@Vus;JR2KiRpX|=d(KtYG~HYojy9bz?)C`h@B6I$54a=RwG?PVwdkm=W^cU+Q70%L zU|@gP;E&vUhdd%Am9ZU;#NLX)*kX&obyc6Ru{CFH;Q)h{!y&x!WSJ|ne$Y~NFL~x< zMQDV)e%N(*Fa7vrRRXsGfH~}EQJk(R+-`W4I83phCwjWB7TGYCull*j;B-TGq~Xo; z;m@U>r<-6?+{Vcs)xW8pWZ$r!UmIplKe&xFzWqA~oInwgw`N-)9 z&-3GX+!qHuw~5boQ_fG;ZofF17&-gWcz(JQ`Ql`i`1bkH%=y{D$cwYDBj;zw0A~ga zK~xfgAnQU<;g4^|LNMzf0M!%c0D=ehB`AQP^*$za^993w$?JS63w(*|9y@(OqnSjb zJMdN1@$-WCd1Ux8>N>N-(Ky}ExF`J-B?bLmO#Pi9{(O^8B0^|4z-Tw?{Dnya93%oB zng&SeIw?$|D%YW^zyr9y2G~#sT1f<|6+F^*L)FtoHE;{udJ||43c5=jWUQoMXAVZS zJV3G54T9AM8P5k9f`aX=?mO0@JcgsV3I)?71PhNja;*h>K%KnYQ2cdK0>RMX$571- zsCp|DI;k50Mu|Q^jy-^y$!LXyBS3iyLK2vr;0IFDM6kNoAqrnZ%G*M+D{Yv0U^!OK zdCWWolh?8N?j*-bR7}FizQU9w0px~QFBs0_PzP>|%BBVwXArn87AM*7PqCu_-GwW+F9coDjI5m$qUO7Ki%xW_AWRS(Rfl{tzaTPh zEx1=*Br=QF<`wrDJcNo_f))<}Z(WH%?LUhRKePrx;h0);1QIR;36?lX;W#fGVLl zaY-MAq`91v39BQ=>XIE7lFRgz9}A^aCnb_wC-1j&UzuW$AY*(Ha;GgRtkEiZDoM)<%X+lc*SevF3Qs$%pRQE{VpLLRQgXGMRL^8I0&B*~eyV$cUuP9_jb7%w?o&W- zKw%BV$4vj!XIKr-fV|2eYf!;cfMT6OUUEaw7qLv%U)p znMy^u(-sj8Si#dpMf*h!0MY=aRGo|khzogf#ZfE8P1DM)Hdu}1CA`^uB{cxkfLy6p z2@8nMKOHNn0+0qud7mYojW;O1V#NZI^GVguDgdy7hvGaY763Njiq|YH$u3>POq}R1 z-A-28>Bn3pFN@|YD^@G}a-?ujgb4@>;bUcmJ7u5&_(%P6j!lO0z|3-V+h_X{(#v|C90w z01a?2Q9kj0CmMVM8sL7%eEy<*;{U{a{-Au~{=$5I2O8l1f%&BDz`ca|#05B?zatHB zFF2n+nFhH3a6W%P4Y2;2#_%C-+eyS&37wQQJPt6>t6QE=OyD0q8S|B7juPay|KoBJPZgNQw zwJ$k>P9rdjq66eLP}TVAW=uPcLEdc@vJNt=`gGVJX0_OPj=7$wl(QS5?X7c7wCs!C zQl=ZlZSo>$0<<2OhKRi6NYKFs{ySB!2T<01Pjp#qw&|Bcj-$|RU*j(&& zBB^5|2Kf^vb4x=b8n-P$0ij^k_nw%K$lk?)$iS6`y48*Wi2WN)-e7uzgzlWD3qpB9Y+(0F zl1l*bro_ua#|A!%ed)j;3J?xGDAV^Qs<3&aNJAGz zjO)1DA_m=)0~$S(Kp|=bF_N8kNX%aR$-2lfT<CvhT`D~O2nL^u&&7(!w!rwwriN)r>fs2%M7}P%8Zz!em8F_Et^}y+D_~X zS8HBM$mnD!Eqlmf+Hmw}YMdC3TRxI}11m!bL&29!r1R#rlsL9qL97-nR}NWR=nXMe z(a+N`1pSJolRerCin|k9L16;I^o9l9Ul)TvU17`3@qSxlYd4Q$^PwK|N*x^i{F$w( zA9@gGo4d{p<}4dbiMm-WUu(=aIS71QDK9?29IVn^sU) zM`pk_Sv;g~p2pR{4Rlcsj%jOO58LiGN<2AP3{6YZb!=nYO8Jz$<8nfQG5O|DD~*}U zz|Uw|iki5GWpYEpIB`W=0%6io*bjt>F2P`DvLR?6d+H*W@XFmcm9@b<@p_bzx-?`v zJY+h5FMP6c%_afUS;_AS+DzeAQyRjY|I7%1fnTA^vaP9yL9iHqop)k}o^N=SA+^Cd z&h2o^rk)MrytnicUWm(jQ?f;2u7^fzbKMN#i~)_ieI!Htk-~m&oMIi`P83I~Neo(i z{opAcxjCG=D0~ea!Gj4Z?q}p#Wh#d3&U8j^FciuqfrM&W=KW>%-czZ{qR5t|5tajs z=O%@*Y9gNNk>yX`TSrFOMt*#rVbVn>WP8JDGTo%z2TIr$e=DO#e{Na9HN>ROB>_>x zI^3OAZ76#GYa^z<8kIUm2Kt~M-xmZ_?`wSU$G2J~@wY<7In*@wlM}8JsiAAdW9qjL z2j3)MU+287uYHV--XT4O8PwozT!`FL8eK!JE1*e<%v>ii6>aU*g9Je&g?CJCb9s57 zP*UQP>iQANIRx`JWdPKHh$Hu?+ufp376;~C@2pmo)!IkXq==dyW!k`Gw-$nntg0t9 zA1Ihn-u0rNl%NKXr3IMSwA{;*LL84m^syA3lxY~sONgAROqLIv={La4 zQbdjw$if~A%g%C5cv%v*zVzqW?3N){MBggvJ-}Kpne0_UpI1YvOwK-%vl4cvF_Ww4 z)9vY$?_|eRM=3XxQf2Y*Q&K*U1xE6o ztn+Xz3e2wnqtil(V2eS)vl34893yAgD^|b^SmKzjLwsh&?G1cW%iU&a3fSdJW9rIR zbtMn;em3SWF|cgaYKiz(MJc5V7Ah}E)?7A^k5PJe{T!+wPPAX<1(Ei(m$7d29p*$a zVX&6xgsC@~h8D}BsO_BJa;Q!ydL6g$sZel14>7BSAPjf85FieO7N=+MB5dV1CBH7L z3C|Fwd8Mw4*k4uRVgb7{kc!{Fn1W}vDH}*g($iA;oDzet%*gOQCQmWzOW1x6-ku4! zsFtFlY%PjTKQ$(tdEBg>|FLa#+AhqUzH?eNsT}#bNwD&)E{F@IW{+Whdht4{`F&8) zodgS_QO?GFl=~y6cc5UR1XT7d4oruK7N|M;6yOXStg6ix0lM1j#sZwS8)0cV2i%37 zJz6MfI!K74k|(HnpyTZfdm_fH#G!|>dcMhIi^6QkVi0*Lv1vO#jnYod;SBgmG0_}s zIFYBE7ZXzUqE_L+3?Id!Vf>1jgOmO&Nh6#{D^MMAks)oBAEY@c9CIDPyxk^~rMM+# zA|kWbpIR_PzSen1npC*f{qTTQ>~7^$@N%a{3~^gZisaSnQKOr4;8Nr7&>= z>m3?K!Gy4wlVh0&8B&ej z*6JBQH#K-4GgsHIGrXJdk8HyDQ#bj~vI(G?(Elgdf=+kA2dQG?Kj9Jb=hO8k6994= zaDS0Hf3Zn_@dJMeaDbFhfQ)W{9JyP70z5zo2x?3Qs2&8Ufdkcr0yT95wcP^2y6`~# zx*zd{>8W&{8C-of|K2JRodgQPUP zOTB|7+)GFT+zTeb&)xz4Ur++v->C#ZYyFe{u=Fks^N&Zd|zl6Z{k0!1)bK@Gr#!tbd9JT1K^@G4|w*#9SRusKidF6-~AOZWL5swIB@U+jHlt#E9cY&nD^B^tFG{| z(1K}Y*Y3QgMF5@${QLjE{s3E^QU6CP1^->6;Qx*-&+uvXOhv%r>TRuP3M z`a8(uk&|3o(mSTjP(r4wb^+Y?e1b8)@)I7a-S2|)<}ZuZKU(UsogiFDwsEr=FYO zAL4^EAlrv7)C9X!Unc$Y>UH!UVBpkJdw*UR%)mp_hQ9Tn4qAI@LO_CQ=8uXxId9Hp zQ?kyMpSyq3EL?!2F3S#ur|w<^I_f$dgwQ2;K7z9rz8J)R#{523tulZzno)J# zEFQ3{E6*O+dR%7XGS;2Kxy*$(R6VqA=b%S?v zKW6`wr^rjSH>TOh`?0Ys!LzENLuv>$_VZmTzEN|+&D|ITx;eg=w@Riso=b}krByA% zPRz#LRWDdktdl)8dhs!ATk0F1bJugDX?H*hzjVGQNDnn z20_SzkHTpAKzQSpWVF)|fBpUJ_hObZLlUDd>O?urS5~DEBtX0w8d*##g|~VtUqPpd zKxjKwB8a9S6rv1t%G}%HRP&@k;t~?bT1>uRkJr$ZF%Gi1)m!<~3CRNS;)ksT65)87 zVRA%>Ppm=cpjOBtm^ur`XhSyz6VfX)h9Ht#WO)9p9gH;kI)i%-_!Wo-da6^w6FmtT zQVC=vbed<6-L3WX?oas11*IcFHw~HC;(M7v`Gi9odYdfSsPolT`22Qvdh{8y)3++o z;01OEHtPKi0_;A0R`Pod8f`Xl(zWr?NOJnf<$kSYj#DMN&zWZW%e5=ZR7%wmrEkZ+ zBB3%yf!Hrs&OB33d@hT}Y>c#{{_?D-K5~IgJ6`#1$=Lnp=CTikm)dpKVbvK8T^#t7 z$ahS07_!sP- zIU$0BvCW7XprW~8ZQM}^8uKK=+Xv;Y_swzM_oBvOTt}&}ia0jJnR#2s)H0VJ)b+?T zp|ra;peE9ip??CI9RU&N%_nE-#zhKIVa5aGHWyp3h^y}sBx`fa!v!|g)>#prMqD2; zNI6;JsS3#W%-|$coji|ANR1$wH)256rHMy0uWzNj?aYH|$a}2EEbxHW@;(VD%dsYt z=k9~ts4#7@x#}T(R7uyulU1d%fb^ys>zs8OQaKHHK9&`y*|V?;uRKy+`0##4pu{Lz z!PKjA5=Gw^+@vxi7FCiFrS{>i6WLE}#1B&+lmw^6^0Y{EIEmwA=Tm?PDW=fL=A4(p zTGvhqj;E~zxNFCg!!;1LNV5@pS8w0nze|Uwne`&*wz^69wu^?$%#3lXb_m`}M0Rbt z{aHTv)3lCth+fYJ95?0p<_DPMu*Scd0a(|8~0y4u6X9 zJXxg1t?!j5SKW;P{x_I!*Z+RJe!pGGznT93e!P->GyVV7^C!D7{eQn+|2bYse@=P+ zdA)-F)08L4@8|2U>y_k(>HqurdiI&*hw1<4`AYP|^#A>QCH`gl|K|A<{WSf5yI&Rl zxnFPcl*;|#`A?Xf$*%hRJYP}JAu_IyB3dC=6s$4V9C3Ql2^D++@1I@A7nb2m;pVFd zE^wr@0$;VlpgHg**YOL2_^}B2FOl+j2m_S z=?X&N1!@t@{-V4gqNuRUgWCml zytxI?#R^y<_;$GhxH;3WObB*fv8HeT8k0sLgnfy!&JE)l9zs1>$z?Z)lr|(`k{*d( zRJn>rlPlDn3xV1c;hI`lIY|FaHK=mLo?&i!v&_`yDC+d-hj>31S!-<~z5aWS}qoS}1qp%O7aL7QE z4#*@KkqPDqmkS1e-K!i)G2?YHV6N}|0{2*wbw#Ws1ktHj;?~%k>#-8TaZ-A5lJ0Rb zA#rkm-(Mf63e@7K;_e*A0e-)xd%Q+cymn!{F5vf1#hdoU)7QR0}IARzyc>_9nKi(do=5dM_mE5nT(ILJmHZA+?Ii$wFNb zLI%$x_#`D-bjrj|i9DJ&`EpOPiTg(R9t*t?ERrlKQ8CE`nZ`yTFd0KQ=AIpKJCvR< zIRXPQ$2vJ)lhbK5LVGFXwDv|?UwDH%Wn~g78Ff-^Qh2Vr1i*j1GscNn$P>?ey8~#; zR|mAQpceNf&x56>7sF7JQmYlx#xc@Qz~rIzNi%xFtFJ|0qNkg;!oA-2$-g0!OF{R2 z>CcTtB$|pL_n|)Jll85g^@!F3fvmkW5qg#h7P{AmPdkV*Ofrmz>t$aE$=LFu;CQfM z>)#@6p!gyzHUxk8vM+tRK7$PN7KO*n!70hUOgK91Die%_f&3O{5z16j7~6D~{A+XG zq6}``Ih?mE7kc) z>-j#@d0v>xuH-51N*IA61>$i9L`?Zm%mNrpK5QBuEP@eBUWo5pSoFRi0kaUUFOLrX z2bK4gso=)bBBi_wRNjUUfXyq|FS;P*Rj?MnTPi%wC~QztXzIrWaCz3rRBcKnxy~g= zTqV6n^8LwLBRtb38AQ*%wwAnNecJ8uY*FUf7t?3c$?~)M-(Y#JM5U<~rAs!7Ypl47 z$)(QKrJqJic|Wkq?_lBrZ2||)vYx!MgK346X`HXqWe@htw(!ak8|9E2aey@g>$vib z@^WlXIh{rne~MkCY6Q8eI;ct@rHsF^YS6j(V#R=Hv1(EA`-s6nk=&0F1H#LeKj8Qi{54`= zL-f~(0kB~}aB0K<=noKF8ZiJIe}X?o48HdV2)^xl{$}|Djz8YdUC)1JJqiBY_59}e zb!iwy(3KZZSl4FmkYhCOkAS^nQ0 zf4pzYo?xkf<+@F^Jh2xK_`NQ(RaQv@) zoA&&IE)9^Z<6%m9-{XGcKq>#(n3HbkN9WW~8ZdM2a-C)*r%~p_aF@`a=8nx+l?OVSKn4V5vZp$79s~yy^OL@HTz!-Y0b%)>oW@Gt9?R-!};=T#92R0xc)Sp)j=@WQNSA zM$o%GzJ@WgAWV*5~MNeoOrAoQq=x^n($!yc z2!RB7X)u8)1StdoyA1uDB~Mn*DCUh^r7^ zJSDAnsCIs~jn?gcA?8!6)a#W|>7d;gG0DYJ%^(m9>3!#9t*QnYQk59vDPn!Q5C&}> zyG&Hfb(3cEm~!TJ3RM^d1_}s`8?>C|!2i0yud%&K!>^Yeq+;RqpbnA;nX1>x-_e`o z&luVo2&^@dGR1w7LHZVt-fn*ZYCCqVxuRRNY?v16);>?Smdk>LPky#l|HLT02nij+ z;jxzEUnWYG-SdX=lrQ?ymn8nyrE6p~hJEb~UT5W&7MX)ulUFoW#kzm#WUDm zNms|z_?V$-nHO2TVhC~mi9h7EmAN?CC@v6Lc(g6^L;H-X)ehV#&q4Qd#@kbL_%pq_r z5lh@I(rDG0v(>sHBr#J&*gB-fAgw+Yj}c_)w~cLM@m9(A`Zb8M5l1u+Ldjjw=9mbv z7eVagAfw1bSHA_&W{T&u{+S0O+oNQV!wN$B!eDzlZ7*ILMF17m9qJq+yL?DqfZ5f| z^~7uGpcqz%B!Uki^@>^eT+&`Ss5O0~K{1PN%x<9(A{YqgZ=(9buOk^Fd_eCfAhHp$ zx6bd#JFIbhHke(IwZ`S|mQ&pO3QE1dI zhL(fHh$gyjE0TcrkbJuh0w$M4MX4f7f;)rUDVDOeN&7Y_p=_7xy!n_!&riE_aepaf=l1c@s1Mb{9b(VN- zSr7s`F#=*;jBC9Mq=yf}X-^Z2q@4>ohpclWt4Sd0h9XH`FKdbu{%E=_^%4{y_yR-7 zj!d?-iJm1`13&FO`mU%SW%_#Xspu9uXfmsL6bk|MttHJ!Fgp?h3%8*M2S$`!;~YT> z*gwU3b%jW~@pG3-99Q`Wfx4X^aXu`Mt@R;xnDhIq$6%vb&)mi*(~KFJd=4Dw<^+D& zrPpX&V%SV4BJ*!8s|vG&Y{?|fbx{XVbzC~HKa@<2bXaD$uSdRz!)%Y@+R55@I5e+l zmuNR2f#sy)V`ZF+bwZ#>*IQq!6{><&;%81unK$a7TAf84n>Ve*0*QS|bACZCL=IhS7-H>U@>$X*U&73`5^(mr<=sn}umN2aTE@ zl@Hv8v1pxMD?tzN?%vQ@n0+L4r+!t%p1suoZQUgYCx9pk0sRZcSM35W@A3y%ck14v z-#+kTJ@oftbC}~_OC<%8D3l*t7kHzdx1R%g3a#1+Dj1En)4DwAj={nrn%{1ho{hi1 zD^Q`BKyS^Cw?!Fk7L*9`4 zp2-IVqxtxTPu7$fiN!J0pcnBm$Q>MGgm^bz$XN=xmBkJ0u2m>oF*YJ+mgYJv6dIxk zL9kG$2byjMedHi*m`6D`7%oTk>R#UB{lwU@0j-TzH7UxZ_CZ{IliJVP{h_FY=jKbh9B#wN(r+_fb{aYd>*BJ+Ga*VE4z^QN+k~QJSZ=_8lac zGTmi723N{&y?qWoOn8EF9R)OL9*Z_8zIqjefV(sFxnE>F;DJfeIJY~x`-{|fW!s?9 z#1{OK&!=7tc3-X5zj(X#R4wJh>g&g;TQ$><&)8=3-2^?58TAHUKd!vzwX9ARw<^%! z$Um>mwtrvf`5pH6K@th-z7hd92BabTw2%cpzR)VCPF{yE(x!un^2Z&bGiI`fuGbnQ zS&JM!&jkJV811>bZ}JiAVg^9Iz7~)i_kuq{>|_ck;0urn5Tse7mdP^JBeD<#2QZea z$z=twxH6x1=r?INvIX9hdL@7+>wj(vb?AtteNwG`v^?Enz{zfm^fl0u{OhGT;%So%Aq%8RxesL%#6|oxOyJORp$8evh~WwW-i|?s zi9vYJgARDF@oHa#F|zisOelP(z)qx>p9S#_^Ed=6=N|5AU_mAG zDWT20u6JxFIe}ah4_9PZEPwZTFXFC66su`ubJfF`D#VhP0U)wSFovB4H*?uI6NMh6 zI3OynCVciKJ+;Qw&|UV|tI+xENAOQUoYoRdR-(nPm{^5xS`fz0^2DAvUGtXq=!~~~ z;6kdiYft7L*XZJ=S7GeM?5?puaSKCwq>^GTNS3yT>U71Ib~P^7Toh-XgdG%CSg$ya!byX}+bT+s`}b&Do;f+jJ(%3fx- z1`+A+Oo8e&95j2#+-js*Gf1WVeNDa^T;Vci1i7TXOuH{{>?4>eokZkP@L~}1kM;!d_xi-; zJxC&h=YQ`(68_F&_y@&6bZHOryOcusx0F&qkkU}p*ibUl@Q;)NC@BO6jkTVQbt#R% zNDM?2P3=G06FJ6LijX3Fz zqU^(s5$lUH>`Uq8thFS><2=aIJ1MpCxfGu zL$Ab!#tesW-*^p8rVdRv4b9FDy*(MaGEX`DUTk>DaCpUQcnvFcc%x}}Yj!xE;sURM z3Nrnb*$Fy6K0Y})IXykS7@-^)8EJ2Cudc4%+}zB|%bT5@m3C;8egwRB=|>$hk2cfN5)y`ohoMktTU*=u`nsE&TUAxn(9jSsFK=*g@bvU_ zYinz6ZmxZ?+~(fO=HBWS@b-bXw!Ob*Um|B;BJWTl?^|Q!TMIm6zgpv$OP#p`VX57| zpAUDZ*GFX>JLFv&Z$olMra!)%{g_ZbXB;@N`}s)Pv0dMrz75+~nls)YR1fR0sQ?{yy3{1j)}oATTHx8UhOq3y+A5ijIkmi%&>|Cncw( zrlmj4$jr*lNjJ?eC@d;2c~)9hUQt<9T~qrlJ5-R{-14Het-YhOtGlPSuYX{0Xn17w z<*V0Y<8Lw=r>19S=ia`XZ<<(KT3%UQTi@99NB8@Hw6pth|I_EaFNX)mUr&$DPR=l} zdrLFg{Lu+nDM?B*JA(1(lnfOvV5F$C=;ezNk(4)K*hw$ddt<3JA~ZuBbCqKl?)9fC zzK-fl5pvp?Whu=ACN?H&DElkdl(S_bVc1Gm>7x1j>2hh9T25GF8>g*B zCx}ik-q5Xbxj!7u_U+(d)C@4WS+#xZhtX!w(;f>>(t(oda#kFaT zf*;@2_-~SYND*4BGLC07dN`Kr)SF1Ek#KeHlk?_y=DoZYDtxD9g%1B~=gmzAuh#rS zj7M6$JN9=L2cCM4i@ATI-#b6naGO;-mz{pZMC~$%qA(9ZWmK5*+2dc!ko+Laybxf2 z!)hT=iY$M@fs$l`#^KsSIG+EUdm_QjzFhF6JhP1!nF*&yz(N?`8|Ev4LMsK?tO8%w zFrZgaF_xle3CP|@!z$in!>)g+p*3dzqM;S0bG_>t9X+Gp?U*`@&inSpS8H?OMiJIY zF%~ZCi^+Cpm5V73a+52m5q64}^m3SbOX&(LislI`AlUsl6-Ui?##B@rIca9;>-R$@ z4$)xr;t1M44T#8<$@pfBp6lO#NdzELB0*c+H`a-Weq zWm3wS>v~=iqHR`r@CkXqmR|{Du^nxJz-Ed5fZRZFy7#W)cFhz0V)L+Zm(XXm^R)G& zrEawPM3T%Zh3~UbM3Tv#KUc9`eV90eW&d#Ztp{iG%P*UpRU@eETr%e*cIEz0Ir<-D zeqx;7X}3|bdDR)0qnFmX0~{#bH*kQh&36uVblLqp$s<3b9Qt_Uh)(V~=E`jRj97Ka){0GTw)39x(XAxz?BUaM zl$YrsBeYh;?k=BO3*j9?^3{)N!kq+3X<8sbBZIM%SK0)~450EVWbbFP!HiSi<9W(s z%ddQjUi5wHG~6ped!*_-Ic0RZk}pja^U51dDTg)tQLnc8gZOeU)OqBjR%Ogjq8YGx z#uPBiXKPmPd#M2{8wCS{VqEZnCoKDNIW>1fq;&M1Ovje8%BM+Hxbp1m zxsWXc8P-)^vg}d%G+sLk3KA8qtPs}PWuazX6E!phHrI&q0@#| z^LkiztrL?W%YDVlBYqM4)NtKX67^DY7d(BHtwJ0F<#1SFf*dlNFjHgg!CZN$tjf6! z6Q`R_61Q_)BVUoxp}tP8UrcEMO0Gj3=|B|^2?Z?tdaCh2gcFIj%Vu_=Exr_I-?(br z{XI4l&|xP7B8fl6cnLXw{nY!jJJt2!tj1GJbhlI^yn0ef%nO6n#@*+Mu7W#V+whE? zdcQQ;XFX6WHyGdE_@qP3kkvIyp-uOy>dstvk>(>P`Ej`Rz4q)F=4Opm++5#Qn5of5XcQ`t zVq>vz$50njAN3|HA)66vcpt1J_|WM}q0nhT$Q;a#5G!Z#I1(RBxNj>am5ed{eMab8|FHjxJNZAJCn!wfv$ zHEguJiM1w0Ymu=jgG+b!kxeyK`65c}lCKJWt~Oq&s++HsCscOp9B$N<)k?YXYDII8Hn_tsMR;H8pi@<5ruj+1Twpxk@+d+2Jih&UAsyEvw!K@mK(L@Ha9 z+zqukb74v^Z`mq}QYdt5S>D)Vw-K9&9cpN{hj>q?V!`$csndA^}&o=z``upcdLA1;P0p|BE}qMM$RHwoy;`g9kYdsekCA4Jp_?x$rM zELOEd2Yo5g^O{FkJXsxU|Kh+xWjiALsUWkFx2EsN0V=6j^YZMup~|LY_Ucq%;zPO= zL~$k*uBpwBjC{deXXXiI?Gt_uD1$oDE=i5vwS0*TJtpX%?FKAo^?f2jWxytx38n5` zgPR%29Pfg#fPr8{!PctAiCgxR^#if%500r^_gWg?GBI&Zp&@+HsLS3n6;c_H=Zr`w zJKZjDy}jI1p;3w@wiuQ8xyq?UU$@8AfH&P^>m_3v8Zf_jy>JWt{NAqzN;8W}FWKzUnT%s?0QG@LOgU_gGM|CrE4o zlIhre;TJu;N<_B@Eg@WTZl_&O9-}=(>$$D1^C&U!*0bl%fzMTh!vy`5rSG0ELV~*? z{ob!OwH_P_6u>dnZXB=jdP zKgVmsEOc@*BrCxatu$8ugLw(QAF8v3qm(mtYP_?AlPI#q_f#?mkQXV?61o7QYmA1_ zTMW><@(zi8O!2014cb-<&$)MB8=%%*N?Z|7%0Ol@IpX1wj!IEa`0;3%teN&@%X7xf zEua{b!>61(FV4kaDT610F)M%Stld@bn9S+rbC5uWlp+5idErW{ISeq68fEC}QQAdl zQJk7dl=-h_n#-OttZ8@DWE!?;#ZPD>D;q%V=5ym{$cAi$a$tBD5Ni#}6z3(h@BL?I zEhhW+frCMEF8GEFd@*=mtOHdO>CD6DkaE(b0Xg?6Yy%n7sX2V2nSUrVREYSu1|8ka zvqj=?EfRgV-uZI9TH^7MOcaBhf~Tk&xua-~t;PAlQtmO$kCfra5hzc2sKYtMjW(Jg z(MUH5fr+}CE%|*%PFThv-;UFf5={bV)7)ANP#Mz1E!1d(Gv#E1MTgT0&GKWXIbR6k zxq{L-jsfh;g*Y{Z@&|ERP&XkD$FF9E&(0y7kAbC+$>?>g60`|LhS}3$zGO?mWCVHn z-H||(yb;1|@%}47KU2+WM8TPG$`ozPVv_T7)bv%=a?Jr1IuoWe2q4!jS76jsL>+1p zFBHca7tE*mlrw;Phjhsy23D!sk*?*HV7a2t`j)tb1}?>H&4CpHRr%t{^C6LNNM$#N z6^PH|D_YSZ$0ZFsl^^**7Uo#Q%UN~Xr5_lqWy}Lcn@v2Af)m{WP)v);`mHcWRPCK< z-jFIjAgSSsR!6v0v!h}}g8d4>AxVR1c20TpjUk<&3ay#kZsH zbKYk40#xLBuWqIwPg1Y}PzvV7i9k34dMu|gc}ft_Df_vX5g6A!pw#<#=o+*pjXyvm zP&2{Wt<9AA#v>d*kD3pjGG`boWSyGlO6wI#TJzvMCrXsBpfxwi5HfiisBGHUodwl% z@Si19tEY0@Cot-L-5GCdf`03EI{Kbj!-jmRfJMsUNbx*e12yvq`HkV3o=kqAqKICQvr4-77UA56;>v*K~M zj`pd>`Js9BpZa{x5Fv9R5e ziV01Zd%3qvu-0)^_A7~IU%Ffya`?<#S7uCqf)#2UbD}qD|G8lO6GQ9yL11TVW%{kJ zxa6Pq^p>l%;zktrp0u_|ugbhbM-LrG(Y98-xj!{pLr?|Oz0g0IUa=;gBf!sH{N;{Q zNCTn6^>kK(fy~ZF(3Z^p8o%FeMUUn1ZfQ93oi=zq4D#X)89f%uG1Dl`XU~6Hw98sE z#_KK`q|2Lx7e2AO4`!YA`5m|Rrv8ZyEnXPzpJjq(r9zB9ccK6a3bT5y{0BwJT^0UUt}k4iAKl4ZT%5}mL6E=1IgmfBf*XeI#ek3 zw%l192Hj_&!)-VvQZCcVt&{6{O)(@xkTbDwCOxX}tYfnOW-_=4}zr5e(s3OuU?LNsA5->fUaiaDl8V=$<)GF-pcLW0pHefPE=&m-pOM==fA4%Njh_uFqyRXZ|I&M^vbuIY&lPOxj=V0?iHqpWl5c8Aq8f^`E?lyGUcr}_vvhE@Nfo; zf1*eWBHY?rk3MzEFp()=X#5`5b-p?eAG18%zB;EV{U<*`mz35Wf zIa3DUW`_}Bhk;cBz>91muma$fkWo7}huUF4lbdih0F25_9GgvipG};~Ez)xU2J04@ z2mnt6@M671;Il=Ev5l{?O=q;8#J;|#yNN54tg~CMLJ?g-AWk6N&y5L7z84s?8Ss4*Ick%tV@qp&i&$wBIKE9UvS+Zq2h`ic zQQFh9*|*5rHyPji{2m2h^Z`)Y01!Gh^?fiLD*^BrFka;QVRi?Qe9Pam0FzMn5ONo) z0T%D&0{hf|E}q)fW`e9z)x>-p%s8fzUi2? z2@|uA{B~^6v2XddXk@cjQ@Q_ReBWxEu)1U4OzEIz{eSO_=gCDLTMzCJ1KbpE8duhpau3$>R$0&3V={?ydts_C+w|X}2CGWPGDv?+mW$ zXk#J(xT*{K=riPvW4w-o>8ML)-_!Y- z3^-yR?UqCOR0p3ymlXcutLBw*(SDC$iMMkjmL`WsC(`2$ zg1TK%>WvFak`|j27Uc_^uXm)KSM%e?0bK`_oqy*leT2n0 zM0kG%{oM-cx&TGLE@Z!**zT`nUj}1tRa1xOc0RTadT)eYFKax#P(1muQ}W4Znz@Cg zlr#@69zFnw82FJv`ew53VW~q5&F-=fsq_$fIjskfEY$rNSzr=!yezvSU@1sXBIkL+ z*eZas!c=Zq8V-F=7xa1(WP0%`H*c=KI3=|@3-^vd-h35?!6JWSu@SM+!>IJh8W)i{ z6K7pK7hz-lwIi%45;eYFwbto;h=NisI+61=_+XPIh7PbL7J@q=n~;JP+YCnFx8Z75 z$>|cGbxRcjaTE^2>T8i(Ae&%~ij$k5)XDPy23KQ)>Hxqm&tEkvD)*j2MQjQLIc~m{Hsp#H2{KjWt?XgW#p^UB85C0(`fH z1Zj=CaeNl#~R^3u?sddR_ATF5t z<2yI<5B!+Gv`2n3=s|=CDw+Ml!L(^}VC@^9na2^KJhvdujGuzXs&W)yX z65i?U>a8Or#UE`e65#|seTPu$9RPSBBbjS<%NA8*E-R|`=&q}d^W~-^jHXnab=~AC z_KnVTtsHi^*I>&G|Ine@QEwJ)$_#VtlZT-}EkXXO)o;MTKr|lV%ItKy<5X!m`1X<< z?;qaIqL*DE9w((U=?$y3>AUf?-$27N%DD+1f+aTZ7+x3M#loJKvIeqxZjP>}Olucs z)~5qrI6fyS^U0*vmDbSvhtXN;c)rmu?*pdI4Cc z{H>R*QMWKuwQo?ED_d@iFIlkpstIf8wEAL~YZv5zKLEK40i<$$SCkLJzwE8NCgR9% zuKH`zbZtBWT1X;hanau{A_#jqQmZwkSO}7^6xFT>AI!{Z6%z^^rl?0fp<5*r{!A(& zj>P<8rmVSS5T|RVZ5zZx*8E8NPv3EK@PMk{P%Ub3n|?<1#PkRzE7(OCG3OKI*7k9+ z4h6O@kD%N(d`Zas5cC~NK6HNA*7JI7CltN|I0_TL`*?0czlz;P1>aL+Cd+ogM3u8^ z+KV+e;e+430VTW|WK@^4fSKLJd#?y%;|4qYGG#7gP&OY{fy6Q4%*@~O(03;VqbacW zAl$$0OP$VDYq_`OAYI=yF*vi?jl)Y(+0%;)qYsAiSjA_2FzSekoJ_2njKNWClCTo^ zKt+?6O$~Zz$q~McR}-Y+p8)nPRybtph!zTNAS-XxIA*b4|9I2`@$<~*WjQbzas9$C z@>^EPS*}wSiGA$%9L5$NtzTlccNg#8J&nfa1WhM}5X-!sQob=QG@l<-ggXqHw_NUq zqTFIRL1vwg3yS5qHA>GI3g&UMhULOj9a;0;zLZW zy1p-}R40mSb8iERU9@(yAAV!<6tl_WRajf@cuio&S#-UR_!1UteK79p!D!kgALN~i zHH5MLV65^uW?1Z|g?c3jJFukFDR+*8Jkpl7Q?CBCPpYxWTtRz4R70%55e6G3@Pl3J zP%9Ndb)R*0AVpgOLK!wRXf1!!#CfEc?XZ^)*?_Nw>ZDMZ2 zs+Cs}c~(D1)hBAKS68S!8D;RJHIsY_DwF)5esUoHOanx2g;b%MhIRwn5fF%Ko!lxY zk{D_zEBC!S@qSJ3CwKhBS&N*(Q=)V{@BCzNLN_qa_Tj*V4o{}GNao6bxpnq^IOCkl zp1Xa#qcF84Br$74Ols?NHFcDamOjk$h!>~BeS8M5#zfZ5q@rJ?K4~!3b!JqEIC-v~ zY&csiDx_ZSZgKHgtKN#=eH@l1hcAm%!rw5iRN{en7IIVH)XDPGrZ`sEN$A?3G&PZ3 z-c{J*&CanUo?`^PHj5VV5WN_Jeu16Xwz8J`$(p6%4B@IcmMk?VWN=v&zJO-+b^hRt{|GMN_&TAqEfdUTjr+wyH9P2r<$ zYCWT0XW_+fi`D+32P120a_KdScg{Un(F)Yl6{3&qwKvbCk9}V=)I}Z{K&vB&gU!7O zpDN2`Y3J#J%{9uAGNkC8mj1EXTx)jJ^bCl1A4!w6dfwG$ZdvLRL1fQEXHKS!V*4)3o|~4?)o^jFOdBm zwG~-O_$oNqFO-Vxtrs#e@b5{gTKt*0dF3I`>UEFnVqod8k z>ElQj0r(+DWrT=f`Pb6M>R}=B7q7Z|E4+oT@<^^E8ZH5paIJ;kx_AL3yzH&H-;f%y zSaMxH6aXBNyb*W3$PWDhb^fT0CSkO0%gNrOb$XvmfW=8ay+SOud2OQz^i1N!5()SP zXw~AIZ1*(OP$})AT)iXlTu6~Ta+AO2htluGN%)4Cm1zJ-2+VPk4dK~|EgE21?6m{2 z^tBeT^RWEhE`tAP0K^aVgbd7KMdcTj<8MdLYXgY* zCoguGeXH%aU?>rNE^s?)45N{;eJv6iEDRvUr*US3{}w`2{3BF?4G`-Y#2!?cLftFsU%n9RAj$rx7^g8T zq_;$}yCen`nGDb(Ik#XOM+@>^aChhP9ywH4Y<9$F@a8ay7*70t*%-^H=_oX4BGl)= z83yBsR1pT_4dOs90qut|AA4f7hZ1e$6YZ~!g>hq_zI-My=N%oCqG|~Qx%n`v z==~d9PALvsQ?nhv2}oP8m7R;5k&`x|+WP~*ES znCedy^1R2_OJvH)X!tIpCN&L)Vh|Ufoy~hvL|mxJFfu{p?+myBx2$UWGD7TM zdv=+GN=q#Qs%BK@GU}jt%pBS1SpL-7Sm)?cKxs2L=$nRdU;;I4aJ49v;AKKHEpm~0 zcCA?UKe1C|K3~6-_t9KgkXDZ`3_FM*@H&CD`y(+@7qpY0WXQ22kVxgj1GU^^57&pp`D=KC% zkWM}6lZ!Rf&dxt6et9?3!1!0`fW!p=rtSTpXw5vjn2+b{Z&<8v#VnsvUnnaXpqqhtPdm6NQH5y>~AFf`Je7A!xR6odpY9VP`DhkY0Z zl7Zp`<(*v9YZW~zR`*H~Q6hanoFZlv!A&J07a7Ng5Id;hOB0u1X*sy7{`Z`=;Fe!q zO1S|}18jUX8UWc<;G@4s;%^`V%PCq(+#J3_5ZYswKy z75I*rr@3>RlSNzd%OfS-?qeigdzC;7do0Yy5*QMs@S`KNQaOMj zj(_xbeg3DGobZlpgARrWSGt!Uvtdc)D@pqy2DF0WJ|vO*_f4F;$;pnD%p`UAL!pbN zrAK&m7j!bw{#7)PqA~;+G=h&3D{UyI#4+JzxKSLojf_%b+dA z^th>sl&CgeGiAt=st_s9P%>W1Q(cKJIKxOI>Y{;f6t9eQgz0klK*xA*v?lo&@9je} z9E1k;Gc(GFExKWlM>FmzUEa~%6VDgXQ^c65AC$L@uK2K)1YrfQD5hR%` zv+#GW2j|3mn$&V{Wc>*Um}g zEWB#OhLtn0u&#r-N=kcDe($u@71xo;gNYhd3k0T36EDSHZBB~GPpucsO0oP3WWoBb z*e>pk(d$0VIJFqBh#VWax@EFjn~~nEI$zJ!Q+x806|^`npm_frYMI&iTX=N`QaEoh z3QZra$?@mWPTyu+Bf84Komw~wtMVITr6V><)me$7{2I@h>-7L^@#+17@$YDyDG1=UpiRN0wP1sCt25z^m>F9(l=l2-^15; zSz)?5}Y5u=*_?A!3H*nSV+Fj{S?mO z5U$isTO6NTy*`~l6A4v=#J<;`tT`yo)L!hO9d8j{C%^mwX&_L#-+0j8*kcCRzOy&4x>Fi$wvkqjA1+^v zo)cx)HTGY+Z;WfCP00N+EyxV~E{E0*XbyU~R5w61OKWbgFihyp#bjaa#=1}^s=j^L z8Oy{&e!q6GeX0_>&!XDo%&4$p+QnB+8c(@3QSu(c9mR%EZmsVDhuv+{6>D<%N-e_&fw8k+F`f^EYv z0#lL_;r|=9EiC$P*tSUj|G~DcZT}m#g;xDHY`Xx+T3XIuSzRbtuh@p)-8jfw-(A^1 z$~?=y$UVI*xw_lBE3%v$9r}xp^{EM421$fiUI^ z^}r9*AJ(&r<^@A>^t=$7mRef*r34y7Xaoc2o_sR3EH~UhUGYEKHV}nOj!Ua9UnZVW zclbX>`S*MjCb?W>C2bTCU0YF z>zJ)h7dX8B%i99!wfO4QNV3_sfuu}x%4+R#^B&q>Mdqu_=BWk|%zYcAFzJuUxxLr) zZGqiXGEJ2cv&`1hJe)b!=qx&HTvU{f+UOM(q_MZ zu&t=DIe855EOAd<2hU_qd(oUuSvM&1ouEwIH+Tput`5J|QXey^u&_oBpFzX>EvH3` zK2omeAOp0Nx{*O5ijtS-=s`1HKNHuNQa=MOG6~Tv1?9CS^lLK|GUl~X6qfIo*?up% zJNerQgA_5_@qr*h@%Np#)QO8FGVrwfkT1h7RfUnH#siJ%<)UxGXU|Sq4Vt)HMv~3T z+%1Lud|4T&=V?-a+Ksq5Qthps(-ht>Sj=-jEPI`KSDR#-1tmi{pOTpCtZ zVpXjFqpFb!?$R`^xSiM3KD%8Yj@Mx7RbSRpto^+)V5w25M|u~lQ*=h#-;JW~8si$y zydkF1{j@jeMCaJ1!t3!p~&QHND$;_xPAu{BAmv*mB+hDH~vWf4L#J zetT7)N!Gn&;MK`G^C!u=+-_{$xV8O#F1m7wuh|sEUH$vQ9rFTRXVPL)cN^`8`BTpD*}Cf% zD@Gf?!HZP~xQ!4FNZbkPA${C-f{f9oIr;rW;s z(YwzT+M@vX1DEA6<`7|^k>fykIt_95vpP4iVnjER{OS@u!JJ!O>XKywGua9KTysBH zV_c9m4Xp5iR(i%ZZt)p4L{InQY{%1WRpVlrez3cYt==5*4{MX3vBX2L9)<36FCX2q z(iB^CXW4t9RvQAypQDd0N|mj&>6DQt2CJ{^3lxm+wh0AN^(G3k1QqShEf$s=t*V=a z5%gV0w6pi`n!YcU#(8VwD{P%l__oscTZ)`3t#lZy4?IOSmy>bZ$Etju-8SAn&!u#GpF5$- zN}P%$L8oU~VLmbS2FWgjfqUsq3f=ms-vb(;q%JMq93e%I`GzZP3%C91Q&f@di?`Foc3sxK?i_D0t$tNUgd`^d{T;r-osKQVA3 zYbY!$w$?USu$qTyMbPDrKFmIrU)%1D9;@HW&k5aK`o@MHP-~PAg_6~t6$k(g$>XL2 zvL?#Wfs)rU{4K-JBx_(r#YJ!EyJ;-Q)P2Nton78?bUG^8YASs{zV>mYKFHZdSMl6o zLuR#$meoJ{WY-0gq+7ezq$0D7UBMtOw@>$im!$n~gf&O&b4Yz_cvHE4X5fklPL-_k z@KJE~!;Z8BSvH9kDT9HA>Q6v^^D%5#;UWP@oXYxHY5>%@HT>MfKgT{9+6rxfl$ zx60Sq0fp&dX!6P8ebeN^!UCtyyxXbK@X0t-n`G9A=K6qf{LChqX!ZMFok4`>ONcfZ z30iys!L zMd445Y;#w@UnJs1U)HvT7>1g+KW5*4)qA~+NtWBOXF3rLpLL}%EgyRJHf`H^aS4>V zYBj(8Ywhtqed2g>SU@srD^*NdTb$TzgTGnwM>*YV&W3baG-A7uZ2j*=B=lmA*ZrH+ z+r!{jxl2jbyRievNwcSjx?RO1T$xUfiZ!!+D08`-QiJeElSVab8}Cq;M!b{A2LevB zv!o)ni#?jZ8>mhl8=P6h;S=vyk=ZYA&%T7u13#w+Oihq5d=a&}8aCqLPlY>I0ADg*qmuhGPW0 zGAP#kogqbwwL$BXOaL;kg`%NvArCq`pVxh8;Jsnsznr|jUx>$^Ae;&CvC&-;!T%#+ zu&%Qs45FvRpne!)@a}@GO>f+iut`%Dzm(8kb-$opTQO`$iG5W)Se5~lnv z0~`(k%V;vi>`U~xMQh4{v}9s1lOqNP!$LwqS63txDe?L9G3RwLBC<9@k0D&resLz2 ziFC1q34&et#+ZC@ct`$(kAWTl8qpj5HQG3`LVt=w6(=ewt2xnJC0(e&u00TZCbsaBPPYqkwRK}>>zmY0Zq}2pT#Un?aL>VIHi&+sFU+re&wXfFQ}HAOA_2F1AyBUzoyJ^+9@i^U;>hPNwhtNo@70 zmdJVn{F#%QX^5O@i^!VuL=NS$nvFy{6-2QKYDR5I?&<~KPVu9XjKUp?CCI2#p09%r zT_a+LTpJdAdI$OYWz_E$?K0pqGe9U=3|=d;0!!{W-_&xtjH0&bQ@S~AktMyWQ@~2@ z-(488laR7vxzhCs^yM4B4;Lk#lz32rY!eW9*QM0K*P?e1!NWwp@q|g^r4cmW;dngq z`O@+Q{%uxK@UqhUC-2XSN

50AT=xJZB#8!$qLK0d{r z&v6z|pam)V-zxe4XOat)*+I+NAfzp65FZbJH!~)4q-WKBm?4SyT~9j2yo zYLC44?R#%&2Olbr6#PDTz!lJ&q7)A%2@s8 zQA3zsL)=>PUlRHMIONYY?P;})C#ZBQwQS#i{yBCI@+jGLFNtSNWIo6ZGEXi!Vut$= z)L#^?;vU;$;b9b)oQLG2eU;*KARK-z`oIvAdT5@rob4v&^(DPQq^4G-n5dSRfY*{XDiHJ;-p&hO6lS1=hd0Y1Ht*eLPBeKs&x|3H z-6ClZs^^H?2u8)wU1`?37SV5Q4$yK?aq|-;43$|asgZS&AM!wv%q2_>$AhX!$>cev z3!&l0dR&*+BQ_j41y_ulLs8MAR$5^kzs8c(k*4urYT;rJN7|y+ zOEY{smubn8kn|l)NW=3fHVboYN(ZUCfaWUt8HlP-L9^;f3CNW{o>N=PT07D^iUYp> zDQi_&vYF+z#UjqK(ryd^M`fYL3WrK`yE|AefP*FLr!*?Mp)C!_^~-pkU#SiD3Iepv zQrU+~siiWf&C#vhI~Gbih`nr1<>3fNRn4@!xXL1R2GTYbjkIXcI>iuO{pq88`76@2 zX+6FeyO3nOwKnC1Q&c5J)4O@fQgcvqed>{Jn}d`tiF811P=TjG%vso~&0+%1aDW!9bS~ zLI)Qpj>J6{)+<=rTQ=S6hvvZbq0gtS&IQq1=f1^M+ecd{!7D|wN4qCt6wKw-ccM9Pm@yX&Dn#UHNHeC1P4wlDaT*&mcn=(~n9+TTgL9?_c8PLz6O zJSApk3}u^li1Rm5W+OCGkM@7 zaR`{|FZsR1bhOl!Y#>l!UL%;J8BSrRn>k(aX-;>O&tc6o@{jhxf@;MYJ8yV0O3Eea zsP3(83z>?s>mqe%SciL;p{avg_+Sv9Usu~F_{``g|5ZxWN$bdA#gh$}+a6ev&3x-k zp7oHfptY5wG*C%yRb*R#S_|27;|ZCTs&L6^#fSk~UXiD(p<+-p$~vZ@=WGxCJ<9@r z`_jbx1}Cd@j_?jw2Q{zF&Qrn8A#kOwU3ZjXZSAxq&5E}x>S>39c*ERs%Uh)k?AV_c@iJ-Wr_oK7!Veh)wyi*E&DR z0g72yqin_a$jC0CWZ#OFdB@h-YaR0oW1z-{ZkQaOa7W|HjP9FtDb{lAI#s;V8ocq5 zP>XlI&FIIp5n{w#gV*r~rh^>mY5MXl+pG%r6Eh!?9&DwL-!pe_2D5beC&97iCPzKe z@t&+v)=fSd!>^OAtViup3xWuzw)~w9Ys(?!r)pnDsVWlk7F4g4{XO}yle|JTJcAPU!pC^zjT#!$d>q0FsRM7dX=5L9tY~qm1 zy9t!G0>&L5D5K9nXQD;x0UiCP4^eN|or98Yej|sQBT2<=hsgJsk>e)J7)SWZn}mN) z__H>bbo=^j+7R8E61)Z}CpN7sV{774dT0*#vL&=8pp$du5Pv1Kzg-y0C(mR~f2l4X zdWWmLZkXe43U&7tDE1$srXzLc;B0R*l&-)}E;|QnC*ITe4<*lKS!G!s5oZT-*xP9M z*Oh+Jf)wYpDTgn>bK1T=@#sBmn;SbeG6&^lr(c6ktle%KJ4Kx%M(=0cs=tQ(vcoiW z!M{m)i`SiN@t@_w&gAsGy+-m#MepXU#!@548nO(mzFwxm?Po{d5qe}nY=NT1L=+AI@}ZUrxTmK>!n8f)hMADrVPLAeM%zeVg!ti9WecugeR?+QVXQ`gdx~y9$}bh{c>nsMy#s=%-~TabOX;Cgs>l zi|zU{So^+_h2Lr8c&Wjamrc*k^LBp>nXh{DPmpwoPy}Dic3%XEpbwh6{npU4SxK8$ zD&pQ)25KD`&&i^CLdTustj+O2IsM0dLx%@f!}Mf&HR2iq?uPYLMK@vrdYP*=#Z)&7 zrCwC}YV+-C8kux*Mf0xDvsRVO!sPAQl8Wa$hR93);OU}p3|)Gz(N-9n=G}k-}0bQZU~30Q689e*eqX+oRRR~CL@e_VLTooNs@Yixk*v& zpZ&t37KMk;B`?mU`|-IBPoEXhAV(BxgpEVme(Gb8Y%@#~JW|z4vRkdJVZQinNBSuG z^6V*=5mSeLM3l3fDW#Rn!g)Qx)cj?e5$mRPtsoW!8AyRq^S%S3PD)|s(;nnF&YZEV zq3|T9_@^RZIW+%w_hzM7{oV?mPkR4wuHC7gB?S%G`!!~Fx{RXW)4ofDMKOFvp+uli= zA_W<>*iMkGjEbIEv88`)3a=Jvg{{b);(Kr3^RGBYO|L+(1lblRzE|TPS9oqH9(#Kg zsVSj5otx7|{8d0B+sFS(>T+b5U5T*vF0FRkFWsZpUx~=wfQnoF3qNq?esb+W!(zAR$)f%_?`nFJ#Z42#h zs`9ltJ@3Tq%s)L54_v*TSif2G^IY=$dkOaEp^%}Ek^XW$YN3!p zrcA5|&|9F-8UQ-BtIKlskE}Olxu5A(k(YLJwf` zB1_nD+69byJzWV*oYHy6AW!;H{S*qlDW=LZ{~lz);qJ&CPAX8bB&5Xvv?@biq!g6B+06R)}W| zZXZx)bM7t$QT6?l5ppMeKF`@Ij%v%H4h%`V;D$HaFN??1wOHa;eWj%xl%07u9|J1Z zgB9Q{A*$dX?#%$=-s24U1E5&tSdFGY#fj34{cM|qREgIa(`m=g$$Zwt)Ii;m_2x8Q zg^t}kg@)~_51}f3KEwKvE7?S43*U4lZ8P>>B!QUEYQ0iZHkT#E4u>DQwQZ1`yq1g) zPf=D{kDVLSFUzcL=<2JSS>5NLK>$k;<)To z_wv;L(R6jh>Nw@bmWmrF3w&Z$k@xgLRud%J%aA7aX6q`?y@SlIr^Y7X*E2&R65WcY zD2Vx5w&X-_Ul|XN!l}aIpT-~kj^LY&w>h&4s46g8PNN0M;G_0;ORe|0^}<$Xi+q>q zaJ^rZuDt>7RUlEW)dU7EQb!30e?6#BaG*>C^Ssq`= zMRcUqFrW_&o75!VaTj@^Vky3D8Kx$2N_v}l{KiwrQZ{qQCH1SNP(7rw zR@KQk`d)44hkrsu!RWaQC+@P11FR3b0^$imTlh6~EwtD%&gL~(}A zChsu~R{3Jf_NqC2OH@Hz`F_o#jPmE@b^?N(luVt~hlZi_xDZ-tZfVPytbZbC*mwSK8yfw> zPr7&dy6*;C{z?@j_k9dh*Vyw;$G9z$-1++^)lmDaU~IW%74gT#zZ$dfUoo`dt%eIX zHg(eLbI-l0tEQ31PZud8IyCxvHFe0I*BaONUF%*31lFtHVMUx0eM2sNjL6jJWe0cU zeKRJCz*9_(SBI)>DGMg%ri zX#Xbv9Egvv>WOb^yL5y+?&eoqkNQ?EyJ}w?k}z!HAgxbrtlIG@s$X)GJKTQq^jJn> zt6OG`Su?@+MElG?RIc~RASO}{i>0-IxzjM@J(gsr^Y&2kv*UeW#tU{)Bb&|}IngnG z&N079r#H~zTf9deol;hUtI$uQfNN_CX;-;@d*ysX=(VCDmyh59CJZju#XQzYP$;MFsdyeBc4@fh3PQ%{zK4{80)W^gd=N9POY#-Xkis2H(`u@eD$Q4iH z{)baRaKZv(T#Gxj6u06|ad(H{PJ&w?1lJIx zI0SchcXusrh27`fd$09jjj?{ioMV2t=XoDjq}P~b++;+YbuhWjUgXzrE*$YlRpR)o zq<(9C!Plu0#Nm<>T7kD8owCCsCfz-XQz=?VV zrc0KNW3h-6vMl%KDqLg^ z7;Yk2#S0)Mv!V2Iw73=mMg%=ZypN7>p6xJ#PD%1dCWdq-IxQrM%7B4YapSy6fy&5) z7^HY)_~Vi$3#fimKYWio%?Suz*g=NN0A%Cd zEFcF(;2a&hF<2M%-wF%q9Q`YlZ}C6*Bf#C>Sy$=$+SW^$CIfe_R@pUZ8`@S zw7Eg(s2a?XPRkZ==VU%{SVi5;u@8_&E?MPVo@bCTqHhl5#1fF|BcW;8Mh+{71pWI+W{P##%a$fP_^nL~h!Bfc!Kc^+sB3%m{u0$-wzYGP)I|1w!&5Jw>lGt+#qge0&K^$!x? zhg_7OxuS21Gb(et9Pt@|NIsA}l=h-5{5+X7JU<3>LPz-DIQaE~RXVWvOM$hl4wRlh zXj@Zg+cLTL610N`73)vt=uP*S4R86Pq;(6JX$;aIOzH6`DFo(}KuXFOE7lY6L<}o# zffe^96(6kewa$^W0{;DGN5U1O{$z;XhKO@1!Kg`#7GO;8{f7mTW1wQ8qGOd3HsLis zq&gLWl(marJX4A*j14`DMJqnK%`ytt%m?P?CmfOp)1@=cnrJA=M!eq5hBhK;W>*Ui zRSO?gi(=J$W2zCCtC6s&kpk7oz-#1&Y7~xYl(1@*nQB$#YSnFOH34rfOKSbMEAY-M zu(N-M1R~*s;6_*&9oFcUk43d)7rhuwY}FO(O)}oCQ;veqfJMQAf#=z!87cT9p`A{f@%`Nj}B!}%TFJG>x_g;ay zM@gthc^R+qOZT~euJ$mV-f+(wBLxE45tRBCWFl1EBBip8 zB@(__b_@+vQb*84JrcLx_5^%;&(Az4s@MJ4j&q_DS2&)bgFQsR)LUJOrybDyklm}* z{suc3N7Piad=T3k^czdM=u~M2^cVLoM$}1}o(Y ztC@Qr9DDz^SEyv8hg}WzGmDRwqQ4s|l4cmn&K@Qx9e+1695S3V>Q?{VA|%+$>DH_G zmrPc>LG zg)Rw}PsW3TpXvY=&~E9IF(-_1dKxdrf9ysMN@a`ZdXoKH@{NoI z{JEGvecHq@RYqA@pTFP@ix(b~IGF;E=r5*?d3FbJ%#10WWGGYizRH*-wSyPv*BL}d zXF2BL2%^7h>N5|o^wDd__GRxH_)IY{@lCw*^AYe{IH+Ubr?0+6D$voAV{(1i5XwDY z`F-jG*Qf5lDQ$}$E}`#pz6%GXlt}C=VdwK7KP}9}P4|;voV1(H;4BI;ES~EQ{rjx7 z9#h)uQoH*MV|5^x3nRS){eFvs?8C4&)R$K~Vl}$rJwyAAB68{L#sH$P^dx-b6}f#J zvqyKvt#@dl_$vN|(DQ$oc`#5H)QNiJ_N0%|&p{fsgH5aqQ=p*mt zr}D@muUxrVXKc)Jf@p3J@296>Q>QiMt97gu?yJ$~8Q{NSmBB)@DM%Cdj=8JTXHirm zq%X`HaXJm}4dTDe4P5day0eU6@}7h+G3hH-i<2BO6(qkqTQB?sx$GSyn@wR-nLZfZ zH4=#AAlf@u47(6HyJsJ79NmL=9ie@E6}gU#htqqS(H{JSQL_;Jb**wTX&V&}R1rQ$ zHLT|&cSVWg`o$+pwV~`3yrB^!i5@$EY|?qxfU$soIv$;|&3a+QpYvr~F+=~NuYOl3 z`G(Oyj(M!)N+kXUH~r;j5tb9iT*Mj{;225k_*L5knGcI!lh#lBjniH4?CX;d|Ax=Z zp|{0$zFih`KAt9_r&P}y#|XsV`4)IgHB&&(C!RM|%>5T;{ z?p*X~cwb@~(TgRNT*37O4;=*sJt!nNG$;}n85RYKLi$g)jYmlUgOicc(o>+>$&k$4 zf~?f+j1nj}yd3tsqN=i{x~{gN9?{qg&q|B^KXuzt!dbzA4G5Fk?+gK9Y!L3Df}ua2nn-4k*@W( z`xOaD-?p(MR|Y7Ndh3ed6;R_H)ocd(o8*qf%vb-ui{3~_};&}n!Jq)_`JZ={lB zkMl8Pn#dD+@8_~JE)E-Wp57M4Z!JWSMk!g>uS&CilZh`9JDf=jl)RTR^f!389Sl@g ziCj(HGzCyH@^}RyZo*iZ1e$}BjQ&~4L8|GRA*oeoeI{-(P72%5^li5A? z;)ktWlxc?zkxefVJZ;L4VLU4-a`(b&KCOe7w2GZkaH-;w*tv$jp)0$Zl>dbi9bwz( zb`{~Qz=G2qts(Ta4_L^o&9xE=t(-NfPt6<(*wxxDvx`%6tVZ3clp_!)*gi5VTL)+n^}3}ph1|8OvLvNRq{`~ ziYY!RofS9BzXhk%A99>Ld^p3(96q0+`J_(aJnv8m;`|4a`aAPvXIdQnO-p`66&QH2 zk|mV7R+i@BZ1ss#7o7W7qI{7~aqj-~DV|sqh3W#op+H*TZ9>9RKG)ym0c)L0Z>8$x zuJ`M~_THE2>)1tneKMoYkW1*kJRRS4&Zs-$-y(L@9_Mj33K*#TgszXFOlMkc5*3S28?`bZhnaCa5@4=a`O{ia`D5O zbuFcUT~$5tmJNnK1JlI`=h^N_NerCkaYgSEJ^P2X&pQ{X>FS2^EZ@NFB?mRQGp@=U zcant>^f=6h8<_4RaD&^yx&MOOfF5K+G`acC=NaTMg;HaZq4{1y9#=E2{k=M%Y#hN= zxP}5H(}Y%DUM_RB$2KwhFI4f;(^KJ4j+KT}G<8b;A6};_`N{G$4X^FhB7e!k?Pm^w zbq_b+op8z>MMLmj6v<*Fxp;{%xls>#<= z7FvSwf4Q&83@E8a`mKdN8w_y$UMchO8p7CFyyfru7u?2n;US1|8gEv-%f*m=_j(N_{DfT8-?qY&_L5cs%pzMS zzBFrNkEztYGg^M3o$IkxPq%1+Z`EX=*s>al!t& z7uvl!apQ?)y1TyLv?;ZDH{=jsnc8Q|?sGS1goNr6>1`{wpm5dx?wK|4zHyLGP90X@ zdng=?X&SyLvycf0F?u=IP%ry5NNdqj+{|9GBaunk)4IB~RW{*T zo1gRqG6VH6SnfijVW>AbRsG3_6OiGt(2u`@P+rLN+kU2YVV#~nhAFRIB@_-Gw^RX! z7WXHmALdnG*H)e69GD9VzBgZewKdsni>dDpo*ZQL78^RD>x@wHD~b*eiE zHe0=C3DT!92oHZ8U*5@!M+0hOpDoz0r*YSP9CXro8&+#IopLN9Z)UFO@DMo%eDPV#8ojrlE=s1dXy9G&uAW4OSOYzTmO~XMyTCQIyo>Q>YI}b| zb!WGUa(TObn6i=N-d9KgYIiNef#Y-b*teNYe-}hF?vckj92W-jv_=%c8?K=Y>lf+MZSB~&IS0<#NiDNg4@;tgfQvq_X zecjVtMDk&=&qzO%d^yI5t}|0ay6~9Q{ybA{Ys3nBRPo4FN540%!xMcVJu}UI@H~q? zlSb6yZuD<;4{G4r4F6HMkq}wCw=VK^)Aap!JbH2Nf*$VsiqC?4&+p9}gB7p`+{8!F zRPu1w^|aSCC`Fx|PxR$?iUpMDbAze=dZA>mxkE=-kXfI9L(h+w+aTV`02|E!u7+P1 zKfT51f{9f0U1|Mp&OKu5y_tB`%ifrcgo>7!f2WcP2_`f>z7+Q~HNp=4x;+!P$Q=?H z|3in~R`Ak5SXG(*E=VBNgx{oM7nePkBolt}jkm&Y>;c*3oDnnL_96JENpqXLzN5*(>81 z(kSEH$QjAV-aCz3s~iQ(!hr5nolF{QAW0Bzf8_H!wc^jfY70Li6DM1Dw@OJ75kus2vqVq1MYd*@*J?ctJg(1{2jXqY3KyI1dbCy2bbj!O&^QT{p z$XtZ1-AzDQ1W?Y>c~#T&JC~309pH@&bub^Nin7qXg_pjxZ5miCnl$WT-}qW9VJ0bR zsyFI$q@RLRbl|OLcB)Z@yT!A#b9a%)8n5M)G=q9-Al|(;6=#HowBLz4@JY<&upvmd z*QW|OR*_7B$?6v^e=_(7-C#tdG6o>aQ#}$bPP|s~gJ+BoY05jx#2t4QcTdTN*+i9> z@DS3y&nI+7GU3tZ;h~MG@eKX(zcxe7)ulh)Cyga0w@D<2q)D_VBm`+2E}}=gyo>e% z>w6_>E1^d>X@Dm~K@{lzyPQE`8Y1TREc#xqfJ7fzE8UX@2hbgu5j8TBUnTS&qN&Eu zKMx_^Pcu%7Hj7NuBmnbZgfx@H<3|SaEjUVOhx-;Mw#hi3(8co(xaG<~-51p5zi7a` zq|e_(1;8L*rPDvXNnh3ewh)rK9O==?0MY&vva{*&i$BSaG<@XRsTR$_)=DQ@M(Q`Z z+A*f?ws1aT>zIh zpwu$Q?u$>ZnfD3JDey1huSX?*yic{jB)KzY#}BS_ysvejg_D zS(5^^@n_KtW(9Zycj*}^7C>OI8MRl?j7+v3Y7td_VQLB2f=jY}NkR%^wBkdsNM!LS zJqVR2MGoQ|M&kC0KlIilB;h3o_>Cd$U5Qbtblg;b!OK7lG7w@U3w!ebTSD7aI_XvLFrh{yip3Al(>q?+!VTc-Q`r^t8x#BT&^+C3El1JV;(8#b>DLR&+5K zz$)k}{KMWt24o+R@>$1-2o5C>G|=^^B<;^-V#@c_{&2JyV<3=yzUaU>#P{W~CaDYC zY96NW82b&KN049Jbt5f=JKc{^J>-uSnV7WTdu#faM7JCaFR4)Hhcumijk0l_f;?^A zguzM$GAlX3LNn_^bwSFXkM)k7*mjW8`}ulf6Q}yysL$aI&xdv}%s}SHFd|?&qDW2( zSP*(!E%F$E3opapPx*==^*uela|`qfQX~6VX=&Dgh0;*IM5_X*>aMCNeY6-zsJ+98 z=fp^iSc;ZF$Zc8zyy^X&<;sW?>xe;_Djg>8jHisn$B&}biAf^gH3sn_%FuWp^ zqRRDkvs+40>`h|OD26V{83v$j-CxJC7namc=|KfJj3X({RJ zqW>u)=A>@+l6K1#um!#J;~;zOz#+^L;WxKfWe}lh zgn|~dP#5f+5s3pNPE#s#>AwVuLm0$-7vgqCaT zWkZ$K)wFI?_dM!U8d|TG;!YZi*TX}{$q+}IZK-qR1q|+9OA2qa_QYdjmsa35 zmkJ(t)gHRJ6bDAmwW?ib>f6-z!kb6Qz*~26lJ6O?bmQ0g>|eo4M%f)g-s&<#jrD~J ze?`$5FZWv*byb0w7??`&hean4sj`TsybbA?ILv%<_C9wKake2f0{ zAm24zuD3zW?)L#h(VLbc!J#^Q39zRgl)$3%#a7#)GN=s!zO9-)pPy}iGY4sr=oueP zcpANgOx-_a>qLi(c#e#_dB}Mb)oeGFJ(OO(l@DmP8PfKDs2tn*mxz!JVV0HS3Fi@< zOd)#eQ={?HeV4~2vB_Bm-YX1bp=4R#$F5-37_7{D)mdk1)ge!m)yb_4G%qA}0Gxjb zujU@c&FJelep&2h&r1}}pfp>M*IgbDT)x8j)V;nSjkCy$Xrp{cn|w3B1|7KtbW)wT zR)nk&GyPnHuL}LUz%>9ZOdkB<*;vL>jujnmIs+bs&vuUJUPS-8rrcb}UUt!Ea{Id> z{4`Ra?^hVooFY^c4ll@~obT3L0ttr_yz$f~D(|K&;Gj%-(@KPnG;xygBY$!OzO+Rz zBK&D}8}bEPNyOvfWI2dZzI$YoF*9A%zGHBChU(b-dod5)LtaHg-%H+dunigG0>>_q zMG~RKthRU03&k&s&~4F{?Us#gZxL)|KkwC{=|?i{^Mg{sggJ9q(F60AEbEjI)rN?D z^X8k42+HWh(fnzJluvrbiXo6QyS?$G9ZxDp@6~M|gH1rfv|9O=U2`v|ZerPN-Ev0p zRjC?d^mr%0;H9KdUT>8bcSLh=_wm@M(08Mm=L7j_Ga1$z+xtV;uZI9!@6G!?&eo%M z!;3MS-BR_F;3)Xmz2Bxel$c7V^6v>=is-a$qZs4Zl1RB}E9yoRh(ve0STz&uLpQ9Gi`50TOl;=6*b-#Dl? z)K0m4c^YwTe|F2dc`w3E=U3>V5FV_U>GY%3RbXVDf+gbA?-Or#HD6$vP#ebB*xu2+ z-NArkRCNRYSm%cY7)8tJ&H+{HKJ}XW$JjIA_#Th>oNEp<8^-!~^Mmht(OBA(C>aN; zLW1Kf>-8tSrN{dP^n`@zB?B@un14xlHE@#R!5Ibi9N293N*J{cM6v+BKdwNWIkVwZv7AM6UIu> z_V)#H>h^)>`=4JPG`~H3dbui$p`*bYlXb|UYil58xO%PFxqq?-e(wH}lI3qYh0`|U zt#_*J-w1xWzWseKk@Y8eziwe+1wD-5n`| zwiN@J41acUqt}7g{5{4(M58phHv}ONi8;7%$bc!R^I**g_hyi`=?BTmEns2F3zQC~ zM#0m_mf$`^m^fm>VmOArFJ&zw86$sv?O0+I{?WEE|6gv4`@%}CbXKnWzjIqd5%$+u zVT(($aJBP3Xm01lnx*jdI zfX?z(TYMfL{$l-4Y>N@u4f6b%LKpcjwq52rREFyXe}`-HXXlw@m@ghOXj<05xGlU5 zY8x4qr)67o5}vdV4+<)?{;7yWb;?uxOyK7$I7O0{V|NGf*$ofB-p(*skQa^rKipQF zur)^bKiswnP-ICe@0LQ>mo6%5&0}_zTZH+J$2H|J6*je0CAvl@I$r5f$C{Gv|BCo} zTJ`i>vbJ9e5z&zUpMjl4>q9daVm-5HA`KUEk-m*Eswy<&h);eO`jNDm5yn792`s-uw^ZkyvQN1wBDkp%QBB?igt83WjV||eF&Ms1nQhpW1Qi>8flT~O~r$h7(?Oex)@R7X2fy71^r&tsIZel;!+uzqVkSa2dPE12 zA3Z1uU~nk>H^XWQzn;TOEFit3_fgACPqNn#&|>io3tRkPK5sAh9%8a6xLf)#iEwc4 zf5tkeZ^xB)>_FXPDD_09Al&c76;G!x9$lOvE6H$Y_>)HhcuCLCN4Fm~vs4IMW<>}H z0L)-Rb(6*YI2ssLgCzqtkxmY^vUsC^FNQ8h$CV!)$6cu8T_+@|akM(W;~5@LD#E#z z@`-JjtIg^5=&XmS9Uf1*g=K}E9CzJ)hjQYB$lRM!cN63W4-?9GW^@&nxJQ;zNr+po zH;-J#{BHkUe^A21{WKXy9pkx)!@%O&6(XunzxWF>Vke`mBC%!F@KvjEF`YW~eg=}o z$ePFkven(@Kvob~V4yhQ)*y0G$V($`J!CxfKg0DLGZ4iUas%Qi`TRlSq;j3=Kp9rm zH}q)bp3@Pv;XUSlmcHJhPo(7^$pYh;DI($p9u#QDBn`al;|3L+ud*5CQdWEai~#<& z+&1|o`oA##T6UbKvQ~(cM)|Zh%-BaAQ#tCU4J&+QST(lU@`hc+t6C_yVB3E2xZWr`iYnWVh02SHt0H< z%SqcjDKf>n+d8I6rTy8$2C;FcTbbLm$T8^$rrJ{g z&K#Ndg~zmvM-xAEfz^bS%tic0{6pJ0rckHJ0>T9;1v8xsRTlvD2MKN%8>XTg?K>zn zyr@-PHo@+@&bZS11#G^wQR~dk)z@D&8O@wz37AZ0=r@!H9lTxgYCCBJHN8gTfLCFH z%+dskz{#W%*8*lcnA|VSDKM)M26}1`UjEOa*g8&=$S(u`>1LlI|Cqf!QFukvAna9+ zLoeZ0L=D#{m;b2Zi(YH-^-akzy3RE|f-{e2s9PZ#n4GtaU7|Q9Fj-UL=1D^|OcZiH z+tBG&?p(&Ce`Y<`JX(uokwNepVBIo~jZqSDu}4Uw@iW`XhHLyibM>%1xdLnXNdo@j zrm83@tXmNk5?5Aondv#BG+Ro+VhS4V1?uqJxYdHS;5NT>xn>7bXf&p`@x2~?;^T6| zB4`+e#kFeRkQ2Kl+N@l?XlWKR7&un_)B__Mxv+?+@YLV^=y@97pP5SFu4R9>ZTk&# z=^QwxxyIta7B0t+<$aa}dL}YkC2aZm8ZUOB zb9%X0y3|ir#+n{3jZJT>+vI=a*@&fS`(Xc4Htg!U&L6DKks(p0XU{y}_p!Ewk}<#g zzv8O8hR)q_Eh#O}_NFaeoYgn0$gn7~4%lI8=VC zqA=!$YrTy!=WG&5sfOJbWK-PUD|y@3!XK-wMbhnNt;^l^oiWOl(2^o1W>#~#Et7N2 zA8h4ujjRoab=l%mP5W{4gT+2i$h}7!#ZugYRs5~r=!G~|oYvKISDiTaxo9J4aC-w5 zkLfQjTsPD>fJU~K%*H>94V}OA;vRN~vtCUaPZKUyy`L~hh)>GzwU?}nc2omVM6B62 zbkS6upFI=0HshMQ62Z-GFwPLZh4bk>GR{3`AMfI86EkiT~bDq1-3ETdX(4`s7J%@f|V`}Uc$;3 z+u^9{@4qU26}IVDU59AWoj66St8_xYOs5@Q_~i-rOiXhF*dEj&v4~vrKA&`vwwh6< zwxxclDiXnF7oe(AS6!~d1}ODgI2BLHG2%KvLE~zS<)t#Gch&8x+~CY{s~;4~+pQea7Q zE4&fzyG`dp{n_j2w@HWQdn!Kvk9V}dI{%Le{&ZTVG!3#ucODeP&cyvrulS7LNSnD> z1jp?;5c38w?KvlPdlU4V`_$`3_3=z;hI~s6KIK-|uhTmt@vrX+{6rtR{>7V-{u6yf z=sWXJ_^x6nvG1*WAZA1G0dFuYEMQI5b~03(!O5)trCy-(I*iaWBwEw8ds?_91m~R_ zE+xIYsg%EJF|jT>yS8-1CQZcaZ7b)!AU0{Tx*6{cG>^%8by8{Di+9E=B+ftjJ*_M; z?dEw%-dT0=$P3R$oD@2+SQ2gE)8_o}YW!>+a2Kv-X&=5%8fpnDmjDs=MKpQp0X>Cl zZw2uiq>9iiFU+C}wK#rh`62I{xmpIxNqKtE$!jL5A=NtRy`juCcU+S8`Hda{N(JpY zfwuTmHi|_fZjBW8qFV1rW3+tB_W<}2R>3!#$8SIEsX9rM0Ga!IZ_z_aJLH?MBX}bW z!{YSMEnG;{tt;=Mi7jI~#bXGx74i7Rqi_7%x3^UrQH?lW8>OV!({z80&-a}VGUK8* zuO=0tiHwS^b#V|k{yOiJ-JtG$7h=*Eo!=A3f57%`z?$$NvAjD`krRaEX*1Iw)<)yz z;t_c?FCuk7k*1-J?a48SZ(i+THRkTwolLab=l1Q+8E{LK7Z#UX6fIzta=~u@tx;5T zAjK&^CD_IFSlVNnE?8aKvO@!CMI06s66WpUB;Dg`GtY5Q3;4d_rY039lSanu`KvQN zj(5Stp)PJU!o^`B(JnHL$22XX9|S-P@Y;*BD7I}%RSLWoPzR@qc^b>tINb4>55HK( zYLFR+YkNkTN&gLX^R0=W=T0f_4>bIirqb`5juD?`1$`2S4tD{jJ?+=gAp_>&G{gzy znxXjIjGSQ!N}rP752Q98*g*D8$s&KWl6s(MCf;lYCcrY=JyI+Vpac9QeGFN>C|Sst zX(z>g&akLIGq!gcVLAN2{-{AP_GL2nW4p{1&b*wNdt=E`zO3yfy?brsGoQ|%7w7dB zda9ij%9%w8;iK?oQcH67**4mQ=~kj-F9Zc_qd8h#y1i*ka6NF6YJf;LXv^-zmNi1_ z20X~$de35bO&BJM0}~S$GQnuMdmA}&)_gMFF&Wdj+p|eEGa}uo;C}c< zqXYG}EJ*|zHorS&t1c;4@W^Uz%pac3zuBeP3|J7cMF7prDa_J`Mf}N#U?C z7M4|}j;OfZ3}*y*{bj5UL#o!-rPG!B%{G{X$y6oF7mJHX)%B|WgD8*(f$KvH-+|O$ z&|yhItW)Zh$7pPxx(ss&bB~TpT6hIXRmB))o$E$zWcsIY3d)?+Tx&q(%tG^r_CaRJEt0IdK6n>c}f$M*d~4?*fz^NezDI_LAOF+7k2xAaMo!^ZuE^hsse(V zR*jliH`S`V4cqymFLxV0g~|Q7scxUGGl4@i2bzflYwXeEgi|HP`^~zVlK+xd9^_Qe zJ%+S3hHJqSd7Az$6*uiHQEmZ9+Tmtw42{@50B@U8sLA*7S@V?MDgg?eET-15wB%vk z7UH8;Qi?dzzKnKYurJs$(w(IfElmSZTW#Z`gH^XH*C7znx|P#P$x|#M5Fm7vt)rWC zuUnndV;_DM@UY=e`odH!fh9ubBabRj*bruwx@{5!(BVVrV4`RwS=Jyr?(*;GI-r2C z4i_BpqOmg&Pz zH%kxplfceg^48YFh9AfRn3y9mLRwyq+N{$%VcFxqm+hY; z-MvAPXA$G76iwKAy`|`_*0#eW(6P_H1dq&p(^h@Q&AkXO7%64dg>>rMXnBCo_@y`8 zWvJhVsa+zqKG$PP>ZqG33?ER7XXJA=v6uYi20I>2ft?dxD-%=-W9XWlv@44XzW5cC03y<5#RI?{L$lUH z(FJ@-zhyQvDxH*MVd8c;yQ!Hs{g)&lyGUqAL=al#p~W3Foe-s6}P_HR)bEK&-Cl&b@WuqyUAt3BKe@de!x=| z3|VE?sAHBkG5}eW>9j>tXPZk@H$n^~7{(l4=0YpezB{D`<@8i^g zzh>?)wCsPEF9=iEuFOzZqn&NxnF0wNEs;-|-k1t3E?Z#MzO>{@VQ0TTBjPY#CcX#h-MF+LG8^e5JqQNf$wf$>9qE}d;oSwF zUa(bMi`|UttX>D$UD9IK%P2YQ_pWt6%&k9Y13<)q12apqK`EWmw(*vqSxw7aZ`Ti?P`z4nDrv^ z2IfCl9V8S zxoi3B8+q&7v-|UhWqXCkIj6gqe_-1eD4Rux=SN%1SI5dz;0l9perPy{OLgo^ZSEDM zj92Dy#vLA{x5C5oY=$tv4y(a&O0ntZ#20$iOf{!MI+WTv8CgcHVxLc5&<8j>=lmU;iKX9 zbR`P(Crr#$u01l{@cPW|NdGl!D3+$ZSf#}qkSSB#*Ey|d(gkKO29Cn?R44P*(#Fdi zc(yo*c%`Q1VCsKxTgbxx+;5XhpD1;vrI-j3sajp|SEJQ--y=o2a~21+zDEWU+_>Y_ zZW{_&FeUuV(c3*XPXbrN!@dlisVZb!g~sc0=a;j=S)(xEW_oKVE4HNe@8jL&(b`y6 z>&r_X4cel!CBb~Eow@cpS87m?UH*LFvl!imj>ShUBcoA$Eq#w_Qj&yluK^mRFjpLN zN_XqPFN6k652(5B8g6KKW-;TWiP79Tb&)Y=je91sVmhzZ<9-wX zf2#D&1!+5ZXliRY7iZh4sd!R`to!zx*?+blSPsNXaru<$qOEBjRsU=7!z73PBjjr@g~d^2 z9d%d8Y5z)s&BXwHkAy4BKr*E(4J8e> zYbUFZhi%i=P)n~|6ZxC@V!->`!{^46hoj$`gAd0S63jX~-XlNC5&;d)CoW@Ea1`#~Mk zg4gU#xn)?>;+Sr}c@qHU z&mmRlCnJM6(Xq+rkhyGw8~ir*GWjrTI$J_BIp(#1GW|NMm`@k^ z8KySHD4V=~{*0ec5&UV2caTsf)O)) zwr7iXBy7BT|G_5tv?>D%8mSlQ>+-*N`xu6Bzq7=NqZZ$?uqHcs8G^>fj}W(VPF#p8<{6MtRtdSxB~{m!;KfY# zzpswX4*3Gdg;W@vI;eXx<9|2e|CqF?5bU~1hXk*rJ?4+r{L7`K;K0EBU2z=xV&S}p zd=SA-;60-AOE2~Zm{F46`6`+$xD9^BdDX?_A4x zKGk*NTN!5RDh@vvUE-Y@9-6|9F$YNXN8B?=-lW-N4jxSoPZ~r3uHjk|p>Ujl{WxtH z#BEXWdXuv0u&Gb~YcYj9?VgBr2it*d;fv#uQgEJpx1GA>1t3&DV z;!4Yb*HMPYtA-x`%3O|FAK<^%_Rr%Y=dOlN5MUYwCT|+(pwzqUSuXzbSV{wqLvQ? zzrHJICJBf44E!DGALh=rxGmiFerD|A!6CI36wJ^ququ-7It|8|=SO*Q2xlh>%~2D= zSZ1v4F{09ijCt-V0?0|z!zz=O-j}G|pBCYiSCz<)@L@D*H;I0+W4?Sh7dx92zvad( z&E2q{c~R0##kw70vZ#mcS1Wg=@D)rUTg=A9_%fABl~p1>Ql+H(xHzrBnFx?SzAsIi za4FC9E~6hG;qa>aUAn2^a%d#hrje@ll#xA9*8dGWQ=}Th3P|@FWoD~cNLsDp{)%!E z1ZQ0`Z?5YcigE-pruOaO+URVB3feb83e|Jz;B;2vEwkQUSz8R{CjxjNG}6uw37qw> zQ9GpV=j0!L-c(J!UAvoHUgx`=fKw<|g+A)F3Hh5`WY*j4ENa)q{+o84h@RWL);%Eg zIM2p9QI^bFblyJYX-E!p5SznE>+40xGJzO(E_#RP%dgCxM{h=t1@}uI8<&ePJ)C7Z z;NN|0>sd0nhF%fXxU_iAqStL0>7G~nzrMX)i+!P6)p%vFRwhxqiQ=W-@%HBzncDk& zUeQhqsiqU*5<#^+#RmtVND{5mlj0rKpI%PBj@I|rOTl*Bs22KZkMAq2*IYZ6ibh6xLnCrg3o); zp6BsIa1C)Gn~YYWekrie(NMJZmqBOy8Uy|(yt=2je!WXN;(!lozSDd3us+{}TwhFj zMRZbeo+*!wKK~I8XTorgK1o-woAXBt2RAo^kD(fbmSUhVQy*I9YYl0#grF5O4buXp z@HdPj$uXk}n>Vs_SEF|h&V(MJKsDI@QcYhTjBUl`K{>}758gN0u$np*{!*vvAFB2NZ7AN7F~m_-VlET$eW{;pa4|4{ZG zU`>4A-futzL`tN0snQXU8l?9sMx+T!2MHh&I*3T`2!tLwp@Rs7-g^yIic}FoCxCPj zZv6fK=iGDeeb0H{n`b|hnLT^fGqYx%WF`Bv*7sBJj4gs&WhODI&NGs*FD4<#SgP=2 za=0&PBAyhpk@oF6S73xul=+q`7r9-dSD?p5Faq#=H8BV^7k7^E`Q{})CHcAh!RG^L zfR0xHnWy>0skY(yOaF_gmEzYSMX|4qLKx?QDWtqRQp`5EE!t=zQf4FhdbwyNtxH46 zL}qOO1aBO+;+!{q@TElXj1&KeCEC`;+`;$%6&AI8tT5go(|(L9ajk>x@PP-Y{GQ5kkGoML5S-?*(%!x~h54w!Vj>FAaT+ikw!sl&DCcdOxaep~W`>Z}9<=1)!?u;9S zWG3J)oZw5N-SRyM?d{fQYaV%3pL_z7z+PRnj8szhRhJn^iKRUY*+0*i zI(1VvepPRi#~q2t-9+T}AjC?FH=X7ZIc$bgr%}|m@oh9b-T6Y(w{%DjcwVx5nO0-3=ho%R19A+=#*u6uOsIu#^<|YAAe|XQ}eGSmpIMAt=1(AgsQT zQ19?nod zlj>)Do9?Ce&@)k+G*GBn>SdH9@YI^WtuJUU^J^~lfjAZI3gGUftrFCcM1&<0hTtz` z%*}tnR=@kO9eV?{U%1MtjClW&q4;jum$+0f=@)lElhSSxHy;8rJ1cE&a3+2i#4ngfVgr=?d#BRcS4}p&E^V_-jJ{eshge;%{v#Maa&OIIzQ{Qg|Ef$KcL6<*S6Jm7QBoUyzLgP(Ff z9Hz-wQ>fW>`AC6j2Dw@06_VW=Y`jx&V5nh$j+tk za7wyih?obHL_2wUzsId<9Mg3(OZ1SS1aeV2pA-^3<{KKd*VO@cX=Bv=xyWO<8%#F> z+uAGw5o_*SJ4|o)4S}k;?&?In)f(k5*7EBW5D5$)){gYZWr&OlBrbq}G8of7PX$CY zn6m#E8Q3gW+;G&S`17!v!4CB;4d$56HkcYjj0o%glHiG4W*A`IbMMFgfoME*cfWJg zB9|_*drzV(h|w!-?_)$uv_SbVt?vY-*+dcE^w+P!)J8eJ#1$+<7E^uFV+F{p-4djh zF+-8^$FnDkd(|vlWAuHtA$`qYdkYM4DrC>O$89+DJNbkCUS+0f#Zg5kfkBtT;6O;P z+HUKKnn&pFgx2cp;F)N`&5rGYagq=HsMM*b=iBDzrff&Nb)UuRgq9jvmil&2GauQO zQ7*JV{v^C-QhGNko4HgUr!(8Dj#Lbt)iA4GFKc(=;VX0;D||FScGkHvG`1pH=*L}D zuANcqSTy0Yus%OM$bbLo@%ZPpsb^K^ZgR)YPa*~pUTW-T9@P!CNs$izEBu=Ohe_t?@}0 z@7bjDl|Yti{AKWrb}l|kBZ`FA4Dj8xQu0mBTkDs>ffwJ7ArP(7un${S49*o3kj07S zdG4l_i_8@VrL`m0Mex$>9llq;uNP0|nkIRJW_ghJm?t0h#fS}u5c`c+GqDwlMHy2@ z-(i|d|F*J9)t`Xhin)7_O}3R=SNc(w$n)3Jza^HW_agN_)D$*Or?!l2EoCP8tv5+> z^7p5-#SBW4jFz-Eyb7z))BZMZH6wf0Uo9%Y!n}4$lK94SLq%lc(a+7T>&Tt!MDLi@ zz38Q|i;mmrr6s>Hr-MWyKa<4Rxw&wEWkW;v5v3J(_4?G7BZ9ZS{Y&_r>#WwVp2@BS zl9~oo@{|cSzi;f+o8A6p+G%uCOMJBLEXuEUlzF6YFzr^ghAQ*Y~@lqHzgQ*D}sx?o% zGOKyB^`qH5=OBbt!}6nX#nZ(x2T%_*eX`zv^uVnUXNiF$eKS?^@Q>Hed(R#ceBpTf z@aQ8*YhQP_J+;TyZ-cesJ)_exG;1`me0(OdW-YWR^cEF=#Jp$r9MiII4J?^c`nIR! zw>p;;9nN+*lU4SFOhS0&p;`9YJKp)&5Ano~OD4QcwGmv?~Y(J>GQB?E8@D7#A_mm3v|O@N(qx;%>z6 zD&5G{`*YH9KWB-5OKLCEl~33%97Pt7&^(^O{eTG|oXQ{D9{kcmB-oSnCH`rehQoPk z-Lu0FsNYq&zxUa1nveQ2zQ+J#f9^iK{wcXsA_)t>C0#o9d~9pi8^ct}wR`fe`QR^Z z%c3qRX%t>;?=`ozWkfs3&3$>nZJcIrJfG@aoK`&FsBYrm-L%!HL4PGh2w=D`#3C)N z6$Qy?E?XG>y~{xSl;2d_&k-KovKO^;P0S=9qhsduAtn<`+b+qF;4=%#>b`dNkIkxV zLoFZNkUo4Ymwn*$G47w@wsJ#~F%+_&N(&+s?+#&0v{Y&Ca!z|v{NOaWb*MYxA)$VpTW2XOUCPfh ziO|dt`A}L`&|acnX3`od_OP|WwAtqmk$7tP+Up#mDks-BDjfOliAB${s zpc2~=6`TH6`l`mC>1D;O7 zV-f%$Er!{4GliQ65?u9}2a(FoVtO`EOM1%*t6Wd0v)UiPN*^1uIT7dWJ7u#o-S5R5{j+ z$d&A z$26WB?5eci+}wBRz$N>O+dhnT=^}bO;?li0RQ9gtm!|en54BXF}Q z256IR8AK3y;qNQ*4Wk-!Joxh5^j*(ZrS_MZRa=tNS+i$<+=jrfS}Pl!vPpzzEt?tY zzq@~75?}DP`XJn=;d$Sg1;BcVcNZUU3*ZM}Uts)IoG|SB?d|RI%nS(Hi#b2ng!BRl z30r%6(HOKzaBnLDjeuLxq0vx81sV;~L?h8?Alw8V9UW}~1|wUM5IQ*$RXaMuV(C6&Mr}i3TAs!B9jn zD-Q&X1Zg6XbWkfRAa(%I3Rr91;6oW+cT3JDXRv;Ac zuc>Gl8U)4~A%vh|AWaz33W)8;TEO-L(J&widn_Rx5PK{b1w|oE5GK9|6BKF&16{^I zO`yxc5UhWNmodU76~fCE*d^2igLRHDw$W7x!z+a8Fi^S*DAq;6!iZoL)Fc{;^&VX` zwgk5RoUr!$C< zg4cAQDz`ff$gT-j|C-kuMf=2Iao}rye;kWaB%?-k!C)fyi&E3UYQ%7gkkt@eqo!~) zLlV04uNl36y?=PV5&|4aFt8vChIv*Egmg?g8tLz!Cjl}pWD zzjl^}>%XkD2i&>Otld!cqdS~}@AXK-*Ps2L*|oB?8>>|}hEtzFmPZ|eu$mZd)D6e?>W8p_S;(kKKE(>o(Q%5>Ljkz>L(2@z!fW1LI zz_T;DZyBy9bLaw?0qA+Iih=PsJjgY&NJ|D65UY&?*W-}XncTKZ!SC6AoAo-72c4@@ z+BBZ60(LJfNIu)y{bJxiB%hwatXDu}Kz|=lEf50Y({4!iehwHC+2xGQW(MKJ_~VxD z%Hrkg15e+_ni^*DF#JAHmTUdpZG@!+^%IZnRHL5*|K z0lBqHw~8}>^sC7IoX(6B52??nhg_wd@f!3nfdtUU7aR=F0yzY;aTkjf#DVu{qzSf}w_f2z_MNg+7K|*?dW6BwS1P;8`MZ=t z9-A3tKEKUrqfuv=(Pcv<=<= J)rSiw~b%J$-;lv@1A_2p^EuLHV{t|;I1%ghTPMs}(9Rm9ED zI`JtN^t-#lIDX6QzOG4xkL+z0IjO`kUae)OQl3f&<6&45cqy*9j|-C&f}Mx&=MvJK zJaseX@rYH*A;Pgi&Rs^^RR1QB9e>1?5EJ8KvtZDd?#BE1Ldl8 zBECQSaSur-Wbr;pMAMMP*>Vag=;@Xysk02@2O@imV=@&NNuKmImwT5wLVf7~pS>zw zy__tfmBAbveJatWB`#pMfiyk$K5+V^!XG^blF`sFLVBu>;g>aYg)Bye(51@w%VBl~ zs>wl}4>_!Uu(W>Xmlf8%2|fl6(|DmpO|!-t+UbQdi~1YFgiK;lpKF&dmR{PHgX4*xka+p?j|~ZPIL7MS0hg&zk0B_;PyeIRT6B3NszB8oLo3w@2i7 zI4t4mcRIam^`y?dZ}sRkuX(3VyPR_m^_>lB((+-lmFh2-tnv@jnw5_ZN_ntyQ8Xjw zS10+OE%qxCZjP#~`oMs9Jy~aX0B#+#2$Ho#B_jDF2%+JbBq4vCS|hc0WzIkcD>|8b={U}Gr}PxHZn_{ETs z_*fEugaTU{ZT!B>Cz&O29OUw>lTJ<60{!qYvBh0{_;akA#|>V9$0|GPjr?Jf>4r_&4h7ge2DlzL&xvsrhf4}; zUf|?M;J;k&719@laHkL3jzVsNhRD4+6xOEgHGH#vFRW%I42iqHo#LUd)LQzPKhX*{ zVeN_f5nJD~xip1^R6jTAhz7KOw<89>Yg_({5kL%^L=a0XEMgbzl9G$ zsJg(pB@;Ej7+w}E(~MnYuAZ@fRsqRyKGwyNB-#-5e){ro?gG`!4agyD-Rgf!KkA8B zZH3N+f#n2{$Q{+2B$|XGN+JyPQlk0#Koahj_t(T_!Aov=*{XQ54aya<+@0U6kLlUb z5Mph{jfidO(v=N011sa*+dp$3<#KTdIAJE*& z#jd1p`|4^3^xv-1)E#vHhMlpx`;3YU3@Ms_-*lHz{P;FL!0U+*xHABP^XXK=m-)zL zM%CL>`ICgj$1wd5XCeXg`E~>iAAa>ylNVESg;VoL(&Oln#(Z;#li-l95moII=kC{J z#^(<1$B#pN46$-h!NrZkcU8$_R|*so!Ii0FV^^rbDR@MnWAEk4M(|kOvc}LsWzDGS zr&;t*Gq)hw<&9uk4uSZN^>Vqk6dEniOBPww_%M8$!g>hxiK zpqOL5Np#p-3hR@kpx?KBpt2snPh0xJUA{?Yx_j1^TQ;K7-#9Fi-T_rqAFaQLq7C-< zkWj~c=Xdsl(V_}ia)nni9 zwa?F!0+r@GgvQ@Q=<&V4VJXq*octQ=jdTG zv-xSpsm8^3W=#`UYv<=CR>$roYAi0h_3lVgI>uE#{9IQ#@x?8YPm)f7NeZXMQYi7x z%LZ~8+8P4_4uIp#XLG}N3yUN}srdThsO%p+nkNQ#*LC-;LF_MqfAp-hBOKM&9y9bv z2=j2w74x4JiVu2m9Vn$Fz>MV|iR{!FxX?ZIrUq*#r&!F}=hVNsmvhMOeh|j9;rRtO5xzrEpmS)+3k-{o^ z@#iTuxW;~KA$E$A@Gygh?GX38wClTy?Xz4%M*Pf#X)nXlOmpGGJZa>ENm)m6GGcsk z*5>#>$PmWKc+WG3C3UI`%qh{P&fTAz^u^hX#LMkPAJYN02kp|nIj-k<^uT$z`dG$!Jt7rK-bybMAqBuqd?qo{!DfhCjC~xs%K}Bh{LifP5*QD(anKDYyCJ6{l*JaCLg9 zSga5YfFaH4QY6vELF2wR-)00y2)(j!aBMD~Q~hzVwC4wFkh^`8w?k9mqq&k)VM)4t z+olMCXbylUDET_rNF~kL99QLQqWr@hu7IXoPmy5xnUA9!FA7XDHoFjpT`7%2(uGa( zFTs|z?(i*W^kQ;}Nqxx+oQese(iDw~lm`Gtsz={nL}$Gek6FV_R>FCDhtGEN*asIV z%3P?=0u;opitjfn5~{^n3j%QE-~?SI!tFSp20r(q<&2-yVDYHRzb|Au)`-zCOlR-7vgV2ce z!U)mfh_!7qtwMd%aE>R_Aq|Z$M4~d<-K*%e^mcn>eV|<2QSSV|Dc_dW2{uzfGH6Tt z#GJOs7LK1!i{qU22yvU+h#UvTy9!B#3nIE4OhXfA8V%)Bxy5U^-&xmJQIK!G9ExT_gF$tU!IqBZPb^QkG_3#=cAOhP(OV&hwMh={aU*Rj?3q_QH4=9i zki69klA035&xz&ot252>kdOQy{ugx%ZxS08M%BGE!1#cakx(FXWZc|Fi+X zTOV9^p89s`Jnxi0YRyn3zgaaKO8LZ0+$Cr#$gxKu%-_xSG(?HOTSi(y?7Hh|3)>U^ z57I4^kF*h=Y09!Fdt~4}oX6ci2keA9d$Ost`M;LDu@WskRVA=isXfirDVI?4fxigs z(R|v+($!-W@`y*@ntz!kHZZM@W$M#M(yA&-dk#lw7joyDBlCCc`q20kqgQi%-EQn?v-4N?&F zeGqHgtPYl%m*)6veH@bgAh{g>Q1ql1QM=QZ1s}evG!(UF6Z>1R!I3;mjN&TA5>zA25*9D#B!!u9Wa>~D?&hoc7I_3EdedD)|;3+^DIZ*oBz@P%o;)AhGo`S-k~SO``V&x;v1U`4Fx2wG_k*e4lcw>@y>V_CR^THhL+a7- zPVcdMwj+|&Mvu91Jf0L8sIVs?f#2q32s<;?{3dy1rf*3_2DT`ogaxJ3`u3}{=QnWNV1N1A{x^?NR_&4JD{$->Y4w;HhDEN|v`tNXKtDe%9lr*3wuwnG-A?*xPo{a=t{!TR zkX1$sxdP;C$=Q@Y)71LX@&a5txPa##oKXFFoBZOlu5SrT-%a|y8(11=Y0Vs0gF{DV zy{zQ+f&sij@*^G}HiE`6`We-ux^wB-q~+6KH6kHAyz`f$xdsFn{!#Eb1Yz z?8Ku=3I)NX+O9RZto6kQ>tSK*dY$Xfo~~=kZan7P_`SdG7`4IfxS_wi(S)-RRle~` z5Un_}@!B74hQvWvtD$XBXpLO7J@aNM&gSAC`kgbHPjJ&mm)T9W*DGt&t6?*cl=;JQ z-`%TC9oenOw)Jr44G;gA*x4=+MHpRg9<^7%9tnIAVJHF%Z zw;PzYN}YGgD;bMp$HD}6@cDNuUhife?PRv@+~?k|Tiy+`SO>8A|Df@^!?L$w2yF*))7=|I?G5?w4LbXynD@K;ciXb|19A3}pZhVI)7Mt+uVpdf01xCQ*jHp}Ct45o zmJb$X4-QG0|Ck?$5)&QyW8}snNwU)qE|xL+h6le7Fu0H1ar6KtF~BYS+zjd7wNJLXwA{#J_Jj@Z zm&e>My*H=B-6v}APE9_Z5|j1lwx62I2`P*nTf8~LvpS)Cdu-c&h8uMH_V(2A;rUZ1 zwo~!P=N|8PoIggnd_4ENWwZYnrN43>h&g}w<~-=*h5yQFeA{{W$^~1(dEzBmH0A=f z!s<5mD=GW(EeGx;xzuI)?d3@Rg|gsf&YLSQvkRSfSEX|EdD#@C?N?P}Pw}6f8KV#O znSXnHxHQH%;ROI%Z-2KvyzY>@R$#mCle_NEz8+}59{PA~8~S@k_BUttF{9p}nUC1t z0)Lp||17@wbNb_Q=j-ix{20@x+s+&Sk*dqhcHGU*$D6&Co5S0i5B3`Wp&Usi@VY1U`A^kYCK-RcM&S-QQdXTT zDyYZcMLER@_S7^k)M^|mHupZcmJC|e{L~@vH${84xx@FB4r_k zyEJ|S()bU9^$mQW$i_a$bBgu(Jkuk44Rybkb>Gp$Xk>?3;!so=?yX!(o@Lsb3?dZ=y{T zo*lTZeN=kQZvlIr8L2x zr=t=ymq+6=YQNpS&341T%$)k9e{ipmO=>0Dv&) zXTSx)Cs-uea!F5qk-yZ|e$z5Lnk**5d*d;En@6 zk5EJBn5mG!-MxXM?-CS*U60vWZr9BN{8JRE4MOoq6)a4js>k1G%pQ1*BUb9%p=%vCoJVR>1%?zfB8p**Us-d*ErS%vyIE#<-OcdwE&CLd zDJFX&iJy1sHK^loynfHY>6WF4t%OD*x*vUif>lu*5CBRq2)DqSSp-O_w?%(tHFX=r z8&;>dJSD?3x(>t3Nd;=8fA)5D$!7cG-lAfF7at58CsG(J42M;-Nr;m1-dd=KUO(V) zybdEznbQdCNbn_0W1+egRseR8;N5PqtD4`_2+;>85xw8wmQW19<>*Ko3$_(SNa;8j zOr_8E*oq??b!_OlGN`<5Z4~RIbZ;1caXQW+qzuWGtf&*?Jic2!=CV|=e2tgc>Bc3% zbv)=Cs+{?VRYCqPNH36jI)^~TUXiMF+<($8mrSMj-P6^0R&kCrR_oV{c5ihhC*uR} z&F2f!ftAkRgY&T;=W2qt-}PDu3myuFlqefHw0mz+u$F5N&>EJ;>j^#fR(ZQFHwPHh zv=m`&q5RIAQS>0&vOIx6k4w%D^s=Jw11W)tIuAB@^kZLk$ZOp*L)KvA6kco7n-HUmha-ubax5c2 z>CSj0zOGB;OUDH7atY2cf%{jfp12=7t9y!UcI5N`9i}L%K0bWc&kWV8`1uV2^LGUG zAHFgPHl6%n@N`=R?nh2=e|~7&>3uwBbTTblJnp@FDxAL-)_cKtnLJ4Kd+&b#YHf6~ivoX{A391oQ7bE2v)$8?qnd>43W=)RW%h)+j3?4$?^m|xIa*3Ezy!|J;AT417Fvk!sJ=7&7@4iNaU0)t zEIjIZmz2mj(idXxc^b6IID15qP1mT_8PBxB+)V2ime2X>lRq-ObNKap`L@rTTk*r5 za};9PP)f3ciuCC`Xm(}U-^&Qy>Jq6{*|=F})Y&O=6zO^Wj-U#b03jM&aR=|g;78~}$FmDu%=g~1W%fmZ|-m(|*WZ8FQ9x8HOjRnb`{^jY&yT(KaCmFF`R?{Yv-T#j z5vzIZ+3&l2T6rEa-TK%$Oxhdp@M2Y4Uk<@f82_N%t50J2C@#!zom0+xh%D^M^ayGrvcG@a; zGZ)l$wXW8_`{LGbv~cB5u*64`o?HK`?hk)X`#;{iC;#~8eC1Z4vHkWbMF$S?y&Lw9 zJIeA=gz~ki?Rah-rcdN+>^lew4?j)zICgq4C zmJscvGOc`YB2OKxaP^{-wxF}YwUe^ElhIFrk}-;Iy7S>)Ap>I+8D19~BPTQE{fDAm zoO^7n+LDTog1gxL?z2UA@oKYj5#Q&m?GoszuOC+w#8dM8)g{ECM7-8mlhY-luN2xX zm2=T07Ti59r9>_f-3=mUPE%5no>p2&>#p|dmcdhQ>Qn*|_b8UbWqTBq^p$(*dT300 z)Yci4_Y~9%dYb5D;TKQ4HK%)08hbL-yLIqX(zcZ|272^Fdqe4ZU+DMlX!NG7w!I8i ziG8b*=GALd+Z&kHn_AraYQ5KIP~|S4LhozJKKskwl*52$jD6PS$(C;S$hh%rg8N)8 z`jY2*?Q8p-yxYuM`kdUTpwdwe>wPXgN$Y&~T`2pZepK&7??dI3`#m_QTsiK0xb=H4 zQo8f2Sr_#C`B8c?-nX0X|Hw$?8?EMqHxLM;@Ta`*Ejkd=L-A=(Ezo)(oS!0?_UXB1~I7yk{VGnmiKke$>f8{2qA>@%@Pm)dYPF5{<#6;?|_#K|$}_j{(K$wSyVf z#jJM*Q`a?k=!&x43jGL&aGDo=*K;0;$%4u9PvFXkAoQ68?P(;r4!57)U3*9Q+bz=j(OhMQ`!T6V)N)AYlw z>%(m)!|iw@9h4)T93x$#Bi+g)J^CZPbk-w%ZX^7{!~L+4!Ge*Y+L7U&k&)?<(RJDp z)XB&g-Y8ZTZGvNTQgn1md30KT^qV#9=#1OwZ1CtDY;+#03|l+8&@;L?J-W0`GrD{- zx`KyVr9}PUK&^?Qek!BZ^-&wt)+n?aYBLzM1w(BYpmu6eyFIAAX)4tII_lsAg~1y; zq#Qfq7&{gn`=v}ZcA`IaYCU%5Hg+C7b^#l^EEv10r5XF(Gj=^a_Gf+U=49*^ZyZ21 zj>9>QOD#5jM`axE<@jBjaeVi2f{<~-r19qwlq2_$bnhj+5P;08&v+ zJm8!l7n`6^nV@_*L1i;R?LI+66*57aG(m@$pzj?fLQXJ#n_$|QV5T^oczAb`g=&(O zbCOMLl3itz<0a)Jr_JOe_erjhN$#Xc9>gRsa*~gtcas0xq`=0c;OXS!yHi3`Q^K5- zQzBwhPgJHvU%t54J0a#iB@r?ushu<>g_r^%r{*XFo_?E>*_e_&eIY9lEk`x2z&Sm^ zGOeUCt=Jo&P+pwO@oK8|J-25{O1er)8|2rvR296 z(dJuo?pW)$S@xxsj=OCx{9|rTD_)ftKg?}Mj*B`!ecV8S>!2ejd%W09v%=)dT)D^a zzEAsVm6x_)#@JS^&!}_r%3*__?w1A3MT`F!=J##Dr@)}#kkGL3h{&kunAo__e>JX> zl2cOC(lg+hS=l-N%Da~>DlRE4E3c^hC-J`ae@MI^pZIs;{eKm?64^j+Zf)=E?(H97 z4v&t1VKaZvFD|csV=wd;aEIuBh3N?V|KfCb{-)mlAAvd?{}#IXA4wgS{|?pp4^oHa ze~0QY{g12;gYJLN>d^o9unr@Z*7sB%JicZv&=9d=5J3rlxO%j#d(eDH*~f=zIhlcAS})LLn6~S)T}(UR0l{DCD;mecPC~Bo!!BZpJ&MC_ zpc?;C52c~wQ7^4s`B5LEC;8q{KT8<@@c>5(mfXQ6790;f;IH319v157|1}~y>-cL_ za+AFL7wYNx-mfuv0)dlpWg4fGi3jTcs9#C!pG@hg37k&r8v>n9zZuz8oX(hf?w`(@ zhY66M%~_{7oz2@9Rs55bf41NP>=!s+benZLU-H_lIA8WVC)+SPiCezW5Q& z1+2VSi?g@72ya|K4xdto(gI)^dLE8`DAX`1-Ji=H2xX*#Os<>*Em#%=Is<4%VL&608u` zpVK+JFMrM!Ju%dO&R4@8-(0M3GRa>?XT`PU?R<%||4ntkK=koC81<|@=WK<{{^sf& zgQdyHh;sl0>K(ZBhXI7zIk+@T10DDv;8R?$95`Nw+`YVrKx#h<{3mmTB(5$x%yif0 z>5AN4PvWbBZ9uF<{RLfAktKnG(YeIbcIw^q-w#72M{-FmaJrdvOTuKM$$^?d7X_^3 zUo}*pT7F0Bq2QZ3M63;E(}X({WN&G=kqw zW7y>TadHFc2tv?vaQOAraY_eqp%6)!#)u{TuhaqULXng~(h*zGue9;#LeV0PQ3n&! zU+Hrrh2r&tqyOk)k^Wa`mbkQ6I(X--6%oaE7t7ON6V3{AxPoaD2Rl&GtbYECAk zpA>NGlxP|bO(qlnSADGe{}#yd86f+N>otc?A$6zq$=wKw+Bp9`f#;)J~ zg>2Di2m%BkBn*y@u0X+|FdztsRy2X4O@N9OXeb@9!sIW_S^)zp!hpRf45D{gm`)gL z31WhUi#01MDnN>82o{LNqOQHbie8{%FLpc{ge_p)3jsq~A&Ll~6$)q-3}i(CS+O80 z3d^FR|JoN;L}Bdc(b#npa0Rx(z-Sm4(u)Se5a3=b zD32Am7a5Bm>Vt_~t_Hw}J zUKqL;?2BE(U~u>{1OfqCVSrW$AS(vQiU1-oKm-EW+KNEHdwW}bVMq`J1Jnco>97p9 zFbIeQgTZJF)}=65D-?)b!+?D;Fkg%!5RDLqAb=P;MU0{%28P5aA~7(i6%fq>tl$Cq zVjzkb2oeKCTNTz$VwQbhy>D^fUJNJAQFT@qcIQ= zrnLfl@`^B!BDNETfuNyQ!W9rSh6i22gGTcRqj{heJi-+`P%sRt$OA<|(9sYyiU%Ey z-Af3qAcP8I`w5{KO*B+dxIz<+Aw*XYVs~K=1XmDZ4+Mi@U_~AV{Qr^U{I8Kp6A6RC|BETk(yTvR{eLjUvPKQZ>;DBQeqhMV^e?2? zsqFP=6LM#w%4|4`#Es`)Oz}RQ$0TWi_kS|QT%jb3^-dfAFvYD+?gL{LprQr%!Rfz| z;%f)=LTkoi-H+24@pd?kRbn4Lr6Xe?ye*>1Z)$_56U42li>H|B6tkLpI=Tz5hfgZ@JR! z)3>>@Pqumf1}IJW5w8k0R+2--cd+pZ?|^mQ2hcVb0$;WCZ+zmwl&KR~x|N>wPk53= zu%4ZJrOH$2UWkoOZfj>uk`g#0Ksx}Y!|$7EOn7BBUmnPc1?b@f9FnncN(GcvjH~hq zSf*Xy0VsTXy)a#1M?`D>C;$Kmz}40EAs`AM&Y5PKxp}cyH&CZFiDc6L8=BOe?#X-8 zPJhwX!R3mt1h@)S;u3nkO1(1!%AjlpSa)sM4Bq6+YcO8l>1w<5z{Vw}UtfJ55O!w~ zWF~7YtwLB#Bm#y=92cK4pF98r;&K-=1dQGuvLE#%yx*HZDh4^8OsFqV0&4F6(fT|k z^sNlgeg6SGivr-HXM|f8i!VH?_u8Ypj?;|q{UkOd@fe)xV*R4PK}LD5b&4Axq8ODt zSc>=~dXE5L5umU7Y$a%_s2o==jLEo{CLbpJ4U%%@ybfET1Tg=ci3wD|kr(*9`RE?s z6P=|>GkFD zo!;BN>tDim&4n0oB#%6P0VzZ!6g&<8#GT&jWc6)!S$q!$EWFC_BJc#bzMl!;c-^pE zaWKj>e+MV$fNAZGuhS(CH=|5-3MFpUupNWL%5S}ZFgs383Kx4ug}qSfL(X-Mi(ju5 zIgCE%DZG0YiK`2hRaZ=ojxn#i{8zD2Djd`-l?mk=8kd;3{C}tb55k?&gGI zrdSEcHvvpLRX9WqcV=O0BF`s+?hrLl%K`gvUKS*3#r#W z{AvE2lBNr$k_=#sVztsrQ*@kbAt#O0kgv&SmIAka91#L!WU!u@q_%!?W3h_yO&-L% zRjlS-8#mi*rUO`B#5IXk*{}cra`2p*k2wJk%{uf0jHm-Rc~v|$X~YEi`W}EP_U>om%O9t}1mPV8jv!yR#2Do{UI*}# z^tttXWXHJ(jy@NvbdBh)QhF2w|`i1WDt2mG*BKC?USneKnn^l&_rX_%_t|86xJwJ}%0kZYDaa zw?9?!G0(P;cfAvNweb$t!$QzM-{La!H5G1^F27@|9dZ(xESyPi#^?j=$Q)n==y6j# z+ny6WEd|`yqXu*kJ{1g%VJ5W)tR9Rv4K^M{Ge8Y-s~!zlRZ_gO)+tm?P^0Vn{!NK7 z29maZRkma8-9$EQ0-j6kB&VKZLA zj_kTcpgciOd$h+y@}7^N&wCy%u7U^6pp3F+>mmT~gopwlxn%rVYQ~0{s;7)LS76ie z_bYtl%gXENgfz*X1>9SzxyqCmgsWHI7291;1HU3&6w@=5&Tb0rNacCE9k!eHqt7Lr zmDs!Z`{qaKVS;HcUmEm2OHx$`0Z3_6>iVcc{x1N7KzzTtngz%HEr_!=g24yl7Es+9 zWqp*)TQmj8$Ql*eNeTJ3i*imNGkh)F1cCyRr8jX1_(ua~I=HF&=BIQrrBa#7M9spX z#XmEkhviJ-0Xg7|u~g3w`N07^Y_|)5zykq*>lHUP1juPEN|2#^paWMEt_ogq2q$f2 zMz7V92Z7k#__{(3V^FYl1nz_>gOGkHAb2J#CS3*~u5zUZc29yetN~fsr2NgD1ZDAy zA13DDA+?XV_709`w=W+@CI8Vf(r`)QXz4~Hxud)7CtN4jY(^&kfn~&q=vQI)8S{TJ}4a@$P`zf?XK= z+SGWYlZ#kUqD*90n>~(xYl9a&>4#ff)GK|YaXqc6BOp`5Mmy|7^yo^l44yHWy|72IAW<`sXI0E>wh z+~VRu0VYe;c^78ZC$rjaQxmXLM_y5O4T!3lt9okm^5cx)ZU71r)d8#-kprUjxlzQ< z6#;()e*y47OULdWNkh7R3KkgYSI-cLDn!5|M>j%#iFJ}e^PC?UO)$ImjhjvUKMGIFm?5901DWCQa}I)C@KID z4k(bCAsRPuwU8&Q)d;s`-@$X*#$Z{@e_8Xd|4?(7_2dz0ivI$D1c*Ek0(edZUVc|% z^wE6L*CYmGOoo}UJlKU^Xe;4mN;el?O_Xy-^?*V+CqIaDKuCp{M>d*-Gr#9r z5tUX&<`6aYg?hM$k;8Y;R~Y#Ocq+IxfQM#iSU`tpg5hNt4&XirgD&Q?U-_d(8OK$l z)(~5F5PwrxeAtPe$SHy|f@9b+OXPsc2Z*TX9BPJugd`IypZJQf7>hnZ7p+Jar&vlw z=p2Qp6s2fNwTOa$IEq^(i^QK;jK;_kurX`D=1OBoIKL*0vL=jtmy4uwjMjLK*a#A5 z$c6%Bi$d5pK1Cm;SQ%>=9NBn|=$MWnb%wK6f}^pD+lX^WMU8=?j`nzu_(&Ow=#IYe zYp_;}gh-ALD2>#Xj|O>=*vN}!m{hG;j%7F+&Pa=Y*hKS2DF}Iy7@3gM*npD-j`aAE z9g>kEIg+vXYVb&qiP(lGs4xXdk}m)Gk~}C&zPKGXH;&urO0YJD;pmDm*^@pQg0l8^ zff$HsNL4q7lZ*IlocEL0Azw`Sl!vyI0~3`tmJ#sQIxSI@f42|iz#%$W8K*diEt!>4 z36+3hj%Jybq=phG6Hx}yma}C*n=>wMml8?IHTjePC-Zy=7?fJMkj|r)-*F>?S(rEk zSR!E*Tc?<_bq8t`exL>uXh;u)Wg>l(J3_dV{OFX1X&fpuZLG(DqB$o+F`A}%nqzq( z7$i`M32w5rXcAF*S)(osV0AAsARu;#5t)>*lL_L2PiHoaow*y^p(bc~G-ZhwO`s1d zpbt%OAkRr17{rO%1e>9ji47YOOj)QCmUM=Nx07bo$c z%3&a@i8o2gQdf4J&7&lY*+J`hW7(2^#{+LysXD^aNu0AfY?@5P(%xn4E3uKtejCKc*q+`Dk6^Q65vJRQeBAniq~LsZ;-&9Bed6sfCyd z0SB>%qXZ#XVLA}lWo4m?5GX*MGiVT5b72b50ZX^4kkkRz2W|{x8^1BDDs@&>BdZx> zRttc3&C>xOAUiY0Nv`CCcqcAQB&)cG+{{?tl>gMNHj_6LJvz9L{y{{Fr;D++Z_V& zH&GW6{bySP(5v;jo!Xith$?N0I%z8tc3Vm-=Sr#RDi`D0q>=wh9ASz+$^!tb$)zzg ztOC(J@tRS#p**rG5TH7v2qCN&6HBi*rz>=+F$V$?6HF^dg_saURI@WGr%+ocE~sj2 z7UOC#^gfU?Y)L3EAYd?%ly-5K6e^cmO4ou_*=@8LwG0tM1Hd$KcMvuDvQ$x|GV86; zMk7edl7ea|J!=<}OSzRwt~j;R2@IuNVnN9QxUwYx&B6|ZKLu)&&Qn`Scnz*xmKOd?fhBBcPw5*9fYY^=5{ zky~meH*Scl9Wl#bMrwMEYd>Mfs9;#Gl=`zii>}9^Sq5%FSSwUf9VDJB#EJ5HugPXD zY!s@R6|R4!LKQP+1wjCR0|=~>JIc}k$3hQyRR9Ihm4_u}Rd_gY69D@FVp%I@yR)Wq zdc1UjR~>)=1<(QIun*{)6nC>ueM^T?HV;7L&dKyF#biUEwD($15XH z$H~{FGfn&ug*;m{xHGGsj{7TSoO-C&PYx!ob8NY9%(FLaxpkZycbvZ~v^h|> z!|%H+9rVY#XDvKD#Q8%Jfjq%cOc3e@yoS{POf@i5jLH14Y5&Z!S9*3p07d{&6qz!D zx7eA=+e#*7yuCK-$}|hEuoB0OLc?@y!?_F>mCG9=)}_tUPX9%}#q7tWn`q)R#E8au zSvp8QQxN?CP_O6E|1rh3S}zSiH3ZO39P`O3M>Jnr7%q#>hl-vq{H?DnZD#zvGMhH} zjL)~s%jRnw=-XfaK*9ngPT5RofC~V9Yg<9@KNRN#_G^3`M_Ufk8fpBHh7ijzth4AXx0nQs6_abh?&ZO_m~E(w4Hqu-vU5 zm`!f2%e_1oyR6Iiyc;EzeiZ{vJ)AC8nL+!YMzIH~kdu9H1bWQ;Jy5Gq6Lq+Q?JrKu z#N$#v9dHj#hyvcGOAWve+Fby5V0|Ag5dCm#KZgfhV9gba-cBG|C}3YqhzIT?%`;Zt z!JJ>^!>8qqE@5G^PEZbRWU>X3ux=zW;0IzllL-31t!Qvgm?ei&uKI+D`am=&O?eNUjFl&%$ss}C%A@3as|Ob*8?m)ZJut=kJ{T_ z@*uZT7cFTSd9>lYjugJ$xG6&B^}~m1t`^y@?QG7{ZayTsG9OqH?K&~-@LbX@T#Va} z7VFOLAucB}N@U%hBd96QGyy8)e(&b)&gogp=Sb|{Zo}U`TbV@8M3zaV{2%WwBK6Lb zQ1K*zYVY|@=oRjD?2M7^{$mU8;mB?h038z3oUKzi@RHK-Pa+joGw}?!?_xbd|2>)U zWdZXsKkp_1mx=i|=Pk!1j~^y)B3iY84-bgcXo&c$^1WX1-)fiuFXG0|aN&Z1e3UIl z9a0-#dOh#+N)hoYDcC&mj%8AaOZ1QuAMM-(>#==Vpz#`)g(c6ppG0s5ie*`AM#G!La=Lu-+F`h==DT4$Ap?<1~boW_ZKdQXD3 zM316h?xO*WIq8Qx6PTP|{JlQ=MPcyN&2Y%Cjh?F2t=aJwg%GY==$g7T&Gy%=kkJVWGg=+jMCH=V)F6=rJgg;wOSg!!_ z4?w1W1q~iVm{8$Dh6e-uYtm4n!2tyUTm)bd0G@vVDgwA;FaXDa2s;RLk}v=uj2A5; zds&=Iu)9gp;M&+ z5(KbHw4YI{38vB;Yxbd5wr$5wE%#f%{X03`IGfXA&}2R=%)Ytz|!nSK?! zbgWU_M0@x49bk6=0!?3cwA~aq?%{fmKZhP&`gH1TlP|5FUHf+K-K(#D9e}Zd$^Tfp z4gf-6=f>uPDt3tJANir99j2x~G&PEvDf5dIh`*pH^R7VZuCuKxr4}TrINp>yOSrSf znyR_$(BsM|2d@gtE3h8K&Oi}IB(cQq5VXxi6<1`j#RJ`IEdYWxD(}436icnVn|ul& zq>M!JQKXc96iGdh3{s4vhGY~-$#A>oGg)7R! zu$l{tN;TJHvrUH(WDu#^igL3~JMX+xO2Pu*0005o+i|k-VuUEDk1p$Iru+h!iAa$e zSgbzJB#P-z$R>>vMm#q)EH>H3OwK~G$V@ImGI5)e&cUdvtt+*d0#nP}jN-IbTW{qH z&QJA9buCtHDy+1{jg;)(R$TXdujbE zLv=?L%PM(2bnaxIe>PFqftk8e-BzgrI%%bCB^R&&pp}f!Kv!)vJd=H@r~y2nE$Cs9 zs#b}=)HwQ2p|zqmTcB=vjCkoOZT8gOe4C;NIK9v!t2a@Hq8YC-yK?s|y9NKSH@6Q* zTuh;fhFfLhtWG@ta>;XAcvU0WgV@=@2DpgXWb?vf;=X~zRwVfVC~Zk2mn>-Qaj%}d z#JCfj_uPAVMv=LtUS=64Rqa~&=5=H35~|pbM_x5_hpwIZn`cwdcIO{YzIyA2ZQJP% zG_yAI^8Vq#SBR(HDFI9XP+;283;IaCgb)brNRh+`d1P^4C;$5Gk`tJ!-SBQ%;bsMv z|0yRp1KgWRem9qy$!a&cW6Fl$Ri?e@&w|$rSmS0#u*HSwf*%B-R+`qAm1v78dxD+Q zNRo#WqA-6tX_!XPhXN-csv`pN+I3IRFN!_5`)Z0K;k=yqta74w!LtT#hTt3ljsz?Q4oq=vZTun z*g`2*kctf)V1=GyH{8e!lZbp|DmRtENg}a~uVkF#T3~(bXuS6c#$~fagKfw z&n`29M>jfBVXFfYe*l22kuY*78BwH<9>ynODl%h4Orb6Jv^+5~?<*1N2UNnzE+;lI zH{z6)!qBL^Sc-8rPqfY4cm|cJXw#ll`eH1P2C7trv3bx#+C2ptxe)@Zpf%$p_KaB* z943U80GRTKKx`-yKsv-o1fiT}Jjb?Q4wQt?10|`PG8Mfsg+jPX5#>BdL{?JLHzyTo z76WLv=6KYm-f8J9m3SpN&2Cp5jHCu>c~hbSOpn6CWxtY2xrRCKiwZp2S&?p_gsB17#glcCwQ*N`=k4H0yxnZotub0286$Z`SqjXbM(bw(-n#!+ zTrD+Ne*LRfaq_fifnWec8G!qEm>%@0gfWWboC2__FgTz_go5SkCqr7d0g4niUnw9K zPl+?D)N@=bJ?+fsdX)#Rt0)_4S!QK>mz?@^r$1%hU6TmZ{@t}#vjr}j4%%1D8nn0* z$pk+Z!o#6-0gfiQh*UdLS)O#@A{Z@MM}x~!+_rOiqKa8LEt!_EOvOPH++wFtsZ#Gw zuYcn#VBpS~N!;OPyZ0Rs|J*6P)h>>wt(>G?bxNo8;@8000qn05tl*=<&;igQQZGr@ ztscz>CjSUzP1NJM$}IH2W|gHDozk32&WxoX4ylB4m!;C8_QNlpUk7c=U)=xxcb0So zsPhC&W9q@!#|Un)akXn)4`a`TKW1lpy=t2hx2%%zT_RKrcVhN>cb{VXQd$BiN}Wmh zOtV7rhx6&FD~GhFXEY;@$?Qs{gxSp}I_|B8OWYzeEy!|scNyj>YMB`Ra-@E(=pGuzvzl}P=%da+U?rd2I91xK^^LY&Dq#X z1~94ktmq}CdUsoLbyg3(=q7@xlufZvNu^@nW2^GfXXSC6x=ZP}`O^iAJuY{={|n@7 zbH%ySb*}fapIui3nG%A_$(e9UJRG{L7u{@bURY-p+Ex2z)*q607kiCYZeuCfzjkl7 zBOW`APcGT{_M)fny>YX#yDA98SFFl=ZTfb+WHCk&#{bOnmRmc}F9kEnt#|Hd$0p6h z4(e(FTC_In+~_U(IH!`XbbHqu-=emcQktxnN5a=;syJ+DLy@GERJpXHT-Kv2 zZQ|?hZkbbdaRim!?P{Vo)1S_CeS_TX{BHPg%B#0r5AaI~yGy+QdXsS%F|Zl%0{7d;=anJLmlmo4{dApl4?=H+iy2Ed5-&WgzKls^HnU% z(uOMUGnL78pO6MRNZD?)+4mlwm$!SffiL&Bg9?C4`TKmo3R-NFJ@BgSQn*Jy)0?s+BKZgBL~nZs$0N$kT$r@?MzhgRP5 z*1LAli66hdiZ*xqzPXYl{G+r0RR6#cSvvGvKe|Jh^TRX!8Y$Q#rzJ8ctOF(kjJbHS z4KC}f-Z;MZbGZd{k;mIHB=W5#!@n@P5)*_$oEkpPn>!ke!1T+s#%nxzYOzh4Io@co zet;_q{JymEka6m)un<8&3m}4vK{wgD+k>^otF4}S!Yh0c6{xr^M>02R=OCV)gqjKoQ_#7e|ON~FX^e8ha{#82eJQ1nDm1Vu)y!A3K_ zQz|_#5hrnaJ#%^v9kiV*0{_8z;+f==HZ7YuEgOqN>_a7+w4VFCo*O(v^TlIa6%}X# zeNaYbT*hWp0RfQ4X{5$#w8m@1#%qiSXXHj`e8z9=MsW1Ta3sfZj79+v$8;=5bu>qI z6h~={h!5ySc%(;ktVc(*$9%-ceZ0qh)JK2h$A1*afFwwOWX5bn$b?kLg~Y~FoWM6k zr}+yY*aEkcGsecCLjPICJ$o06BpfZ-LuA~aNQ*0ds-iO+uyuPukz`4LQGpwr2MaR8 zMAS4TIzIVhm#HH>3P~X7NIl!aMJ&=kAj}d%Ye@u~!giBI+XKdQ12F42%BUof0C31U zd58~?N-o-+oiV94+$6h{&%z{mqPjdJsc-_oti%vuoW%g+F}Q?FdU5~D zyEGR@6h5rP%L@vXKQk4)3Cv9rJ!(TrS)86&Swe=}B-Vo|U2Hgr>r2J4GVlAtR(dyz z%go&Kl*#l=t9(H8#XYlj-EWspHxi@oVn!0BFG#Xt1!;ztF!_f#tjNO zldPRfyU6S$&rK6fJGsj8lpKqkl^(3Gb*VnlqDe^dzBw5V=7YWayhHwLzP40P5o*mr zR5Qy2s+45I`TWlWos5^w!a zZMAgC%oNly$Xmhh1iv{U(HH#=eYnBVgi+d)PHaO<2+hFp{0_!bPR1myCY!eC6w4YN zERqZ!6je!KB*6P~J|l%v*kDjQ8=2ya(g^fH)>FQRLpdKr(BrtR+Y!p0Bs>ktq!z=@ zD^&^PxXm-;tq$bP0_9A=Q~%RCb(*S#!hfJqJ1rLns!lIG!VkI6Q~I{p3K|rNznChM zRU|>?Br#p&Qv)%xU4O#M<~^;U%m);zV+Z&kYfdqqpSNf3S2UVNJ~Qp~G>Ag@fz ziuAG~h0y_=Q!0E@gu7H-4McJUSnCMaHVM^$HNU%YLvxDHxiJ;EYt{%o)`kO<_>9%r zd>Q|w)&R8`-O90IOl&(`4MT$!Sss~EquSGv6)r&iK-}S-kX6<)RZe=f&VUmjF=fT- zBhg*uR`C>1rA*O$b)4MnR+J^$0FcoJC0ZzCnTQO_h$Nom!`PPMgYEKDY@+7ft`#vn5%WMB9{|xk7zIKQm0LTv_=PQh61vM6D2}9a?0p z*2~;YeQ7MLL?z+{a~B<5{J76{$p>8!i2xLA6De^$IZY%)JmQqT^NE z11Yb~R+EEV)h*hA6|TfZ-b0O9RWuU{8qzal+o0qbYirKA9ZCd(ip`bEz|B?g&A8&F zODUYz<^|PV3jpO^lZO~r^c4|R-Rp6muU$|r51-@5Mwu;$l(^5JuSNc6QF6|0+^$NZFHZAksV-#QS+*a@u-g9xm zjd~7~Cz!=~-PrQZ-K(d&vZZa>5IN-s^n*vj^i{2Rar^CXO#+=`Z`+f zNS?X7TMb|1{n%3Ws@jn0qfTXUK5Ar}Xgj`9d!bHm^D;x-I59@x6CGY6Jn58LYMEYN z^kXVk?&>wf7V2FY>#?rrCpKkgPF4kUj+yjp6JsFb zs~dVCfWhtndZ35HHjBef?8ZiH#%}DxmTbqS?8)u`#!i63HjA?M>-8K%v>0qWF0XOw zsnA|pwgm0aO>LkHZL2_v)PC*E9-6wQ?R^$L1}K2s&g~8;{{Y_h?cN6N-xluRChp-j z?gF3(;7)GdUT)-8?&oH1=yvYtrtays?&ijB>27Y`*6r^0?(XL9@b2y9Chze!@54Ut z@<#9VR`2#s@Aqc!_ik*hfN%GX@B6mz{I2i)#&7=C@BT*b#J+CrCh+RcZUZmy0Y~r! zSMUZu@aZOR%Z}^`m+%UY?CxHG1^5CD|AH?VhXEjm0T>5zAcs64aT9+46i0D#NO2Zt z@fCjn7mx84r*RmUaT#}n2Y7&1DDe>=fDT^(-462H9`XVx@*@}WBqwquM{dVv>?eQh zD0gxxhw>@6@+!x2D%bKWkMhaJ3ICSw=g#gi7jxhi$nr1v@eWUOG*@#rU-LC@^A2zG zHS9I}HbU$E#4_I_kp9fMO^$$>WQvZN?ICWS500U_CQdcsBrfzju4b_kGv* zexG-K|95fc!AgVdw2MMfB1-p_=%VJil=ykkN0}lc!e)`dH48u zUwC;Bd64gTi#Pd;KlzkL`IT4smiPCMhxwS7`F(f!nrHc&xA~mM`JLDKp6B_WcX)~L z`FbCO6c~g-XoI6y`Ue<BzUTYBM|!^x{Jj5rrBC|2 zPx`)B{G-o%!DsvwFbJBT_<)~zkN;P9b~nBXrOwH?HSL=9Hnob%D z68*nkG`Vo!&iDKVhW&tj>f#Bh(igMNT7A|Z6Ti9q*@s_52mS?4XO5g1k4AmBLXh2$ zK9|apDI*~m9{$p`?dkV?*_4^JQcc0+lI~4tUIRC%D30Rrww{(JEn^M<3O&DrFKB%r zdyW40wf@2t{`p7WF;VOM7t{;1kl+G{00IU66G(3$!T|jO(CgRmAVh`|BT@`N0AWOh z2@Y;Fm@r|-0RKjkENSv2%9JWiie#|zCCr#IXVR=`^Cr%mI(PEy>GLPhphAZdEo$^A z(v$(3ByH;SDb%P^r&6tI^+zh!tU)CTyr@v%!mk4ldgS`?V*-O|$#x*9^(|bKAH5D- zm^Q#&ya~hVwJVk{Ux5k2<~{3BVA}wRMZOjJAui;|k|$HHZ2$Q(=BWYlW$x_xGw9Hw zN0a_}HX+&5sLzs3P1r2s(z0Qy7TmgPS-lAP=1p7JW5}`%87@qnP;$%J%9k^5?)*9Q zf&!vbuWtQ1_Uui?Vq6{eJJ|3F4}!O7@bJRK+ChIDPVcVQyMe>EUMpPqVDhNT4@~Pf zHgf;WvmSv38h9Xr?5U6-gAF?PAcUI1CSG?Fj#ZmrbrlvNTo~qd7+3C5*xz5Zbre`g z!5x%YavZw&B8)LI7=S)9+IS<5IbKKKeB7;OA&D0r1`&ulZs*;53=KA7f3K-!9+bC1 z>6LNsftVYBm^o%7mt7{9rI(cjuwa;Fnt3LgNwtS1hKYCCXI5dqEtIBDXgPTyUSwUF z+Hk7%s2hK~eKnd$1JZdYqKTfll%kC~`skt=+9_ChJq8vQd6}+t7o;(r)s~d{jfdxw zxJfEeetL1Vr>Y^kRa%Rk$~r5p)ooTQuDR;EBSLwFda9J(?WU(kyN>kHVkLfMoR4G` zMiyP&UCO6;{V=zsvDs?>yKPnhOvf#_;fkx(M}JmWSCgo1I;wHGe)P{`ecl#lh@}EI zVsSnyS?#OM#X2s)0Sipim;)PpFv2;p>0w~Ony68~kbc@Lp8ecKUP%9Cx-4)Uktk|^ zD^gcz!X=v=E{!LvymG=~St${wyq$U)dJnN1aibHrSL$yQGHV#Ru4YJ9cK@<`G}4F` zoHWx->#Evs4ugc&&^|hftYQNt7OilVit5#2YuQC@c|@;=tr)Cf183mRLs_ zzqwdQzk)|Bo(+;H>^SPFE9b51t-F4@R|wfgxwp^Z-Gz5x<;!lAGYiOERyr=B z4TU~*->rTq#38;2*A{bN0 zx-gOvj2QF~1H}S1nAz_@r%BfK(1=4M>V{+QLQ??d!GJH8v5#+CnjZsc!ma@cC_X{s z2(=*AMgl$Yf>4a&Ml1%F69G#j#G(hCzCexvkmCa%(BuFFDgR2ht#6d3Bp}o%#W#ff zV>tFP+m|R9#W<=aUO$o)?*`H+^O~0-oU+=#7tzqa$aR zL)hFVS05+_SkgL>$oGsIK7#>IdI z#I&e-tC3NYYBCzK@P=Zm&~s*}Ifd4As|y)0%kOWC~wadDje?9)zDm?O4uoLnI+Ptj=AxdjzQOo2=yj@E>e zf>w1Dtu1b4MY}55bfB7|P-s9ZON8FicQ2!@gQn920W7z<%zdtNqYK^XO1HYyrLGFD zYu)W?x4YbJ?nu7t-SLW7y3alDa?vXQ^|Cj;?p^PD+Y8_MGWQ=UKyGds_tO0GS8o7~ z+?vXw){fz~F&RWmL9AL!o8GIy4|=VPnBw5G2|ync?(a%Pk&59BABvqM#fx9`bX>+_ zMXC!XF2z6$PcMB-M$@@)(FAZ`4u3POG`_Ky&d3^%^ynaT5z~Pkso)b=1jZbKaEv7U zI2BkJ$M&@{a-ERuiEQT=$l&MoIZaN`BF11dwJoAMCb-J~W5MVlQz{Bwr56>7chV+z!0; zs)r@0qs(09f}B~{6bmIAe3jJtbflFgv6=qtT22ExyOAC>$~27Y zUAqyZwS$x3E^?8obUDFS?eZOQrD+%~JJD$7b<_3>?P>E0HWw>t7Of~jsC1dqC$1?% zvn6Xk)7nL2YBIHh+2?e#8=Z#)2}8|u5Ij4@#8$oZEWy=m7wwyP`j~OM<5h;w)djrZ zB&khA8iDOyhcT%bjR_%$D!m8o1*lKa+?P44hC(wf`vJ_O?vc z;aXE$-5C^mH`-eA|9mC$2sL{a(_=?9rUB}I844;-!cRk{IFM7ovB>-^v2Ohp&l29tZGr++v?SH%N->=~LKivKHcfTnw>nYeLX;f*sNJNxGmJl9_VIAY_-hpV}PYeJ6 z06-`eMQq%fw|E-*;fmOqQOUI*{mEbb;hqP+)ds>JRlVB%m7ocpAPR1q#w{O9@XKLr zO@{o_vA9Lmd0_C#7WW-l4w4)k001C}p8);?0GLKTP+$epiqiGb!;#drHPpc7i% zPb3r-x?L4s@SzoEAr@|-7Iq;Q;#&%ap%{*#Wl@E2p;%9S3Z>1Jk^s;D$(s#=8DgB= z4JM5O^4<>e96tcS9RQ&KDgYlw+64OH5!RY0MWNL#VHCO=B7RxclpPtSp%+f#7gC}n zRw5>5VH`#xCw8J^Vd2$T+E=_%whc``bO1ecKv>YB&oP|wAfSQ3qQZbc0}|rF0fG+- zV(As4#_1gf9#%-<8x;Oj+o7Q`zE#nnTmjymCwgM5F{3g@BQ#FqGFGEBR%0_J9~8Dl zM-UAGY(!ej9$tK72XsW?8Cw*gk1P_%9qxqp*->_;C_dvxivQGs&6%^@RDgZs3WXzD z7SI8d17ozMP&A-y(xY5r1#A`sEefG*jzj|vfI!USY95Pj?&d$BKobRk1I%Uv3SnUu z1PTCvarz{>E#}n-w(bU^u`fNzq-5HbdG*5XQJLIZ*Teg%Mk z-eeE{r#rqJb>_@{*`HNtriXrLLKQ}sLDE?ElUYR)isDoq{l-Y4s1=pyhsLOk&M1v; zrtOVqCKlv&5~E?zg9iMl1`Gg@W`YI~=|B7^krL^Tmj6SM0;vWZDU(tf0#F*IB>;NX zD3&&(Z)_=EaA^W~=^LQL0SG5t+(84@rfni-Oqi(y-Xc5}1TIFWQlLO_Mkt#$XF((< z01TSZW$4xX!vP#X6C8jF2r8i(YM~zLpeAae`XY50Dx?l-qz0;_Rw|`l>ZE3>rD`gs zZmOqtDyVX*s7h+125O<6s;NRMs*>uVrfR9WYO2O6qaNz2A}Xz>>Y}!)qK4|Hjw-MI zs;>rXuMR7)605KlE3zJ|vKp(g1}dh`YO6x4w8|>AMr*ZBE1@#M5@;*8ZmYL;D-(b# zw~i~hhU>P5>$sk)x|%DyitD$k>$x7lw#q9L9RGkGEWx~j>jBiOv-YdA2Ee&$D;Lab z00^wRg6k6Kfxk-XtvYH+_$$Olti(<%#a67vUM$9Dtj2Dv#WF>9<-^B@tN?)Q$bPKe zF-7COMF()eKLqAnswqq)XSpq?oCd&i%Bf7gr32y@0hFK19-N-?sbc!6-WY|IBEXj- zZ3ieV(k1}Y9&OS>?b1fA(^75IUai$;ZPjiq)^csuQYiw=RZ>tX*_N%@o-NvvZP-q! zl+pv+F0I?9E!bMAlv-)kPAT2u?b_CDIZmn4#_iJft<#{A< zCJB{-V>#BY+d6IIc0d6jz}~_x@BS|E@@|)YDe>B^(pGKXc8c*XFQt)$-PZ2&MlY3~ zM0yFRQ8)nrC_r$g>7WfE=Q$?;z=49^Y!mUMP0B<7V6OsbkpSu`p4vp868~-LDGOg{ zTmucx{Zfr)juQSc2k{QCZae|&i2k=A7Mkp!AaL~Hr(Ec$m(+16)(cDN_!c5&#{o@B!7=2k<+|o4Ovq1Q>x(8*hD#`ged#O0F>YP!o_Xk zVmYhCU=E=ACcsP#z#gVEOH?SIo?kEDvkk}5kfci35JvU*lX#F55>=0Q6tV9d^mo9L zA}e#XA>RUzL|~vMN5TtQ>ZFzUSR{jGC3o>~NHi5{GBUf=PnfZ3JSPVL=uMb0F7Bi& z3gJ$g1e?CHOumW?pR^5=?}gqYKJWCw$4@_nkB19_O*s zLeM8`kR_~*M`hcGHIL(6a5Wcha+)l`(V_KD{ACbcg&E7Ra>8Y73}HIw0cw-Pfd;@3 z?kV`91RRe9Ps+6OVgG1f^K^j0Xjp;>U1*#B1Q>2)OoJu1B}pF>mxYShoT)Jjw`C(8 zo}^?eMo%k-&Eym!W0`uoBLPq|M`w*StJMp3^-G!w7aSc)|3quQF<-C5ZL05D|HB;| z97(Knmdy9g@@AcmL=VpQZqqdYe6IoNba3A@efSL%HI$VF&Q|k_Xf=!xHC1#o+Mi?; z+i9khC}Tisj{rimZUA7^DN~Z@#|JgG^Iml&TlQ67G+BQ1HrF?EkoHfQ_H&r=5cVNV z$1+>j@Je(5Y!{e>zUkoEBaSzC$DPU-KOeL}c|glg=cI;6Vil^qsPwt`0G*LpX~@Uf z&r%BYmJ^s?^#91It*4jGbQI4H8^LH)8=h%cwu@GFMzaNbYqwd$2rHEKDI*AMI_OO2 zGF$uhOT4A_(u9!%0HWW-m@v4L|1k%}QI`L-4Y?B{7q_#B)&_0Rh6{}nmBfm7B8YV5 zll;u0EgFHny9g7&cc3TWv7^Kg z(rIF3f}ZX+TSq7$0y+B9qJ`EYz)iY?_fHy`N7(dCaVj&7HjZBj$5@7BT;w7H*83u7`A;WK9>F<0QTvpNlqvy!VbLdx6~M zJSwLUqU=Bwo5T>7@=MTS9v9Jb%CiAYr?&Gl5{H=ec)Gb~#1z$0*oc$4I2s!@x!TAP zYbjdE|C&@wriZXPtJM6tj1-;=P5tD%zQ>=Hd1ZJE)Njx&9IgAVPhmzEB%g1Nus1z+ z=r~wAJUe22T*AR~HXsK80=4614#Px(Ul>0SIiv%3$WuBSS&rm*OqK&Q6Lq7{+)MbF z6z#yg!6;vcH2jB}=rVI4jZ1H50ThkD$PlDN|y6 z86ZK-05Ngi%vcj+%b6%w1~rM2U_W{Z2n@_fkfqC*EotgxO0YwRiXcg<9GUgxK$`)W zW@O2;YC@G8&3+7Eku5^4SwqGZDL3m}l3Ovh)w}=K;*zy|4>tMt7jR+2hZ71WjF@rb zt19#@c08GKWy_Z_XV$!#b7#+=L5CI%I4V?*IJ-Voy4q#Oqc@@2?5Z>6)Sen&OCG(O zw_4x7fomK&{H$w^P*a2TI(untnniIh)j552QsuK#j%2QLHgo4Gf1*6e7HdJgb*F#5 z+4W^us=BKd&ucLFXN+>=!-Y%y?%nge4dX&Ly#yExzylR3a6kk_3qYSH7GzMu2`Q}5 z!V59XP{R!?3o3x5P6F|^=~@EMDWG6WaWAFXu`d32M9(+= zo~q5JoMv3fwvA%y%BU7mY;no$QcCK)udx67P^#@TG771-KC&t!yhdutBOfS8C6=OM&84OBN@EY1WN2ZHmn3 zm`b3fkoYPO*!#Xj5-lTP@|C29>?~DOLgAcK+Wq8=E5G?PWCGfW{`~YX1-(5?pFzVd z*W7c_O;_D@)y!%r>6T(IBQsCrDXQOUvlqN1CA}^qb{&0I;Il|2j!lmWman|>c$(>? z?qtMr-!c7V(kJz(E0f8c>gD&g6{Dm>$*N!x?&v%bQM3uK63b#%K7$9|?X5ZGq%%G? zFBI@w0|Bn6+nys*foGtJF52j$kxu%mkVkqgrx%;URjSvtT(v}*W*l;3o7NrJYvBl1 zc+}#yI`~JiPlZvr7YXvt#gkMO%i)k?Dv8RrgTh$Il$Xq|HN#8}o6e(PT~8^zm(vN@ zhptWAW>EpGd0M)x_3usw^)&2fr4s^|bA>klT=dcZNiW@W3q6e6v`WWP@6uhV!&oVt zSiep+r{=+$XBOsa&0S(f{r7eU`o-++s~YSS-=-A zb!Cl|L}@TvNqn1ti0CA~Hj_X^P z28)M5;|VW@kYdfF?3Ef8fklRXOVS|`XcN_uMTICjpJUXLCal$ z#4F-&+EB?pawBG>_z){9ZIwu);uc{UOIgk`TzT^i#QvzZ=`<}%xm3;GRwJgE za1V~V6Wk{|iAO$0Qehq3N#!Q@ zxRt@gX(QMP%ZiA!8X!s}KN@sqIw5x;tSE6>)T#>3D09w+w9=JBtD-?^nNWo;l%W&~ zrdMdESU#%HSE}^nOkyz%LUs@;eut}7EIWKzaNfrS{wn=5?5t+|IP~ejI zIJ*e(l!Xx}LtZ&Lf*LKX4K?do(VA8pW^7U(Dxq6bmYj()$!%5B+BHK6(vXrWj_Ru_ z*u3|$qOufbH%zAb)Z(`21PM1FYY`@QiV>8B%$%`AH%7h~htkmLqr z^Q4F=rAaKaskA=&>=G&av!f`SDz&UVtY7muypl@gUPD3Ny}YuzeMQuTtLr0C zgrupMqBm~0BNII|vroVxRy#A5PW3>RN%ppkEi{|iXP1TGn7Q^NgCSx+mngr9*!HUn z5sb_d@GuS2WWyQe@P|Dd;tr1(#3c4`KIkLj60carEgrFkVa(weXL!aUuJMg!oMRgA zIKv%C0E#OtTp|AtnaKZ3;hADeQxBCBfdOW+Hn$=>$rd+97p$^)TRK>inzV=sOD!`| z^_#!$_c-aYkYE_qx z)U8%QK2H5=S;N}Zt&VlARh{Zo7ihOd_VurU-M^HL@u8Y3WO?OW)1Q1Vhvda7Y^z$w zPTo<$b1YV%2|KnSZr1>YKzYAe+GXcU`RY=p8Y|80{Yr_#XlD8v_e3Ad-kLA9CC;q$ zJb0;z@;F+(I1@|%O8p>>o$h&Ual=v3IC3zwwNhP$*cLHOaJUNm(FBM~JmM3#c*QYJ zagASm;~Ag$0R#Z?0~o~8AQw5Wx#jQ!sGQ|ZSh>qv{&JaH+Tk#_`OI5xbDh6@=QZc~ z&wU>Bn4cWv2T(!L1yJ;(C!Og=M>^7%ZgQxD9Ogj|_SLbT^-sZTzE4Y|q`0{$p(4pu zxk0yPJ9;)Oue94-2J8p%T!iUcM+wtcny%s>dP` zam_O*6)%u{MI|#D4sWBa*og|mRXwW~Y@Sv7m}yJF>B82ZTnzxj@zh$quAcQ?`$`+z zh(L0OUhB30xBcyF+3xb`eINtOti_mv#^j!8g&gykv+dqJZ-3jc!V8{vnIC4AdVBiZ zhI{nqKJD=w)gv`Is*))NU=@~8dRv=as=jEX8iK~U#o~V7_no41)5iE#LVQ069IW_Z zm$-)pspvtktLTID%0@2eK?9heE6T;{aIXO!@Btw(l)$HwWQaPlX4qCGf^vgicF3}H z$)_UkB|^~o?5O`v3fu4uk5+5THZP>w@A(jI?a%@u$gKm3$GzrH-5$ko=C9A*<^pl2 zL9*|qjtVdCib?+AS1iy)mc%q>0x|eV&+I8|IEba#C%>%FS9;|4b@IQ_VGh%q60SI4SHv`XzxM? zGC^m4aUv=IvLY>#`>Y9(c4GJ_344^Rd_Zyb5Xnr85MJKuOj@TKU(x@raoV=c?Y0rz z9BalTT>CXzzFc4ZRRCOPJY zc487B!;JYY)y z6v>5@B=ML;9}lAi4Q-D4F)zOBG@~&@=8R2P~ULnF}Z_PSVi*6agR*nyyWk(T7(*#@45gpAU~8i zFvOT#0^hP{ZhX>ea^sgoP%%AZn9OqsiR(!Y$c21pfSdzxwK2YG-(X$6PslC`XxLdimg!p zq&>{BqO2#TKCxjUNM6_rt~}H)&(rM+>mf5_uSOG*gy~Gv)FxHMH&19p)(>?)ud&vN zJijv~7L{t&20Ew*eA0^^<`a_oQ2LZ_sR9%4TJa$|b<2P;6JC=`=A?DB#7eO=GobWK zUG-IA)m-+=Rs85i<%NWhQO#Hb65SFHcd&%0NrcAK8NJDw>T{}yOPLbIM5l=nk29F` zhK+Po4s9wZp0Gr-&un^Vop|&);)_z%>{g-m7)=B;G4%waP~iB>1c&nOQnOw1^D3RR zRh{(y?6n^*^D-?}PY;7sWA$GFHembYS5xF&*aK=1CmJ!zC@0j-CX^FZLXX@}a=Fgtzr<@J( z<_+Kku0CHtwl&5M26_es>Q?BXA%ymLK zOKivo&0-JE;HH}VW;K)rW}9MfCu6GWbV!rc(5w?1fp!%q7qJYZAI#wm@+&S9mn4j0 z|N65cqPA*Dw{%UHsIqn^DGSM(OJb?aWX*Ovk&Jp|R!KGrVhVS(h9W$A)oAz>wcOP> z!pV`omZP+G_YCyTsIxqt!c`8FV!ifZRbo=fD|*Q2P&uP7^E5<>HeD+>RFiKyQ7MH=Ed!58N)}L$js762 zMHR4+`LhHQ_!#l6jl!fwM?_VKkq*ZfP-2oznqo;T|8L3OYJ6s=Z(zeYoR_T>uYfH= z^VSuGKMMw}%(^mFCGP|#H2QRd__17HBS20#PUBciGQx~e(LeIN~%PId`hH&Vjx zH|0^5P3N2YvQ(ltr{3zK&QUeY_<1wxby);^+gV_sVuaO4Cl$$qed?zCs0n96ef3Ee z_flu$m|d+6^pG`F+x4^j6KP-CO3PsbG6Da5Y<;czvN3xvuWwyqjU zKKb;hpILr|`BVBq9t0#r67rvXB(tsix<3kC!?3gYHyEh|CTx$a>+U0TYMiU%btyId zj`v7<6`yH`oo5?-({?Yl)1Kuei_fHf*Klyd$VxB&H4{e)Kdp@f*>=2LP}{Vum+=^( zy(Izi_(qs0tFe2+G5eqCg1g(SIKU&c#rld$8_z}vle6i`)LDAF!!|xLz$y!_KA2w> z1z+m2qaPVPO4-CShGsXjZwe?B)y7OaT>t)g4^xo9wVRF$@qoGUmm}gymHAVibOS&& ziMWcwsr;dr*Tn-!rlfHrYnV4X*1!VtJD~)=%&&^)8MG-XiFrJGW(r;Oi%pZ57=7Zl z&G;r3)kpr6JqxwD$WuAA1gabgS;2h3f!7aXdM^#U$YoHzgDe0Z+{qt(3lk2RtGv>4 z7*$k}md^3uob;8ZT@!eJkInojB2yg6` z_C?L)MSVzma*}v|{NK2ZMtym}S&-4`SniNXFM_$r8>Iu9yTGcYx-GrhOIHuOJj=zI zHNK*uzWj*|xt)56S9i6iz_!WoQi2;(dcElr2|`wHNO^2^)L$L0Iyq)5FnTR7QlOFk z=y@pqJ8};FCHonUhn+|_@z@DEQH$du%p%em#laaIaF}Sjt^fVvVfDZO%bP_crQ?(} zX6e)KG}Onb1NUYWsY5-_l(EECqcP=q*a)KHW&!t0)Gw;LQ{>L$4>by>hkm`qGZbq| zgYv3}f-R(k4W4@gbnUt~(Svuh3wo3Ofyv9|$%!h$vD}+6e(T+F*n^#2$lDl=UfZ>8 z(2J-@wgV0qOPhAeOe%>dJyqDoOsYWwwRJ z)Uob`Uc`18zb;bIE9PTGI&Rx08|65FNk~{oJ&w?F!B}48Yb;npgKSvpgdgqVFTo$L2NPWH-G#g5N;D{+u8l=Q*QwMuR6m$ zz2iBy!_Kd6B)QSNjfWQO-1JLVNFyX(23>!Lp2r;6>i4-X&ESP}a zz=8-bCM2lQV*mmLNn#8c5+Ol>9Rl=AnbBZOgfIKe49T*j!2l~$8tm7hqR^p4D<*7t z5$VwYN|`QI+O%a;ry{9(oLaRi)~q?R7G;=V!-lV9%bJxKcI?@;5}9_8dXcT%xpeE= zy^A-m-o1SL`uz(yu;9Uj9p;2v5NYAXfj0^?IsdTZPl1dpLkwV;Yf`RL=_PpS((}%P zJ9mO!Sz0E?ov0J~a+2jX;`wYBBDnq&8j{TMm!&{icgUyT_# zMwh9bixvnPcGK3AK{6j+)~eH};=7tZKNay|^of@iFOGEU)wo~#ZIB0^F9CUK^Iv|e z9%lXxNML~m9*AIq3NFYXUm{`EV1)me=9+XKNm!SF4{^5MX6+^99B|mdM%_g?Ayiv! zt$}8fi71xHlSbLOgxX42q2!x|yk&(Wja6YdQB}3gDC3AL?Pg&?IHIP~ZzHy((T+~> zmLiD1LHScgyW!>;kPMv%=6LNvHD8!ulBfyhgk6zWUYnZ9ry)^c-7r*~5{C7Lgl?7R zWuJcj322~#4oYZ5RFSDzq2MXmm1Hk@XQ!fHTEt;i>AB_|X*iz7sBx=FSs`tx;RM-i zP9jMgqai*A8$t^uIihWyh9qckopvPIajL~Sl9sJ{n%Z?I##$<5y+KJ3LP*gRY4ipz0SmIh%9@_1;9{QW^P-(S`XQAje zRd3TyKMgg3B1LVlm~bUZwO+Z+j4-hLV)%7imHup1ZU<@V9j7`n$r`~Z)tKr?=5d#7 zl1B>L6L%NKdzr=}_4cEm#k~k?*RkPBt80=CnIuj=edcm%$epdyqcehZTFQdH_0)Wb zcHTLhK4UelwqyewbWt7LV6M6yXwcudoHLzu?Y7^JyHNih=I)?oyk{smb$avd?%1cB zIGb*vcBGWCR>H`v$t`lFDUvnfG-|*eqh!=YA2-{Ta9(OwYI4*q`w{lg0ay1)d$0B) z&Di-GBEHX0Hn~eS2O6f&<&o~M00C`h5As&f{^dKPi3Mp3tXG?U2f+wRa9^~W;9PRn z6$`Efd9>+>Ns4E@^2q2m32PSNu0uFp_2@dV>5)lLl{wMW#$@KBPLxW7DuobbItP>- zkb*`KZOJfSZ)$EdR%PDzLho0m#=rlE;xO#SO3nhf}!FKRJc z^btz9_F~1xyrFeZR5*(db1z+(nrLqgoueR36FFjMj}ZPZ$a>Z73Lg?GJMS_L?J5KBVj|5gB(X!g;XSo zlJ!aC1W{3L+M+E1=pkIz@_6zwo&QkxMv5GbUBkqkc=CwMWGZumN?GPt#Df@TQffLR zsT*o6lSX)jurv8fRrZ9ZL<(h+g^&u#3%xm`Ot$Z5NNFEZQi&X5BD;lfhWqoFb_X>5QGwbd!3>^Gk~Yvxf<+NCMuV zfp(S4f)$jeNl%)`45D}mIhS?Emb2$;*;u)Y=$sW zpGR0GQ%j0uVadY@gM`MYf!R=&Sz#&%h4ibgNfk0XI#~Ni!!@o|a$nS9)9C;xKmwv^ zt=t+QO?x&qkKTuYd75JxU5eMd|57cOdi@eclX_B*KqfmK;n+4gh|tXKDq)Yp-%)ub zIZa(Gp#VK34P{lr8_Hy3b_p0)zRA}D9S~Ha3Mc+v=ouuU%|%qHn`#rP*~Y#IG-m@D zRkI_SbzQWq75!@gZ3)~k9&e1q#hDrB%C3AHMZG zHs#UZfWsx`;MTI-AriqlC^5+*HErvo2O6VF$*P9dKb>r^qe6AnjHt<~BqN^A1J!g{U2Zh zE5BWRc*KVd6+-bR8xMBV|Dl;In0R|s-pW!XS1Mj#jbC|U;8D+f_r#}YIl9grC5(hw zjZQr`OV6M}YFScTZFhExN{lRyCPUmXpxk&Z;08#5m+~zyH=5CndW9k#kOvzjrChzP zSj2R$b7PGBTrT1nuZLV*te)E7rgZmSox)PUoWnfnv{RD=4pUAYIzp9|#w16qn(}fg zt7}Rvg!tN2;eZ;*CL>a0o?VenDLc9Fwyn`}OMrqwh?rhN_?9Lnt}oq6K%vP;mQp)jqOb@j2gQ9Bsgdbaa&d&(WmusvoV|${JxfWLjii&u zmd$Q{4p_eLmIz9-|0QkkSU6Q4zO#jLtF4Mtx#e;43Z5$Ta-bwE=H3cd&%wpkt>KEn zw+a)aB1NvSgG}s*OFT%yPWI^hol**|NPL@aYfaY)YPLzbD~|KF%9K?@5h<@piws#k z?WYtaV>4(~0{3w0yVTIM2*+F*WsV5?bA}R3Z&jL~l!Q7OMBnf;`+JvF(yB`gMhoDx z?&2;v%9V+9cwAeLIL0Rq_BtE;*k-Nn+T!+gz*HDaIFh0D7N@J@!OgLlepr^D^JtLM zQ+gFI3=O-A>A24PfZQ2ytXvjDCH+(9G?XfeOugBf)7yqG0?F1QFK${jO`~SsqFLGF z`Gi+}fS(`Q5w0atjtzwLW)z8Y*c+1e*USHYrE_-a>CArLe60jQS!86!9%~^anY>YL z-@A8+en?bz>V=&auab^zS$;y+RfG4q$-pY2n8kZGh_0c?kN9&U!&uxQ%i-NFtNolj zE%a)A>qIN+ah++)tOxJ=Kh8+Z4b*k^m*Kzvol9bQmT=YeJw2i)ST!2a_C7NNDT@?Y zhlV#zG-ML1(|bHKb-eX?Z1R4(hILsN5g)aBzNUKrXAw%sgwm84z~&}qcOo~_ zTEB-pBF9fLF))L~g^!jnlMTY2Lq-Q1%CN0BNguSLNzt)5sa#wbigOdnB z0SJI$c8MAUaoUxD6gW|lUDDKRX8 zr*6r!fdQ6IKejx<7dbE^iv<%zACYLa^L}IIgShsCR~Iz%r!T1k5edKrN2o5Tr;V5> zhZo|F;mAAo$7jJci{ki4z4#g~D2&e6UcQKj*40Wm6e^`cRObkS_5>X;Gh4fdi)$xo zBj`SyVla*cPnAb3{Dgz}(-QwQp@n<163nNErXfY~W_JYfYPC~qID?26cUBU`U>j*V z0Ld>Om0|3H0P>)X_*i1}f{rh_eu4;v9s-jyCu5HFP(@Ni@D?oXvn7z_B(m`_C^CHD zI8#7GgnxlA{wkd5eATwYm|b7_X3h*CTwmvpI3y(oRThjiTZfR2S!F%^{b1{zIAh1V54 zDHL}~xrRR0P?yA0hm=zh!!{E+PJZ@d(?b%D0$YhjMdoC5Gk7-%kUO@Lz@MwHvM#H$3Tv(@}@nVUI#5gF2Bnji-2d`ELR@VLRA}XE|J~ zxfNgZns(9wYt$|Xz;NMMU%~00(a980=#2lFk3czx9%4TxF@`P|nQ~J&qEsOZW-ES! zX}!~6nADDF7hcmdJ$&b6vyv|-G@U=EC2W%&GN^pj#CRG(Y3liviP)YdMs*|UKcaJH zxwdDsnV%}jpYO?B1PY{JgEay;G(kETlgCs{_DzdKSbL{Je)g1_*Dh4VK?7AV@{|{& zB~|biX^yv8p>l$aq8d5YWdA*~90x_Ehx1~zr8qx>L`xAtX0}^CI!4ZjYp7|KAGtes zwGR%bE(q|Jm^gYaYNUl~p#u6qhI(g7hd&pnY{Do?$Hy8*C45pUFXu&QX((jYH-x28 zcqZ2m-*{Dbcw6e^BqmBC0VZyYcXE5AZ@w}?FdC9kS8zK+b<$W%_16$789U>G07)pQ zL?e@mO03N`q?@Q`#TpoM_oe$ZFp(CFhSYtYW|w4@bAP9)a-pRo%32iajvG;#OXMiw zYBHrllnKb9P$p3IHZl(SPwC1yaX3Len49crj(J+Ai>R;gmr-B=tm{$^f!c(oXOzln zu=GWawiks5>lb?$Q?aVTi|5LLl;n?l#X~7$h12nY^8`qK6Om<7J_V>8+4EBxTd4@h zf?esLpTcSmN?^``FOcJjoS3WmI;VIUpH^3&Noj;TN}K!_qY2BfL#uXUN2EQtFA}#z zGUI|E7gmy!H;;!?)e1sSrZGWPuxEjFn)zv<_DGYIZw^|4h0|D~A+=Rmw&{6359Oxd zDvK`?N43gDK{%hby05Y(B-?qm?ZzD)Ah6r`uS!^4$4a#Sg^PDNTf z2Wpwffl(!2LOV^IYCj1xJd1ZE=Z7S!q^g+aZ9Au!J*Su~qgnJu5|GgnR!WgE5~^`a zHh7Dcpa(EBI$?{3lNiVq+?bz#>wh6+xWlW4!rmDtl878mk<|la|{r z-D0lIo3W^pQmQLpGRkhDf*n^gCQ}NpKKTp7<4bzp4SX*C#3Y zv#-}f1PsL0Nw~?jutBUBDP&&25_lQ+cTxpCx~V&arxQA|tzDHGYNDXvX|`=DIZ{h< zQX4TtG)kngPg$(KsIkD*%R6F3X78Jts#zw;J8c?@c&=nVt@{QWkhOuLpFIqA6*etK zEXa7F!*@A=gRB=k#D_Tt63QgP=LItmEUE+~x}L>EU+6^Dh*(uxbl`MZUOOZ+5wkCr zNu{E`;iV<+l}t|;$012ZtU0LMg`^VGDfXi`+?cI@0;qmGakDFUhz!hHYqS6;VZn?S z^k^KY;$dBDi$AtQG`EV1oILf@Zvg3~(x;e^H^f-h_8i$`c)w_tEuo^Ksm78E%dUL0 zV05SJoNY>+o@j`6si(&)3&j}nzl)o%l5EWT{3_+_Od-k7#fB@{6T5-MeZ$kC>r*Gj z!@i_=N@FWd+f!hk`Wg)>t&mq^EOK2QgF{i&XwieO_aqhn>_u?Hxq%f!~o z$Mn;6UA+iwOjqNZb}h`mnX%aX%anMb^($YFW+lz6}|6xZ{Q!(4E{e;J+i03W1e)CyWWvSEEU>n`hrJ*g_OR5#(rMC^D zup+VdC{B{WZ|C_deY+I%cogLr%d)D@>;_6`=dF_pv(>@CFs+gw3rcXpAbmW02$!hr zUDtap)I|*9KI}coEG6*FS4g(o!!t6ZL_MyO$y-bs0wuAd+cN5w*7P>LsW@KGO)ZIa z+~I9O2drmv8ic^)lZSVnMQJysB@p^44{wnG9Z=;7kN^l^;`YBt0rUKw~z(nQY0)&l3U=RP5rhHDsjpylGB!X)LnoX6QQ8Vld1 zg3PQ^NRQH)+&D=;6PALHH{yEQ%AfN&S+asCxNPxwhr4H(% zPU@y!>P+D3pzi9iF6*;S>$Ps{w=U|gp6k0F>P%n`cuufBedoZ8(=AEjA|~gCi@jHs zrXTtqs)%L$L@H6qxQ-Kd$k$~zLhX>bWUr{{-8v=P34bsY)O~evY_+rE!IO1(ZM6&> zfe4W>tpVkr0rfrr^^WiLPVe(>4*8A%8u0J?{_hAd@CZ;2_F(YlVDJcE@CU!}1P|~J z@9+(;@Z~`87Ju;suki$7;s6QV!~Y)8g>2Abo$MfA+v`|_`PCOLf zT7T(wie|hW#%+&jF;$#UUf*4tDutPpLA+fb&-KBa-Ywbq9lq?TYAP9v6)PWp%ef?{ zD#AxcqC?GsAP0Wus*^cXn8mFsQ|ujvRufSI8ZV^Y&G^Q#M!9JlcQ^#0zpZjvGO3$f z=>0A1RG-gNzf5!<`8W6G$N#VVaW0)5=%l+%A~ip?AKWe1!IixI$qmiCOC)mJ%YDB> z03X{N)<6Cfn6j{8z+uVCg`0Y=RoWvibFue6d3J?JV2KZOd z;6a256)t4h(BVUf5hYHfSkdA|j2Sgf97u5EN01>!jwD&q&5@E-F9Rnnw zC$Of#m^5W}q#1MI&4E65@|4*x!O)oWd;+x@AZJYiGlK@WnbauIno4&v{VDLM*P8uq zjvXp6s?3^SqXO00w5L;@I>jap(81+hybtTj>@cwJ(7!Xael_b!e)4de#69w6 z-%w!Rz#ad>?|8QH-lkcP5+*x!>v5-;Qjf6dJ{wED#SGl+z1G@u(6-a&V{o7cE5t9b z3NzGj!-_iM@Iw$o6mi51p9_#V6#et5t*=&O%(|i&TQ0%N&{~Qpu2#e@G3~&@EG)TL zluj_5zH%%rpoChB$ogKaiNxJdix0mEC*1L`m~K2PC$AEd3pu5}67xRKy2Q+|7EOfC zI3}yy2}(FWG6{`J=Hi46Fgx@A)RQy@_w@5mKm&ynIG=o)lB(ZcoNTK^fn!ss{>)^o z(eTck&O6^0!<0C#8dX%k0M4pvEH1CZOs6EVi^)DNV-nOg`sAZCFnWwEl|M{{GP0>2 zF`W#nH2sTIGerMeGr>&f9O~6q%aZWI)R?`?G}@%K_F9NM#CBV5yY*J1?M(d4zTi&% zuSZ1_l`L4rq=Ssis1l8|Eixl*QbwDELYF~ZNo_YeAWz)2EhH2Bc3L-;J&dZSnzYr+ zB&joXDgwz24zWUs%s5}cZfrNLdSzYq;1N+vaXpenYA|J%_oT07m}8b%&qZrJ`OHQ^ z-uF1f%)=_XA8(#6V|B~_%Gl;`5eyE?V!L8BKNca~^~9N0ooeQj;(HdqCEImVtA&&M zN$JDL5_QP*##U5qkNxu!-HjEF7VGvzt}-wT>BeZzYW4Q}L2dsQd~m`=JGtT0>>Ud1 zEg8P~G{TM=4#=P3`tGK)O}(mID0G2dV-06MM;~jbJPe{AxUOhJE4&CecZ0SUqdFWNcFZzg{r~dlEW%|uU?Hy-I zGDy89YN+2LK67c|fRh&~Nx$cKDzN)QE_`CsK9Kv-$HbRH>^uCr(xmuIEnU(Xb^2nS z^V!cx>M`HMGLx(C^A!+54$RbaoVK$5g=QyPD_#a)A~Fw#P$hvAVF^t*GZ^j9eeAo} z+7_oKD9OFZKIX$0c<|##P*o->;lZ0NW$1216QlkJe_leJjFej_s_yij{B;o(HODaijK_*h>Y4cvxJ~OSltGt`F!rWb zVGL__Naf}34JDc$xG| z?)KF`MdWU9w_IgtP5IIPf4lIcEzQ(~WXiaBCr=anL_+Ej$$8Xw0Al0f!3L= zEMaV>0OF@rJAxe;N6cjYIn{KtGq2db2;w4p4vFU#FtOQkYcXT-te_<%r&}xOOH1Uj zF;iHy3)fU@XFHDdOspSOSgsyn?Ai5=i>R+!@VNTs1Q3VFzsg3{1>OE7fB`RyN3Tp0CPg@!YC(^(KFL%A+_%`itDs$E_H=noKj_U79~B3uP~1zf6ni=9 z2&-m`C45V3xc~4KW=c4rus*$+Fr;EYZ+SonoWN~4pTc7;XWE%Ii>MPRFku5di+QHd z6D}|FzVYEZ24R)adolidi`eq5AG5#v8!K7^D=(rI8F3`?fIkE}8oDqd3{;w3bEO)D zADVlmV52~pfj%YwygsXILMY6WG#js?88d4l5Kr-)+c`LK@wj2hKkj=Z&ms$ADZTO# z9qb?&lxi#Z^P+vTt~0C-{!=N`AUUy1w$&SwsX>!6{J3OUwj6&p@5R5ZK zK`BHltB@PrdGzmLT(T-RPwH_R{KZ}oDlO^p49;m>Kx%i3- z!@+n#Lfqr2)=@UwX%rSakgeLD?g^kJs~=zstv(tVM|8q!8%AS1MmWU2@iD`1JGfU< zpY3BXF4~u)2`zHDkyQl1;A*)xoGZjgKgVFJw~|91N|FS?u$t1ZWT_ug#2vFzlxHcw z6XS}rJD2(Y^DDO*tFU22WE2rMg0_APNP!$C7OF<|Su)7UC?7gSD+I4`o1dE6!#x_B z`}jX{i^253Bv0bT0+}tbp|beXNCK*nvzo122{_;QxNST$o~kM#bI3@HNqjk@f%KCN zdcx?z$(r2BnshVYNg8E>wRGgbJ;W-o1FxdnLXK1ub26v-;yDZB3Sw0guvMS;7Rx;aImAm#(F?(@1v^kBmx=naOMFrCzgS;K^ z*$U9?rvPgVB@!}& zl1b|=;c1K7DHfl2#hY@dj~O8OE4RdfyV2?*^C+$JNzX@Aqr3E*@Z?YtRYGsy?YL&&eQDQ_IA8Y-8fOc5|S zO65_I7$h4HBQRKkB{A#IVC1@+h&J`0l?j?EhVeF{D?y1lok&u!cnYPaDbm8^t{wqX z-TxrZxNIYUgj7q-ICr|LhyzSGb<&zC(HLQqpIpnVtjCI^7?Y#Bg;dbylEb+2s;HRC z^E$_9%boX;&}0Ib@pw_#O27mPmbG}thmtCayeI)&z)P(Sz|cwPVb)}Q)=7gL;Zu{4 zTO4#tF#<&kt@0{(5zgV`#;S3aU+J^YJ3W3G)M!K+B9tc^b5;y1rEbEM1+}wZOw;rE z#0x#qQ|z-9Y0!l!9|T-erhwMhkQH4@s`DgRh1E7lWUYomMRpU_)cDH{mD6>#|`Ku~4GdEks97CfM}Ra1UjBG&S}?p@jsgD7?> zIjI@C9bqoQRK-;JRINc(Swot~v#ZA8+OHr*)^nc%K|%&q5UXq{Y*Z7##1Z_Z4iMg1 zh44@e-nc1#Vk@>IXKToxteiHZPhrx08XSiqF7ZHR2jR# zh>FQun{&D~x$&8P5y}#+VM<|K&yvt3wU?~yk}KW_gRPPdhTTJMWGw!v%EY&Z1dvfH zv!et-K*bR}atwq#6jPmEphBAXsyD8SGRGL(4V^qL0pdvhi{qV4dsQJ$X)l;_l9t=K zzpYF>&Ra)z4-f5|pLN|{9%jL5xiCZ#HEt0d6uZ$s3e^rvx3N>o-6Xr<*=6NzQ{()h zy;wo_OVV{q3+H>0J074lrBPq=z*%aGtyK|Qmc3rn4v8aXo{hA*ZB`HemS=nxp)8JI zWz0`VK1wlbVIG1T>`fc{5X5!~*kK#M#nM_|+$+)Is5~W%=$pOZT2QOVQ^qw-zw}E@ zrZEDOKhMyyd|utk(`R6A&XSHzpH$n0WMAwX)z9i8L=7`cG`LRkIo*AsUc}F`;j_l# zFFgH0+Y4vtOy<4e@*w*hM; zV_W0(vlLeQ#94mfjXWAe8@hZlCGd^DbTtfDHrsYu$rCHGvr^ZNl1HZXwXC+ds~&8_ zUc&B++xlc*AKObMPVAcwJMwDaY-)>k%vXIGN0}0vHwK#nG7Tu)(u_T2z=f&5net-G zVK3raur12i82**R##AqjZQ4emi(0<~-oFd}Mutu^NnA%SGP(i8 zQRksEshB{~?ph)J=}(c{gY#{PrODdXxWe9U@2)^FW7C}mJfRd2BtvI#qpGsBk@d8o z$>Er8o+z^-?#$Fxsd!^~yR)_+(J_Xnk`-RU#bE&BKK@&?dj4)pn{5PNaPT&h9$I3d zD{n}=nXWUYK+L-rb{$O&n>H@ZG&B_|)PYKHO&7V#sz+w~X%nDIddHFb1}lBw~vhxnFkN=I0pFCj7%gO2x^x zWp1otzOr$nW@Ihza+`$2P&2i^R3n7+?c0th7xS-olCnRQWR%mYbowtfV?hc^lS8Qy zL+mTUxHkdT&~U9n)%0@bOKCwb^f-wqbVIK~u3gr66Nq7=7H|9(S557QD~byqK@@H^WaxA4bq=4ITOKYKeDQrDv` zNKPFZU;L)gmG!1c!?}*ao=K@R7nM$I^;ef|XMc8S$Bo}}^PhU_cPcfmWemsSiagDm zzwO$y^TVMXnwT;w*3Y(>p73V+er`u3lOpGeRx*rfk8&=rcYM!~!SXJAVaV>Cn220Mh-aPj%K-YH#V z$?TppN+<$2?D}Lt3z#Xz3fo-wk;eFN9`~@+_)%AKaGc+H+~6pmDPbC~n}NRMYW>zW z1UaCp5=sABWNaFqA4Xn&da2KgfVZfrzxrO*d92_1(cr!4rFpIo`%4Xau`hd*xEZrg zd)1A4wQqY3=6AApd$||SKAC&FKYPHvd%cI$xZiuf2PU+J@xLGZ*Q`r>!asbDNHfG= ze7R(M#`k+Be0<42My;QG$>$u)&wQxwd(F3dTk3q!zxaC(eZ;>S(m#FD=6uvod&ggW z*N@i9e|^0Nd)cr32%AAv+uwU(&VAnJH^c9J5(R$YxA!CvYIaf(8#FOsH@n!-ftYLX0SJBE^apFJjE7aU;i$ z9zTK%DRLyqk|s~0OsR4u%a$%*!i*_%Ce4~QZ-wH_sdFdKo<4s94Jvdf(V|9=B2B7v zDbuD-pF(XJAS%_WR%v z8DQ_-zJC7#4lH;u;lhRwBVM?7apJ~~A485Tc{1h7mIp7+e0ekH&YnMm4lQ~#>CC1{ zqfV`QHS5-Eu3y8BX?k|v*tT!u&aHbl@7{E2)AlWVIPv1fk0VbW5;*M+%AZ4zE`2)n z>Xex?4_kZbcI)21gAXr$Jj?DRv1jhESG{xc?%%_YFMmFLhUjOj*Ia);|Nj2}1E|`4 zi{+P}egPtwpn?l-GT2~R1Ez-`Uk*}Op@kP>m?2FN78rnm>oKOGh$E6%qKPN|10jLT zeTbrqFTxn3jMb^wp=lo;IHQg`^4Oz~o~gKEa}t{Oqmf4^Stdv|f_S5svzghZn{UEtCYN0fsV1F5#+j#{d-4h6 zlU+uc=9Jm>S*W3hBHAC9I7awoqa7w1sic!qnjD&U4r(W(!ByI+r=Nm4+MJtWY9pft z+8C;;tFqcEW&?H@D5j(4=&G%^;$xbtSbx^(sHsBYsH?EU5?kz3ni5&3l#a4#th3KT z8*NU4+L`9E=#eU|w%c;stx1mt`Kzqzd0Vcz=b{VItl-{BYNHdT8?U_co(n8?vc5~D zVEWSAufPAgw&uE}-U+b52Sy_tC%=jf+-|0)B^Q5y~V}m-sbQ3|K8r#_y6yuwdS?O z=5w`5t=63H_wSsw)}^JT#n$G%&hJWV#cN|@IyyRYr9!>doCpX2b7OnX&d!{aoV5_O zgoJyg#e_y9-Vm+cG2V4t%Q{CApm1@_wOM9At3;z_xJBoYo#Fo=B3Wo zl=uJM-tR(d=J($3y|u;W-tUa1@9*B$rNx}Z*47~a0RQjr*520t@BjDj??Pk600008 z{{a6997wRB!Gj1BDqP60p~Hs|BTAe|v7*I`69Zt}$g!ixk03*e9GOrf$&)Aps9edi zrOTHvU!t5zv!>0MICJXU$+M@=pFo4!97?n(!=Ol$DqYI7sne%Wqe`7R)u`30ShH%C zXmYF9uVBN99ov!WPO@mzs$I*rty_^@-^!g!x2|2XaqsFins=|?zi;~j#%T8J;KLvV zBVNq-u;RukAxoZ2xw7TUm@{ieTiEdC&!9t#4*7Vr>C>oFtNvPgwd>b=JLiN=ySDAS zt#2dDt-ECJ&Aflx4KBR6@sqM+9;ZuuxpTJ7pG%)ky}I@5*f}ScnZ3LB+R=-L0vA5P z{=0eV<( zU|RHLnU|f3=5UQ=$)k;H#)g=han@PVc6R2e=WTSZ>5`T#<+QjElzSFxsG&M0`sR*1 za(C#X&h-gtT9PK_=9at(M(Cv2Rmy2toyTuD;5bfm1yw z>i|?htG__8-a4zUpXK_{OMt>@;-Xh{PZE3Vvx_CWA+M?i>+De^T?;L? z+xkf>qlz}FQ@7;$)s?iWq1&W(fxg+6w&gAv@2gVTO6|OI;;Ue_Hl~*=y!t|>Zom-@ z?A21@YDng@REObRWgI08C{0`vJ(O3?>7SjYnebY|h_UQE0 zzG2O0yZC)1wZ&gE9Co~83lsncsXzgR1WGqF01q8-VA22`SinO9|ELP|Pg?hFHbiB& zhh)itKV7)rh!^Y^4@*P=002!$;1CCmL!p2`3itp36m93-?90y{p7`in)eKi}L6csV z>NiQmgAqJ*j@1Av08j=LsaU|lB+5H16VLJIEJ5Dv@_dQd>0^F9Co74*q~rNSX#bO@?;3D9Z=;UTCFNW_5*016fm zUJxZ0h5H47i7d3(L#($1xt&S^O(?@XT;dQ9E&%`*um`cImY)IQN&$|GB1jToJqvWv zjZr8H8Udh2ui?XS`zs>!^w=gYc9Dz<`y2Cw=q;56fCm8hM-)2$kTJ;sF#i_k?Y1U~LdMLn)C6g}B9qZWfS0 z0#sr?5JKpXxm&{8#2A2GE`T2{;z25w;DcsvqnW8VNGbr}1LPp`2@%{)2@`1|8deiI znu{hMO(ZKqLeGpx!3m}uC8OwN$DA1f*#O-54**=CFY^0G6Yd8A$?el?0m$IvG|+@4 z(C;7c(A?t^`8W%-jc(g$!a@tWf)Gt}p$l~=4EK&N)Ca*2)8HWmdm+xNk9i6CxMTikRbUCYj*2sF$nDPh!Q=hK&Xh1mK>m^ zx|7ceG-rXzhC~8Sh#X5?$N?O6AgCr$K<&b)l0lR`l$fbV?I@3m4}9oDB?oTF2U2i> zy9J=1E!|*TrrFb!1Q!5K6)RCqG|eUuD0sy6n)5^JYCCSW!M-EJIy;y#Eb-Zm=Lc*uWAjuJ?X`hZNH2_OENXmCJ^MU9TSCYf< zRc;+6o{P-HyFoFmixfz5wKz3C@7VyRd4czEK!qKTq&^{u<0er3frVAU2Xa^OCZuAM zZ;rwOSNV@6DDCMzMIy(=INs}75qLSxX&DAw5F_9;?ig+FyqmCVLg^!aAFu$sCE8w% z=(mP$rfh#d@PrhITYvXSKxy5RiYRzpvkGu>86-TA-Twig)OlM$J%cR}&x0CQJWv6< zEieL!Mj?*@IG=4Htw>x&Iw31B$#kjtZiR|FvS+I&=<{ZT)sJ8aDL{2eR4@Y8TcDaU zG+Sf=K>Ad7@wuzvU~*s0KCY>_=nG4_^&6vr44#l2;g;R7!@B-rtMCH~`0jdb2T8gC zAPT9dYyb}6*UQn32b#4|8}K65vkhuq5!E7QOSo63|uv@B{|MXy(;*7r;!72L*UWC^@%3_Tg3q z5lvArW@bYUn^q8tReM8aJGim}&ewcQ=zPOBJpTb8QQL4i`^R25S5GBpHctltr#C3m z)kMRiG`Tl^9n?Bx7T2lCH$ZO60lgQH4jiIA9+;2MG!%0iaXHj?g_CABy|^0#pgG$R9M9%B;zSUN2mlx-R`xLfhzyYJRiywz89Pl_o<$2W?L_FX)N=1$J1cZ1+O%~&MEc8VGkXt~xj2Z?xJ86y9 za{=x(JoOZHTG&@uNlRMUg{ni3#lns!W^Ze_D(~nn{Bv~aC?N!3k=^(YA6W+bvwj31 z5OkwYbBUBsAbJIHZA6KbP4JYs@-~(fXT)Se)O3KqNL{lLL@Pm4#S}o9s8{EPK`HQc zD}iSJfN;DdlAC!wq^LesxH+4m5=n7(|}51SZj8t#OUi)EB>4Puw*; z&1f-mq(pBvd#ag~yRn4>c@$sxlo~;lLG+UdadWKWDPGAM#g#fbv0U)+IkNvIRnGOC z408rdm0GkUc(%1j!sdtmNS+H|V>@XNZq!qC8CD0801^;wDW?(-_yLwDQdg!C7(`8| z^bhoK4T&a{6;Y8-uxA>vJsOD*2`~U0uvj2TpB)LI4=O($fDc-d3Qus6c@Y5i00ji) z1KY3!>y`>UkZ>Boi;|X{SojYK;7L|#5YhAmuoy%{m4&v4oB|PYpO$EfW}^_7l(|$l zAN6oGIXq`D0=6TdK`2o+lrilD4yMNOS>D zphO^X4`uKK9LR}IW>|t(1{Ud=Dq#<&wgjJPsn>Ig9STw+x&&?oWkIw8o`g|_C2rsL zn*gw(3Xpuh7F%bPF;Djo!-<>qbQII%jK7I{R!Wz(qipwA1|NVoW}}R>*#R051#vnk z`Vethc$~euN8=}^TJbz!$(UQ}ecz`lL4>8<`Z29Rt?$AC8EOGWU`e1DiZRE2Ha3@T z*{*pr5BC?2AlC#bP(p{)g7k`AF{DLGvr-x&KNpaEQJ_Tf(^erE1!b@VffW+-kU@=_ zH~8=YrOIG|_n`I>L)ar98mmbOumYg!QDv|J*T9aQ7*m3VXe$3ut`CS~0MJ`7XFCB9 zX%&_bN1CMCWP|{edO~$b>(+cf%WkUogt&45Ff|WsrFD0OD~$BAVl@b!)TABMlVccU z^|Y_VC9a5)GHRG&W$A`h<*msjw&$^Q|4;_8YBo%Tknei7_fwZ>dx+Q5h-Ncgk&~lL z%4iC(0#|lEy6Tqr>K=i1O1bz;I(n|8W}+Y=KR*T~_A$5}5Igt~xirO~u;U*2lUVZM zlJi4qgHUIEfk91Rpem674|F}m^d8e!M~5VQ9@du0nXE*MD=}BI1j2N$RT8TUh1OFB zgV06fpgCwNtD^@NTDu>|D_l*&wL<5P;gP&x0=>ocDIfpj1A+mF)bv{InLX`kZK=8es!9EnJ3=GQlaps0c9tG#gw4u%?g%0c)B)XY)k`wE-5O0=+Y3 zxPoMmqXKG!UgV$;m1kIL1FCTpw-G5qKxDt5dByolBKtcK5>QwlP+t$`fgGoczvMVC zTAKp#O=jas6;_)7;A5B;MFW77MxlCOiB94{)<}MsNVX z8$=;@NhCUz0YDDQggco846ZfhiEg7)j?G zv?}m$CDa7D@kB`^!8Gec4zx|~ij*3_XiyNe=d3$#3^{{nXG~qhCPkNy!^dMGx zHnrE&95`mZ?)}(%o!-W~-a|v-yPdT#Vc+%JAf@ZY_yDNG!vQL=0kBtmhix_kFajGO zDfY1f#ix4adI0)h%@z=?!?OYiS1>CW;Q74RY+`iUrz{qZI71#)S%cw}%@UG_6v9p1 zeN*IF+alKjI^n@wwa*g#9Z-%QE1}^kfyS6PCqe$;hmz$zH04w-=Ej2N9YR`L&gM)m zB4MuPNOP9<>lS7%=T*{kM9vj2trEoD<)C9OK!-JCF1B}$+jzb+KvL*}!xbvQ4}GrT zz#Zrr64r)&z#=rJJjg!jWV0A>bQaB?%fhz zF6zG>k3xa!j7}r4ju}6lA-xVFnGWj7wA2ek0MmKu5&><3Y*f2G8qY4E?dz>yC8ggQOR5t_mHGs0r?0%n;)a z-T*{7RSh^#mF*Q%?OhGX(zVO;n@DoDt@$@5^hl?k{5BET#gF?_OPc%=IMfcF>NnSJES4Q)F5Z82W4`EY~ z^V~8#I{_;4XQ=LFPV|uN9VDL+Of`2Wad?|8R&RFGL^E9%Kt8wLn13$uN&#CJ4>)!! zs(=$J!KZOZFIESiB1S*q*-6$T^7d;#EO0;fKrs^(zb2AX2Fn@}1Q2sn|GyjhRdyKR zNl`F^OTd9=&;f@pKkSB!I3fWX-~&rA0;ym{ZP<#Ee^}q9TuLaSxOhzw>(HMnN+!Xb zKHz#6Ao<#@O3$SEXx~AssPEHrVI-`Mx*aK^1M*#&1W7;$yzl$J&-=YE{KQ}U#((_C zpZv<7`~@KV&CmPKAN@(t{M0}FNk9Sa&;S&`0GWUW+Ry#lp9$eV{uCho6j1)>kN)Rx z{_5ZU?En7h5C7|*{`6n}^S}ORu>Iwq{|vDI06~EPg-mD!5;Qol;K79o4K^S+5Fx|> z5iDNBm{H?KjP)2?{1_qRNQ)#to)no9B?FcsS-ylBQ|3&T6#m`BqM1|YPM$q|{sbCS z=uo0XjULS@;a?dZNQ)|1@-He-d{Hep)d@hWRww}mK>^?+RTHfj4g?6m!$K1cTbnH5 zffnrm4r`;hT_Qp5UcL$}lu4mA$`Y_m^ai*&VC-GNWl|;hHvsI`e_G4j44_~`6UG3d z_)v-yvCsgi{zbvsGi_xKsa?N@9b5Kn+O=)p#+_UDZr;6t%KZ)8w`<|UjUPwhaO$}bn4ZuU&o#uJGScnyg&Dc%^&!i&$%1m9$vk8Y_Hj`hyPxFeEQ?l+c)t2 zKKlIl_4nt`yY`&RPrL&A^DaRJ$4d~v?;gxALhvGFkU|T`lTbSiH+=4&B#2WkL=lBj zs)-eJBfzSlppq)KCZOO!0uBP82dpTXD8m9MQkfAS6;g0a0tr&_L4$41LQ4S^ZiGTe zBEyO+fC2cLE0q+Y7(qxVp13QD!~TJw11>F)V1*8TY{H0tGLyi@RAM|JF({&lqRD$Y zSONg1L>sEK%4iE9P5@dPu~0(~MfA4O5@oc{Mj1sbI4X;SG}1>e#Wd6Z*a8T_fC(mm z2cJ(t9TlEYPenD=R4JH%RaaYu^;KC>Wfj&~Tg`+ZU32AiR|`4qgeRlsICzVR8Q<1hz#rrPwLqlqw@mp7!!- z;+!UCQ3cUFzA43juF`2zNuxw-kPR%m2c7{G7(ky6G$>;Q2+$&jhcfD%tAi+Vo>Iyc zd`O|Iy^0P10v?)3VF_}Ql%Na%o;WNs&OVqLEquT#V*t!L0H6u~eE8ylP6>)WgM&P$ zqO{`={*<(1v{|b-Z@u^CyKk&W_8U-3RhzVMz(b5XaK#r_j#Gd5VO+NW%JB!X$aCWq zo5LNw+@}dV>^yX%2H+tTC_Y#l<9kw8V93)u;6Uu0&bWt1od)RucATn`Dgb;%Vn(^n7@rRkuono-FA8D{0Q4}UfCF?uNHbbf1ET@} z2{^DRnDHGC94G-EOetjnAi$kCNIJZU&V(mKVG31Po5YR(%y2Jk;c#NOLK@caW6yyA zZZ^la=JaicEPT!niz39n!2@(QM4BmFs2C>(BDmXhqA=r=?PoO(OG&TJpk~<9$@jo+4kZy4K!gJJ^&;j$x?>l zai9!1^T8hz=>w|~vT#b&(*XB!fU6xwdY=g-AgzWq#3b-){%e^Ju9itnUhQm>>116l zVM;#ONm!zYr_Nje3ii;66qmr5Ae9h_zXWD58%0DD&Jc!6n~Ri`NhRdWP?*o` zoZ~D9&7t&gZ^V>M=CJw9&wU6`)HIG_=(L0ne830)PiVqYE(oR$K&=crfVQ9q8b+Kv?Fp1kXjs;k8K(kGj0(N~iO-$@1wBgf0(FvIPn2RkJesejS5g5F zRv=s2^1y#QxDs4uJ6jH}z<-joo=PvkQqLSl0itX|ZD-2^=2nk=p3$Q{1LFhcR=2uH z8bBG)0!zb8Ns2kbjA;O{6Aow~1s}l2niOD!H-;uM=`6!RIWPj$49c(f#qU<}s;L%= zReQ-Kt69y;F%~*AzXul40z%-AGaQ(%es$b_k+@d`Hwpr&(35+;^27@c-~%{AEN12* z$`Q!!p7{*z&nU~HoGprXKB#9xQI-HBXaL3pG!2Y@5?Sr8X?ytK$6n^a8KoI+pFlh2 z0)oslm1-9)*=s5M$f43Eh}EW+$}RvqQ~<*P6+xC0$Hvp zvH)18l&)gm3fI}rclJ$MUFhFr7A|~66EHJ3rDr_r`Cx`(XYy^EuOO9HzQi z=t3z;DmfHQWG{eNOD`d{iMdK1)i_z5h!(R}nV6q0Kqwj8Q)I17MXZgGil}64$4k-J z0A6fF;Xi0R~Kk_44#Sv{2OlGetHW5AsJqvZWwvJx5y|r7~czbv%kV z*=ai0BuZF*Oi+aKch&*fZ~h#~O1R7)xD-%i<*DIngx8JMOckIMInZ(oqMxt&@stjO4ryvXUhb zg%p5hYBOt7)+F!*8%rSBpoaqd7GOV`qs5RQ9>BC=+U(4T@(CZ9Q33|A0w^p&E(yQ~ z00avG(j|#3O-Mx+*`%W=V%6nKV{VeJ;(Oo(J1kQ>*$9%#{54^98!2lFo$Ee+@&~i) z{;hX_@BLq|kM-+%HeC4&jdn!|-0k`Q)uy3!LU;MwhIhtGihpPVjX40U!n-x8JbNOw zoLGUM!8noYxJCK6n+UlBfF%pStHk@53&^OkVz08f08MHFAI|^{1i}X+fQ#Mv4Df5W{v*RO^c#|Czp_(5_7l4{)VHzYiw`QC zjWI*6TRYN#yZdXjfHMHdIYZu=ydbpLJdY{^n-auxL`O)G!(&Q6_iI1&V?U$lw{-j%JEOnUP%sA5!~4p= z`{F+tf;Y)fC(%eKC?bb5gCfk}fneN#(O?lziy|YSlLt&Fl#7%Aus(?Vhc&_}gp3jw z0ty}rld|~-df*Ee)E<1`01m(h1t14~pc36vfR@AudmsmtL^_+(9&)%sKpO<-QnuI| z0CI@E1jx1*JcFVNK?~>-C>R6?AiR2_ z$Fej_$st9rQ^T-ZChu}XIAj{)FtD@4vwIw(w)-a0VKljVt2$j zfDO0+fEt>0@~}5*C@6@h6hOYzN{gV{fZprD_u;s*!4dpa0kSAe!*e=v_yErHfz&gd z)ibLl8=xtH0Df421c1o^z`&fOlA2pG7sSn8G@P0n(HH!mCnM2!Jkfr$Mtu9t=uFNR zTF)A_(KUfic=WS-gTuDeH_4gH8}&E4Oc}f!&$lxGNb9PJu?fCHocZH`B?^qrQyxZ; z%r(IWA{nW&QN?bXw$o}NFT@%gO_3K#gg)jIoVoL(Q9+AQ-^@`-71HXw)V$JB zw{*wcl%XKa)Vj0F@Z3iT3jxi+ug_6WL+Q?6@(Lc9wGNnv0T_e|BFn^Rpbv;O83CLD zIDm2roOC<_~qSR2;FL+fPN4?G$T0`sLREspzf+ajpCHni1reN3Rhw<5H~rAt ztWbqj+H8f%>Ve5Sg-HvISPNCA41HRA{=Tj=LQu>6A95b<3T}*f!i5 zJyj+@rMbqnO{$I9JoQ?P#n|UePJZZ3v0|ITAykm{x{mF+*ZA4BMP2f9TTW#|P-Ruu zOPOzdM2VAI4$#l$G2I}#6udPwF00n}d$NdiR>ig27lcWejMhDM z-iFoC$IaH+tlsM7+TSM>-APg0W<92@Em@t!Rs!M+5UpLi`G;GLT_hsj_MK6;ox?MP zUHJc%-4GDZkx{GLl{Il7HAxC{jfUITdF8+AXoWZ-nA*4R_p+059TQCtd+ zp8>ksDaq4^4Kl}_5{zZoosrWBp424+%JBu=)k9%&J;|e;Ux^Vp`<3B$TV1S;ruv;> zbR<=uO<+vn-zCkD1P%ZrSSj7OnXr+^jp5lG=3#dnu%fZxCHu2JL#fuQ;)bcO_ zoJ|qV-si>MrgheQ%UbgtTR0_MbCrv&wV(0@!ZYsH^bLwgNiQb0W3_YJ6r$NX_RhD; z-#Ol#9zKpA<|=dG^K|Ue296Rsg)G3+RD1P8%nqVK@;W4vCG!$W!^8IP0NMW%Z=Jl zR@k^)6AG2hI7LxDb1pW1A>7PR4^dFFif=LN$w)1BED9+6pT zEwZzWp5y{=Z`JLR+>fC9b01!5=d>*|>ndkXig)FzNR^)%6q-36BpABNQ zWk+d#*^gFU>+Lg@X6I=AW|+L`%){0_^=k~h-sXkY0UGB!WnFspP+}upkuB5_{nj`g zAVUS$g>KhKVPB!vY=iCR&8AMr!DvMZ3Yek{x+#>W)*%e*sGCTEwBQJw<7i~}0*CH|Xnynv)Cl4gt&1vu_SS!+ek46R7(n>fYo zqpzz;HPpaB;&|(Mz2O>u?zd$>PQBCbeeA!UUJ+*JYYs9E?pkZT-VYXI+oWK@j$Vt^ zap+!UL3LrBjN^8eB(QDabd!F|J$zt#IT3Ad9Ak=qN}wpFMym(15fK0IP9x)tFO)CI#m=1!xQ81}v@MO0offPb&?r0N!N= zfIOe0sJMV`3xGTsuA$y@!WC6N7j$V>Kc><4P9^l^eOylVQQMqnnCxJ__S$;0&FazM z>K$n@mfDBC>7<=!m?p3f4cYUp&0}(9&E0Pf$ZS==Y<4ARfcK$PC-{`1N6_|XvXSH5 zpmogYfGY_xtL9sa2|hUE%6kBIhz0;9MP}Q^NUhjPpG_YD77bA<17ns)K4%f*5_pc8t+8iR<_h*C0 zeZ#f#0UrQ1BP5l_QGi5yw`#IwHIxPm0pd_84X}XRQK=^|;9^%gk5hBBu$r#;i6GRI zI8PZJoVRvmd0h6`Yd_p=PjT(MWU_1gZLe;Lb=*v6XUAn;bN}Xf+ca!PhwGLPhphAZd<+<`GQkyzi7H#VEDb%P^r&6UEvMScBTDNkA z%Csxkut5WaFmOif%m5Oir0DR~>eh`0(9Yy{FN%f^bZI8|7lLSj1Qt?Ui1H6Wlrm5T zvi z1A%`53{nOFBM3AWfPVcKfC3i)umpt#p7qyI0bBz>89aQTf(0L}P-0CIK&HT30a(DI zKB=&{S^$#)U;$$>1#rPX+pK1k0R}7y;ZFjtM4NiN0q5Fpyy<3}aKTa8+m&9{#vFR$ zC4|yK=V@2lNeo4loI%eKgpf(KSp*$J1f^*ZLc~QxkU(pql%-Oyft1`x@p*KRMj%0_ zAb;!qgAL06u=WhzM2zis+s?K8f;gBnbejZ z5o)D+CDA6PmFbH6t+yyK^qfX*78LKJCuLOLbqOpN-kbB5OAtY8;(OA-1C96QbodgQ zUZKN1T3<8E>|0$5(%V` zcX=O95pfQ^%MyK#I%+RTj9x32q?Crexq<7pn^CZ4lZzDRrF;K+wdtv6ta{32ofYX4 znSIcK5gphLwO2ps+#du`9JYi7!3Qsaf4dKLM$?c%mO>L3SOE3X|5(6T0MH&*2IK=Y zAwAVTWLCTFw&Pe6v6@|;R%+*W%%$7xkM1p%Te6MYx!(rodEB0ZhwpqS$v0hi-W3sn zBy2i|@(y|CC8v~RCxHhe&vf=j9)29oJa%hHaZXZ`_qeBjuc4E$I#)5)kq~nNdf(V4 zR>Bq1B~&bo(hFnAxs%a|H32Y;@^aXL8cyX^vZ_$>atHto_>NXH6AFly@DJy~XbDTG zmJi@TwH-B~FEGIe4wKfA@-2aCI20lPjMWhzMS+K7X_Ei?K4v!Ic#drNOPl-3*2X!8 ziA%vrQ%Kx&raJ{@aPOKA;x1wk`4l8OdMl)#^ftHwYGjakN}NI{_nR~YhdoD06vq~{ z9(-s{hMt63rs8)t6@IdmMM0e@Q>jYRq3U(~`G5tQ7)uL`U^8F|jq&aifO&wgcE0Q+ zYxr@*pqybx+Zf+J5}<&`n2#gK0st~QQ-*(}0GYdl%*{9g1sKtCmMXAiN1E2frerIP zB7+~e&_*`;vGIQGWMc}e)TMIeaa|01PI$!Ap7gj$I`N7QqV|OyziA|3*HI6kP;#)^ zDD0o%$;d@289IhN>|rCBkHpLwmdsgfl^(^G=l+)QHWaQ=Z80;YMp=l;kHU(jE`1OU zSILrGdX%O=89)@GpqI`lAXzLKK((^@k_1pj1Ww8mqLThO z>Y^fn(k6lV(w9m}ZB|vMDpx{S7Y??uj{m(>D_KgGl&bEBLm>ytmY{$Q4d4J&oLWtG zRGODmt(hD_US-Y~fL=X;9xzKEO6f>d(T$R}wSD7k;nu$0(o=AaLe5AG`bfS_P)!3B z$O8E#kqClhPHb{$!Gv4KDuFI^Nbw+J8N|`Y4%Q~4`);?gTf$71a;4loFM1mk(~q94 zS*x_EPxJu=#4G^{P=ICAnvjZ@Z~!l$ZA^ZP)DdJ|K)^c*K+~A@TCk;3oxFu(ZM!OA zIA(Ra)BO@|5?WA{V00vtWE@%PIuYjJZIFZP7b69d(8VzaA_?Imy3}LI{$bK#CJStO z*^9QrZcLQoO`XOHm?9;yTHe#fnP9T!VzUE3YX43KytWWPGK9R2W#K zDgc0jYl1~EH_Hc65aWRr&@*Wjxg)Uk=m{;1PC3F8t8ScaSKkSqb&Uj{`T|n`e|u2M zO$SYk?9Oo#`apzDL|+S~4c_irUA>MKK8`w_QuLGL*cNt;Nf~dD6$|PgZ@Sc`&dPc@ zw7FEL<%O0_5j=Pxf%`%MXasQbu%N($mngsj8~86rVw6S>Y?*50jH+bEM$&XvIM3j< zUkaP;oRBC4qizao-|C7dLywptfV)yaDK~BFoHc^V(c_;GNy%;B6>^n6%BLj7$y2}Z zo!wcsvV|PBP}67M@TNBm0sk)W(=3_2O%8bK;EU2XFIJ9L)vBk}mG7&<^Q$O10|>?1 zM{XC|t?%(RL+2G9gC=awf(qDkE|MOAxr;d|zc)rEO~}4UOjP#K^uU1}WKojT-~5)+ zwgi;(p%dLJ5;7x7CB(H9J*CZBQFwWg-Pk!-oX!)bI{R!(Pug&2atcfw!Q|8D$1xIK ze=`$t7*)75v7I4>dn6vw4U)jEd)@7}WQ|KDZ^0JbW24k6zIQvapA#Oiq3%246F-)M z!{YP%&wkIR+DIiJ= zG980v#Gt};)VO4B-WzytSMeCTEmx;nb$ySD@D9his6l0XrxTs|zW--3$_%cp4qXiD zpZuEhLG4ZmOT4$sezu&;;2i|{7nKuP~Ldk?2Vp04S>w8 z&w6kG&OIPxMOcruO}}-@0(xMa{aaKtAswBR5muq^v07ps9a>o6R#=}@NgW4fone(r zZcSmHZ5^24(V3{>xwS{57!qzA+P#!Xp?#ZjFiQNTiN@(sd;c8Ai5bpvMbfsdSZ-98 z-$@Dq@SPQEQpKI&h3OUv-q`Y8o)d;k@{Pp;^4ujZA}2myWEq?R;M#lrLk3C~AuS(V$+X-rhKhqy1TT08r?GM{}v3x4g%nncj8;4xGqfdYs+a@lCr? zq0MDZ5q6>og_0^RAxs6{0Ct)iA=ZI4p^uRcGXe%QhNA{*q4X`?M`54_f?=tBph@Y* zDn?u*l3<;X-I-ho-kb#9FiIkc+ap~`!WZ3k2R>Lyao-#{2;ouUR%r}wY$Hg{848h0MbOCS9s%5`mzjDNdij zwGsG17{#68JMIs*IoFW$Y7wX z8$kri?>S>frDO|DiHHl9D2qs?_8 zDW)TXx#dUVmKbW?YSM}Noy1t}&6TL&*s)-JyqF<@1amN`KFtXpiIv%*6LwN4g9fH# z*55-c#6n=Dsh|!;?k9kd=J9zYDW!}O23|MDD1BJTQn2O+VjgPlri%vY`nZl17O9bH z43RQmTpXzs!UHKXDXI04kpgGmWnJ)Lq}F{Nb9E4fM!!k_ze&;Z_RiDTUW?QeAH8sQ=yPeLf+KMk)Ud;EgIN zpXN@Z*4tk&fIBn*17t#_N-CvVs-;fqrD|%YeyXR2DyWXCsFo_Jo~o&ODg}J%s)p)Z zTxzSLK?6|0s?usE$SSSkDg{hxuFmSN`l_z-Dg^}VtTMo_)+(D6P^mt<*kk)n09rC~ebj zt=4w!KV&V~a&6dlZIf)R*=p_CmMz+@t=g*X)OIa4ysg|WZPe23+{W$QMlIP!Z8Stf z)j}=U?rq>^?bQ~p-X5;g)-BosF4i{g*tTuhmaXJYF6E}J*j_HyGOp%kuHM?M-Ab)~ zPF;`QB+$lT|2U|;6q?sbObt4UbIpe?a)(CI^$iWMO33+}eZ>%!hK zs>d*rrvdKWo*M0rUZa8{pKvm2SNtgmLh0e9g`y7a?!lDuZm)wxYXPu;v=+ekiZ2s{ zFAIFHw4QJJitn_J@AxGtGC8$xT0{n zs<64Ls|%lNw#Kjwo9hhMunkXZwU%!^RO=3-ZvlkvV_l>xj^Vn@2ME<;Zy3@^q}VU| z3x_6$5ktpwZX0HH=*z*2i78UG^~pk{n416Wo|=9FF?#vwjmAb#o`?TLM4`o= zNgNQwgq~5h2N$>A3L;8#nBK-sZCuxDFR(*Ll6odAK`f>uOIVYO@1UC z*C?WTg<6*6^OmKR5?>x$-S7onEdLL4(g_oLIRFL(a%&Lt3LR=j&e34bToPAWwY7&_ zk(E9k3Z2-Z|Fq$rl&mHbF$#+*;~}xwMc7@;)kZ5<3eF^=B66<)f@LbU z@r<5|E%)-@-SaNXCXj62X?i9zAN0{;N(d-3>zGCW400j25JIyOg|Q+$MKN!P6(*G& zArZ%UaEC(S3!Q{seW0?R6i1!*4ad5nKPi-WG*m)0R08qS55`E^& zdgN9VYCcnMPg9|3;_O5xHO-Pmmo*p!Gqh6AWX^_5SAmc{R@54DvRqxz>FJX`%}3p^ z-3fB>7?a7oRHn%m=Mx(yME{W7bJb7o`7%u9sT%ij=EWw?Vwh^mbyExgj?D2C&ZkG} zHTz6ULI0>4MYUp+3|Y96Od)eb8!aMt7hhVU>t$3%lT{32+su`7wlyV~j;x#d3l#f{ z02vfO4P~1qmz|h&>fsBR#x)Rri$n^Gp4#>3?sY{Uwm}0$Y-|RU)^SIYj@W!99xpa> zizZnZGn@@FR3|sRU8#>!+oAc^M?7f7x|m8xDC@c*V&cZP@l7DvA%Y1L&hyOjbk9cHf$FiLMulJd>3u&~p$d1_6?gOYHgp#_G&Z(mLAHS>UQtKr zPkIF2NQ`uS=|HLOG@={S?T$$|vk9GIM-LthngZsbv0QBb)5Q)~>T#I4)%0x7HeWO1 z&FQpcH+X4!Ab3d}3PA;-@^(P?IP(^u(vX)*EjXTvBO)U>dpS3eGkEZg&Jxp$tJ&w2?@mn1d#zjZmwI5xr8|9?Q+yo?x$A&APXFaDUS$brHyQe@O z_koGjU%N(>6SXtk!K#Zqj1AZOp)58pT_I@gmsrQKwvo8>HVqUlX| z)1sJ`H@&dzn4op0zxPA9IIr6f19Cu%x~R=QERP3v_nBl+?6*b$cjbs(Mg6lI`#6sR zy0?%PVjnxfcVeN-6a+_h!AExzUbpFn6)t8(p+RyzRl0Q#3Q6c-Kq+P|miV8YbYV(5 zN)HrDKNN>E(weBVslT{I|H}8O=VxxS<1-VnP)CV1D#~`}dRjJNXToxol1=8up#J) z6j7(NL(dKoJZHmPeP4 z@%MD=Mn5lnC^s=evSHl{Zd-S3Sg=&jMvapYs@jnA~R4R<0P_(gUdL$PQs;dV{Ibaa??)34O26%G=wx7Kmmm2aR@B2z(cLA zgjTHWH4JsTu(AnhtkK3Bam-OinUdSFrk{fHQOF^Q>?|Rw66k9$*~nuJv9=Csuev1I{^>U>n*=fM6WEX^4e?3!O%Rgu{R-WsJI3l#3!>Nb%ZF!&@!|z zJJR|Dlp^Y2lxW4+Qgn+pfoMZ7LkvIkb2QsV8x;S~OEHD1x=cCkbUK{)@ie98o)gv7 z)lTKqAwy(4H|JE*G#-hMOW3ei`;kNt!P1QnWNXuCBr=uTiV&GD|S7 z`Wm@0_L4-bS!yqwRx)d6{O7?t2j1+@c=??4PtXbt^jmn$-E&byQ5*nRK&QWGQHhQo z7qy8+>mVQ*jNSVf&L!T80y0wSeLKGp| zS{$gYq!H?GRy`$;dGXUxPhIt)s*V~+p;nJwc9Ier+A8<9l-s){Wh=R<_SDvjK8rQX zxJ}&=D=Y5bJ6=-CDB}tYuCxkLO!+ri4p%r!|AA1m*<0GVLvFc++;Kr84?qEU5JFcX zqPNssbJ(UjxZxMk&)-k*`S0KV|3Q^CQeo|X1r*?J{_&7!-R?ZKBinvl#k&*@W@1>8 z%diB3LAw|TWMHBdRQR#CTrFmKf@#WsWF;UqS;c1nXIQ`)2B41tBp@>L>J9g1mMsH% zW_(R+9Ba}?KBcWMHDKwSZbJB$uYqe%k=x-Dq5l}gfVB>4uahEgwsytsVJLV#cMP~M94K>6B|}-?ueYs64Eq>$wvkE z;CZ+&Ei8ksl9FL6maK{?Pe~wTmm1v^KmRWUMoe)OlU?Yzqy$)LmapVs>SuRPw3Lz^x8Yb^|L|@*_b1m&BO)GWJeD0K|J&j2Kr*=x8 z4i!&&`VXV_rXgk_H9TK*lvbWcNkt_UZc8F2L?bpb7n%n@0$SNW6-7d2wg)nL8JrDm z#yt?^fvum!ss3ElEkB75t^@JrxhRr3mb#Q(?aCtrH#a|X_BBRCHSA##OFMaLOQ#v} z7r+v$lEy|BB$EvZi*R{KMM}v(U8IdzyyP&6Vo6F}4HFngOPJ{iODa+A3!BbXGG%5j zO@lLBTIrOBN??EoS}a{7wJAiM!T&3g2;l%1Q}eK}IRumRdq~m-J2DWVm9o{fZr3dP z)5Tg1C3aP6tS4yJt)ONjt2`;c9bQo7@Gn%F1?L=Lj>Qvz}I*xISKP+^N ztjb!LI8iq#2>gc;6o45i()B~vw5AYy_>jsa#C@pY4(?(k*qU0>l5Nq?XCY|BgoKyG zCFW@tqd%nCRNFo016Sj(E$v9>j>aUJU- z?3&lT_VurwrA76nosYyM!D8n1xQl|Id24UNLlN!vB+4rgeuIfPueBcEy z_%Q}?3|1J89<1QO$MhdB-o_0S?$L<0D`BU|0U~ z!VY%g2Qr8S*hBP#(1++rzW~vnegUjMy$oO9ZrQh<_U)$q0;2menI(?!A-}b+9iVv3 z|BLd(mpleK72%>M62co&vqU(64-q7U1NDaY&RGrcFhd~V1rIaN6K;>6E1difKZe@R zKXHq<-QsX}{|>lrcKO>2|GeG~fB}@lUC73+{_`(N|6c<0fTHpU@L!5x@q|zDKJNf` zE%*`-D0VIJBG3V}p#ov<0(s5W3QrJ#;qL~|1Nn~Q?%@IMp#VS<7{|bJ=pRZBT0g!3b-u9*>0l<*eW4DVAc-LI&K7u(F9Gf4j%vw*{~U%v8cqYivESvqO6Ookrc7<8nrPS zxp5o4u^Ykh8^tjkyOC_#sK%ZJ9r#VKssksaMJ9^r8w-?1L$(H`mX9`!LF`4J!e zaUcJ&9|3Y81#%uS2NJ_6uc(a_(W1s6CYRE&JM^Y#sAMj_!Y{JoS?UOoyyBSj!K?%< z+fwnjuw|7<>$l!+{oGCkD^C4fQvJ>^CRcDK&u`&s^5<-F{B&~s63*uY5#hjZ;9ASx zsx03Ah$!VP|0%6+aF~+R2x8uv@+sfVEsWB(nsNaSz||6B5b|K)xUckV@Ah!76g5$% zlFK2-Q5&CeE!omq!tPW65J>Vc15l9e=(6rm@GkW-FZmKL{cGvo&FJHf6ImWAip?6E|%$H+eJeE{@yy%)Q_PBlV*$_$)0fEjGv`ZqAE@ z>cUo(1$YQ5HXcnsQmAcMs4(IJ0-B{MVJMeI^0zR9%)(P3wr~K?^8gv|0JX3^2~gM^ z&-09LRz8RCU$ntKtMCeqZy@Z@?lRy%MZhs7GeCEvV`%^MA&Mpo#&PUJ@t(M)2+y)Iaa8L`n2SOElv|pe9KtmsCs16idytOwsgA)ih1X6iwaqOxbizvGh&b6i)5*^9Im4 zkOs)C(HbpHEo?=`E~%@)ilQclKK3X%F$T0srkI4s)AsA?_F{(ELLZv!tje>9Zll=n zAyfB`1odw3@=jEXu~bQw7(MRqhH+I@74SapRY6b?1n(DlQ6O}bOu=;X&NNpw??f^^)*HPwO{FSUfER{ zzjf~HFkeL=UKjHo6qaEjv|$_8V87Kr*>y4LkS^)aU`NnFFIGfHG-Ev$T{D(oMfMI! z_GCpgKp#M5S=J6)c4b{QW?^<_U$$mt)@Et;W^Z;u+x1}Qj%3HxXY&s4aa)nrSwY4;Tmebz*kmTJHC?p9-|Hcf9lO}f70!@weil1M!82CC3jHV7m) zMCOE`lR%aVTH1y&l+2GrO@_|Ewsda>4Gdd!XljW6He5|MTE-{WZ zQ7{h&S3w1rAhfc5khSSLblcwL9c{{&w&FuAvobleGAWZW;ZE-GF!8c=bqfp(QH*tA zcUy`B!$RwCp6*H(h_otgV;s$(kP|WZh;2bjQT{@XUh6>gsF@((BV9;&Z|m%G?;^a}x+227w-q zbyeqbIN+SQN#u_Mv;AUWm`bGw(4s6x+I9aw)&UjCPh4$dD_A!2El@}cOcXmHR34G_Doq5W!e@- zc3t_V>Dt`rxUTUUE;mhCd`VFMqioR&s@$t=ndipHMn7(8zvcpfjioS*MbzN!fD1>G zrP+yPh7imHK`7cg|G;nE2&nc(P|2;2ulGx@gYJyv!v^@xqP+1*|Kx`Uk**q zQU86u^G}}_{gqpNd*q^a{Q@kgcw*E-d4d?ofX9f^D6pq)kJN21Tu3jbE^n21z!Lb_ zq&$Zpy{J+nqRx1g{*;3yf*v&etkGIGK4-b^XHRp7dxqpE9N^Vcz1*#azKhD-XS;$v z%Z+4Yg+7dI(jtz8I;?OVu$StVW=nZrh|~mZvfJBhs5-(i+1b72Y|;3yl4eClgO9-j z&PF=hsePt&qmL=6#4EX|OkLeM-nVGH<3T<;TII;2Jxmx)dSqic)5O+gq_k4TOZNTA zAla-EB)WfU;2p%O@0(e%9Gfpa$WvQ=%GxwmpwlVdr+e9qza*2YTUFRRBJ%~Bf%#8h2?76_6CnQ zIyNxrsAyw-F7zRIz;ZO0T$IbqQP~VjrRouXb!t2D86PS3SL{h%nEqnccV%Og$1qIZ zq88hDUa0e7=!GnmBdJC0E84)O9HaAOi9DE-(0oybyG6>na;Z-tIc(cQ7(5$Mg6{=L zroQogKTpuzmV5vAJ!(pJmtzvUc{<#p?5l>oIBf{ZsNHsjC>4fKjb#F!GCav5F8VT3 z--ov3MEWGD5W9X51WTA>w*R>WEqs@%DB>lR$_uZZ2feEa(S``0Jn!GsGNK8!fA;>G`r z8#`VYAcWw@lq*}lj5)LB&73=Xemb<`$Bq0ZLJX;IW6G2ft+s5)+F}6Grd^Uu{cz%d z(m6%y?A^03+qDe|Dox4(-)E;pM~<`EaNLdV}aa55R+i)R1biQqMZn zy4v&Ti4rur{IXBI@$1{akH7Hb{QUd-hq|9*k)82hfd={&;DHJ*$Y6s8Rb=rI zw$Vrzn#m8FrkUB0RS~&a=0mxaM$!N)0f!<_y{Tvyi|)CITvGoDg%gX^u_q^b-!%jl zJ@z~*o}fixb&;jBRo7INVHz~wr=pHZYN@808h`-=p2}*guD%LuLRfudrCTe1z z@X@6yp#K~PBYucBm!42bN|l<0)6p1~J><{>&fII`7PLK`+z0VOIr*C|yOSag^ytt~Hb>RZ82$PfHE^>D$9pX1o?bg+hDZS!2za zDax`*CoWp>bT`OZ5`62`xYZR|UE3}8Jdl&=(v5fCdhZQd2wU=f>d$}=PA`aimbBH< z9C|F9Xk*%$@JbLYa7JnaaYH1zPUkj7q&6ifENBDLQgA3YZEb};i}_;^Gk^ht zIKYUHj7l+&Nh)6o=Xl6$rZeeA#pq!XBiVV&wT1A6mk5vireqhl6U2GPxAUfnWd_hJ2%>QRr-a)A2T=u#|=8%BHCuKw*p zAj2f6O$&F^pD;0}Lp<$ispbR|>$pUE6?)#jOrw#DyoQ_K8)L)Fm{ByEa$;MfOeDwh zC^V*%DBSYV4Oz4?>EO?xw|mzD1WCw#`gE6j zO`b3Z#sqkiYWve@K~Skmd(O^rQkmlnyOUKOsC1>SjGdsH45G62C|URt*&*ii{0#MS5OHtEor}-KFNT0yyX21h<;_omF%@E5;Yi3 zC}+9KzKtnrtyw_++!ODImbiRv3JXhWEF;BRY|VPk(=yUh&m5g7qSXfI@B}_>O&(^1LIEU z%M+-MFTcB`m>>OHl}LZ6I?#L` z^q>1|XhaLz0PN87p%Y!`M@t&fla4f{DeaFuSK7~!2DGCO?dd&tdenjzb*V#*YEpmt z)T%c1s$cDDQU3!QcZRa9JqlL|-g?(i+}D&h_$z95Gfxnx+)ZjC#xvXme06@01udID z{XR=SAN{XL$wkWn^v+x4kV!;WM!W0Z? z1yP3mwQ_=8HhslN#+OBxAfQu9-2;PWRwA8s_VA8zkNfvmW~@yP0=rKC5rTUq-tbOD zd3VMC4*B0MIr5aZSQJNEm04llSJA6&)pm1}fYo;ZUIsct$XsULBIr4Lmx?{k*5)^# zg#)cGCF4Kklf>IA@#UUZs{C>L%KJ0;Z}lPNi7))aAChz{Is4GsTN>T`yx6240GLe= zDt>9oS^HfUa90S@ZH;We?+pCdOo#k1m0S1q!n*Mckbayd-&Lt%^8D|=z(jt8lLsTo z;iOg-q^E5BP!oExY|Q2<)uvkl!CR{5WJ!k>-$6s&VGn7;ZJL4=0fl&@^BIe0JivE( z_{S;&C4wbr8SytW*_0KWWF~t;Ii7SAQL}8`b6aB;M^gnlNKrB1ly_*hEj}^;P6ubh zdGu0*lY3WpNAHJ%Cb(-m6?sd@ghBC0o3avXvKrR{fH&bizHxxq7ZlKSSVd<-tmidC zF@f~ZBX=}=^^+}Dp%w+Tf%f8lzNcqU7==xyC3mQYO;}BMQY<1tITJ-+TsT-Ml6~5@ zRklM_c9$Gyq$xeI9o=y@1Xg#uI?;*<_=S~NPu$mC)`k|^IEnOQB#%^%3%QWQ7?FA?iukvVOh|PAWg^k&O)64d zJop_I(h#pV2(U*S$Q5nJRFcUzjLMgiPbgj@*pgA0l*7}7H7Ilo$d3ZyUn6lI3NU`$ zu@;i`DE<_b>#~&gF_cAlhv_(fUul9bxmIzfhTjO42VqA*X(knJ=4RX2lg33*T^TbB zd6Hr|mXmjue0gTemvU>-6N#vZpuz*Q5mtPMi}bLT14(-ud6x>gmpFxzfSDL({|Pu` z8Ft&1kC=&#P!t{qzyaY=nyF-(rb(KpnM$q61Fv}iuSt!F@+T|;7TtxD850&gf|YAA z4=M$kRd<=agqGeH7;zY!hoPB#^N`@>nbmcg;W3)0Nt)A{n9qrw*F{jpnLMrOo!{x5 z5YrrqQWW6%E#pH{tiv5Z*pn7WSAu3QVJTNU&;jV99|O<<9gtb_DICe^GoHnnpEYc1 zN1#4fUFf%6)yV-0s-V&toe%1u@RXesN}<`=KuXk_NKuWacSX}^DWWqFe^L?Gd4@Yl z9A)tl01BV=!k?Kr02crNJfNK95Cs|lqAP-&LFbW{XlCj3&MgF2+chD)b|s3}vW4bh}L z5SvW8q>&1#lj@+0YN?lsshKL70)nKc38kKDn%wmdXoHKE(j(2&9PD6z1nHbBic{lw z7-!%E0FVla!4KDv0#QH?{Ryjf+L^f~k%r1jq=QxFREvKq8%yJ-=9DR8Gd6p5F?`|@ z7xG%wI#LFMpqi>t)k&HMz^x1Vt=?LiPb#V9N~!0nt?8<+>uRZwv+Ahh3a=5fjpQ(| zTm=?IF{b&m6mL3OsWP0xsHT3z0RR91_8F^90014(7j^2ZmDi(ng{&L0tmOHl$67%_ z6>Pw@A%Z0lDsd8y^N!|Z6;@$AaWW?4u^PZ~im=79Wa6@m6I*nKsO+jKP->kvi?a;c zsNlMw=IXQO`m(Iu|$8oAvXu9XX~lgqP``>jC>w4Gb2j|;lq z@|~A;7S+iD?7$9|G7w3C7V{t(aZ#ROIHQjr)vv=);lf0lB^)|M$>XD216{iNGw7Vs1=eD ztz;7E8zKZFIhePy)igmtS{~^ExLF}59ON<)Q6bk;FhLcwYS+HmdOB)Bz@bYP=P*otk0>5^$X-lwY%p4-wEvU>5)s z5CIgh5kcXqOQZlske?ku!}+unpMA9A`!N{A^$TG{p(RH($ ztHACGuGUGpmutD2tI3_r$&Q-Aq1?HinS&p!Q723Y+hr93A)^6M4oN@{G{nLzKnN09 z0EpMZ|0{7vnTT0iOR&HE4*=j>n8C14rT`nz1R4O$!K?vHalFZEumkYRzzhH?&|SVf z&HtbQ!+Zt^`vH8>yvJ$9bS1rUrFOwptZG}mf689cWX1^PJlBF{ox@57lSL5tAylNk zl0!Yz!Fg8{zUezBz@iqm0ZSlZvT-vJbM!pw`x4GWRE&HPNwq1Xktq*dJNu_A9iqR| z(a}YvG1pYUvr#Q1J3gOW6WBt=$#op}%SE-RwZ!X5igszyfBl0xQkZ5m3_+oXHHF$<#Ty z;i{x-p@TYT)DcL!mXZVvfDg%aOKYJP5^4q!Fb~iz;Gla3w>%IA5CRc^1_6s%|M|_T302=TCqrn4FP{aeV0B0R^AHcN+8+7=9un;Z~9S#6b0M2B+*ee$qBSrz4 z5aTm0<26p>H*VuOP60d4<2~-wUasX|4(47?=4Ec?XO8A+uI6S=0bbtb6Y%D7?&cGK4{a{v zWM1bKQ0GOS=X;Lhf3D+6o(Y4F<2K#^nGj(z-T>|pMgfZ+<0Cd=3nuCA zKw>1e=nCczmVW4<{^^`P>Y&a546x{y{^_ZH>Z$(er~c}(4(q2r>m^3(|F=%-ua4`x z&g;A$0jC}T!T#&R&H%!0?8lDm$*%0mUhKmj?9CqR^`Pub(CpG40gycpubJ&GE!Ex5 z(h{)J9T48%?cLmc-Kd-0=Pupk4&Fj>?%W-<<&rgIno4LvuU;i6t$7wkKIHxG@Ba?) z0Wa_aPw)lr@3!9Sy^aP9kOT&ii3q+N0|2&VAmZ}D;>@A2DZpe25T|)95V8sY+t6JH z@C3BlDH0#z1@X1XYfuDI%Ybp1Ee=q@o^Lg8^EZ$4H;->SPjCI^^F06aJrDFl-}Cpj zZbo18`-b%Sru06~^hfXXOE2|NPxVc2^-!PlIG+St&-GpJ^X|U$*8r~GdJXfy0oT9-$UmL#zRap9BdG_=g|= z`Sy!~mUxLRAQGVA3dRg(PJLQ6!I`TAvTjX!6>C?pU&VSQ8`f)5rDvIX)jA+R(zb8U#D$x{?bf?< z?FPWhS8v|9TOs(>I#@8_1cuW>NZgq5W5)r=B_*d8?N!WTCl5G7ZUE=cpy!U>Jesp- z)2RoP7R?#$XS-TQzjo~!ZriqSZ`)P+TCeWWOJ~=8&D-~Fv~z9C)*ae(>*UdoOQ&Al z`gQA;?Ka+wz$2$YS z0lyoMFYx$5&_4ygWAH(S>Kkyv1+F?zyaDwKkFUJQ`>7@nLlpmUL=f@efW&{?Lh&CG z8Dg=;5C=PBfzJY1fdc|KFj0XWb!=<9AA{TuE7^whPAkl|OA<+}mOS!FC6Sa8$|<9y z@=7bSta3{%xx9)oA~7p+tpq42fXw^Syefi$4DcreLLzwcsGb6Vry?I3qEn}aUaFG^ z05s^pEqxAX0)SKo3g7|&mhdP5BQ%JDhnWIUprUQ`Jm@3onp z!~=7s)>vl`Byl`vYtr`GZNU{+TWAO5wOC@)tyb9a6eR!AT^QpH(OG77wRc*1^VK)r zdZ)ejUlF@)QQ#4o<%eDp2@ZC!d0T$q=Mus?007sH@ z99d)?H-%8Db**^tz!`!-!NukF+g}G&9MG*9@3uQ~d)*ySam~kj z++e;-*Y)7>4ljN5?+V9T*UP6>adLncX6W7s@y`GK^jnLa(7yr!RB_#cZ#UlV(%n_Q z*V`Mn*LnYtC!8V4@vfeC>Tg$2SGk2{KHkwm?7k+31-|!ncoBw|VT--33E^qm#TP`< zcYS|*m5ZN0e^Ngxn}3eQiv=diQ2|lbBb1$pe+sjS+jtg$3Ct`f0gw}%A~1o51i%1d zi&X$P5Cue;txrC&R2iO71^@uyA5ZWBr4)sylAOT;04Re3Dm4!fCh8&yNR&uE#X$$E zt$#rzViApaL?kBBZ4AL&0Q51QCpr;|qH~jn8}kahHP>3iQsSOo7PASWAx`knQ~>;80G?4~BVQZa2D)rPJ#HaLr&(m(RRfr>y;0GUBO>J+@h3?tlFni=t?5q> z#J#zlZdY*B;u}XwH}Yvwato3xz8F?kyfHOI?wYCb()X=k?hjowEh!lB1~|vMZgQGB z2mqrAO^uN%gPR=@2VAHCY7Rn~@tQ};3TQwA@JIn8I~h2QH?MJ0RHHG0pagSr1`HNN zgYaZo06I`m7uJ(#JPCkCEPzV1CzJ6)g9P- zr0gq-N6X;%?eajN9I;)Ubr%p>%w16&(OYLF&KO;burVGHC@n`z3YTwR{)=t@vKS&y zZ7j2xCCC5DCXQGOP7^5*m`#=#d0{q`-S$=iRFiG18p2!R`m6~bFhU26 zAb}gX#UUg~fejTUXM8xB2OV&s2ie?AfefGw8j#3Oh6Y6PM*3>sm2{;oeQ8OP^remU z4^K&YQl`4`KVJMCQ}yWPRgt>%JA>(7gtM38Ch8P*s5`3W*h*ngGb^amni0Gfm$^>^ z^^5c~Uh5o(Rc4is)>YKifvTz^xnZ!tFsd+uP z_F5~$Sxh6l5gg)Ptg!zy%ot}uIC-WuNKjS|8u@~GtXMO zzF)5xi(-L|4|yVfa31+Xs8_1#oD1eDgNjQwYxiZS` zL}d^c*_KUyH&FU1%=;D;(;u|*q+0mUi(fb_e>^h-LEgEX8t zgNj&!m>U3mXsAE?89k8<3$R2?OuvqJ3zdr+kXXh4ORTvB_yAMXE>2uPT^v6I+(lpf zMIrhR)zK*>%N1)Gp1*rLDIz@y1VTUBI{%6K98v%IP5!R1+>@39|JY9Atu!|(C2a}<%U zyR~%G7N(LV&YQviLbA7tJ!RXaTe`#IlNXN}xE67?AY(AxDV@dRC}&ByW5TR<(!=D# zGRs<-O4*o*8$?6gvTg#Wk;yhkG$yn;$)4~yUnD=`vY}2X$qrCCMFA8507TlNlu5Y> zzBv>fa&#)4y?dplfe78*jura(s21C7obix}ew(dcgfvc>Dp}~oa%jOZB7?Huyv>3p0 zLd{~q0J1%dA*bSdk&#?7&icIto63e0#FBY9FFQnhXszSZN)@q~7pqB1izx2fMV?eX z4oIO?5I!d207n5j;6tINq^%iR0-SslPw5oxdrGTJ&x*1>^=!`qG(B@<&ASLfkNGPs z(z?MhHCE#mri!9DnjQsvMkU<5`4q?hviqH-0-d)rLPz>W80*ix0lZS`$kB5o24y>3 z!XK1MuM49`uF5OI%E8Ngx4qfCx{{XIYq7{QDu5Kdtdl!%L7x4*D~r*Vf8?ZYAws$X zCDYs&$K$bpxuqoBND=u+6fH~|EyvPn36JDVJ?l|&Y9{t%GV9};(4exC#KU}1N|qc% z93dc>Osy_sE$4JjQz?M)bWhs)hv>`+d3pj1NQFAH03Y~=CP+Hp`iC+|(}Tc=C}@HK ztQ4Qw2BDZvH9gczn^HtwR3d^KT9Z7l^AE%8wu>7nPiwZ$eAJ`j!p`)-R6Di}{T!PT zu(VP;9sEX2MKy*@muC|yIJBGpo6^U`OuUl4&CoQGA@*^q|QD2+KnqVWiLl--O&k-G0cHI|U@;6RGEFN>b z0+Fkibz4FjtT7}#0y{V)b%sC!@4S+)WtyjHXZyBgIXau&kB5SJWazh(pff z8(7QL&6~hJg}sS{{Y8f569tff4^YJq=m4WslzZ5KlFL|_!-)W(2Y%Yzt6|fV{gm4O zy)`SM10EFX3A#%x`@V(h?UouiS$H}=Xv-W8B7%A@_d6~l0>qa8OH6;aGo zHPtiBbacmu{IO!OLA{eTxSAtQ<+oW8(YUNZz?Hpni;xaXti9QkAq#6tU8{ znTrKTkt-$GF!wzeBLbA&H;!JZDaupceQCah2JZ6c23S^z^aFEY_F^hbK*9_ZSb zR9dleO|UvbF#x_P31oVV~%!V;Yv z!JMDRLCE?lEW4^-5A-X``xa0OJx2vMZHYoS3fmKH)sl>u18GEJlPY!pmcb{hDi`D1 zY;%@&rql-8od6cr){&oo&R*PRe%&-LLpO=hNUYNy610P&-sLNJs9 zyI+dC`&vi#TG$>sjH5>Fc-wkCXmMcOIk!6k_{^cK214zP_YF?Viy&0_+i)^udGSB0qXtmzC z0!MWGsVt#7su5RmQn`U_`lAWJ?TO-&fPHVX?Ph>;h9 z`Vge3{;+!<@CB2fQ;Zn(-ArzcR(rn8fx}U!Qr{}t=WZ3`1wO}V)69Bi9s`q>J5onq zl2#TezD<+f6HT$r)Z_l8?A-3)QF+ALeu`Co*U%UMcz7mD86sI$NrClgnr_?`zUf1R z$>U5!%PlycAnJ@o8L)e^H}T8Q?UyxTKuGH*IPUbtB`)_C~)=KU2^R1G3zyc$5=`> z)-D(iBc>l4L&v5V6b0;g@gpFm$@xKHV2wXFZ-rmpnf@@fO{@`!;P# zUp&>W>;>OaUq0N=_!z{^^&B6WgCGY2C&~8JmqwS(-lk=FF>;v9>77Oyl@Zq;PwoPE z^2ZGkH+gPMt8xRJZY|$-Znr3FzqFt-DIYY(fbD1PwNT3G)g)u8V;v)v(js$j%*^yb zy$UvbPO%cy?7R|RdAFWxTi%~)kiD9=oboTGs>{!UK`M8Nbr=ya!i25G8L%P^SMF7& zqRr&k<3X=l?6MN6kEB{b9u@=MLDZ`ce=j?|i_7$>y@o(6DjXja8#1brX^(Fp{|#2$ zTr$J`*IaK3D^=+nKM~zZCZNu&#VuSRe{ka#b|TN0n>NIp*4OCsWUAL~o>11(Wq5*=q6Vp-4QGd|LNFYvkwD`o-G znBUsUbYH`&uiyNZ{rdO+P<0{Q#t97|R2QSi*P@E#PrCDQoriKgywV*13>xYE;8P&o zqNWihX}$$&le~P0<{x8+5oDM8e$DAXv*D{hldLy@l(^w2Coiur!0s3OsQh;DAOAh+ zs?nKVV0l2fKew|w)n|swW~Syxb?YTHb7Sj3_+oDql)+E!T7Xy(AOHk{2KfyvI3R#P zgbDu)23R<70Y8BQD+WkVQR6^;2PrZfC~_ppksd=LwD`~>Lyl(xU|gV4;6suN3qnXq z65+y)5lONfC{gIoq9YZOEGTd&Oo2;97QD!^W`L6rk&0x>w5mvu52Gf8Iq;>$n@~so z3y`)XM2kTaV!T*V=0v#$CH`B>w(dxTVb$i{o0VkV!i5Ff9H3Zn#GH#C4gz z5lh@^6tT#_m~H;d%z3Zj(rNXMCZKvX>(%%aaDWZE=uoDnZ{z+qp+JJ%zHt*WaOU@6 z#KeyyPp%xffz!*MLys<9ky!Z`Q%2KH z%Ga_3)l^eX(l*6WOCAL!P>EMU)FiR=_0=qoCFNL@MFr=`Xul9KrPNo0(IwHq6FmtO zK}v!(1>a-TUxs$X{xCLnYt#ManbFTYIebu%H68J<~A$Xw~9Ts*=M7j zw%Tj6-L~7w5yhWdaktFlfcTDhtb+x4L@bvJHMBRi2TK*ESoUEl)qDRomPrvv64kWO z8IN-~AG~kX*-%g|E^e1ZAUcI)R9x~a5<44k_u?_&2Nj_)nvSdjM<=0jP#4e0<7k zlz9%RwsWfHndfW1a+P)7MxL@+FoPQ0AO}0>!4ICzEZ2(Amg4gfdztV@6w+6@s06NP zQAt7u0f~YD#~%5ZCOv^1Z35m(ZvKeK`rC=bT%M_`1lV~-JEl5$3!8WrMorH)@3sac>LIReP1ZFE( zv`fMM@(_)|1ztgv)1vM-sfzthLoTCP%>?)|oDINF^;3*&WLCbwaDb9Lo8O6;M!i){ z>QP%!lqUgb%7LuLfK$U12MEA{*HlnBS=$cR)B`}Wy$XcS5ex=(>C0aNGnm30CI^l8 zx{%B$Do%1-M+O9?-8pD@v@xFV_{K5}Nn}4YIaW_%G@k-#gkyDzB9&%Dk%Wb1UqFeA z%=#!XX$>fH8OmNC3G_*)h$Sv|A<7`BH?G(zk0hj%6gr=o!ybKja!NsArx{B!C3Uum zbvhyt?6OBAH*QE|=q!>WFK4{Z*>i3IOw`rHhZ)ACPcadiBqb%77JRVOegYJY!sv*Z zqD_-C_Uj*Kj+98!)TuR`ni|!JIyG4qpljF(fdO`U$~rNqoKKx?e@(Qlb|D$HXem>vJ%CXUF2h)^qv9aSfW3GT8zyXX&$w zpX|!R+zOO34&_=YlZ!>**t*}{b&zaI7av7Z#)uV9j@{(kQ1n&VwjOUYeH4;qJpxFh zM5{!e9g65+$kV|5LuO?xGU!&Rg66p zLyu4el2C`qB%mhA$$#v0lywAZLwPzh&@f7zJDI3)g1V`y8C3zlnWc6J;SUD5g8}l5 zuYBQKU;EaVzVp4We(}5C`~vvC0v<4d3vA#68@K_@K`?_G?BEAOIKl|-Dp@OR;R|Cp z|HB&$i&>nB6|+33A<1jVJpUrOgZywu)pXB5gNslGtw?*jGpvXpyE`_n?8fzk$$+97 z+8XPyN979Lfa(d+938WNH7QHKkYp8QGU+8xOcOa3CMDAe4xOz-9Y$-ku)z%UhjzMZ zoNmXKzVQ~wa>6G(n|ZkM?M0;CyseylXOZGMG*WopOG+UlNuH6TXGp`OV&KP8(&UkU zUy7YmN1A6ymC~u%daTSWIlt5>03HQAV5ZV*8un%<1WfgZ0ZMJ^Q=>Z7s$O-fnd)kP zFqPG^o;9s&ZR=a(I@h}1^{ts2OAP}%*uow*u_GAXf1Qj+DPfikMI;lwo={-k|3yp- zFM6fg@MT!*!W>kdI6CibNGyx37Zu;)ls_iL(k&yTPCj;*ENP4-EjkHEDovMnNz`=z z)X78oJ?)I8&YfjGS(mM66a@(nQn9Vm8kw2SjF^yh^IEo1x|t%vQfy{>shD5#UAl0y zbftcKPycBer(gLHM(6p^{fnks%}D!M$-h)G(O-l z)buJ~0!h#sb{G(Wr$asJQlC22t8VqHC&3I?-#XX3?)9&OJ?vs1JK3N9>tdrl?P^~; z+i{Ax8RZtl?`oX%Kt$3s+bBv!!jP1&lE@f0mPcGZh)J4dxCz0IE#*Qp|6dgL;jWBm zL}1}j!D=p-D_4w1J>*O1&e=KR^$VY3Lfk%cdloRIDe?TY3pwSImUa@4?tQGC7>y`e za2w^@{lX)eJExMlk(lH0QM89#oEEtFn_4Jf6{!Sc{|QvN$q8k72+K80 z2ZF`9aYgbmlp^_*!C4!noRVG;P4CT#o=W`)QTY87j|MyDa22f^Rj&EFLAUmyD69|B?^0?RbzoZzI%>-Y{3l0=J; z5<;0E!tK-Qb8k50^{~J`Koe1*3=DnH5S?6p{`i zkxN6!pG-lT$SL2RXyZz0AwBqv;&ox6%!3v_#u#=Iq#56$|1nbcOp~06Sm5-IV91#T zD&8p#U8iW;`VkdAL_h{n01Gf(rPSg;(!>q6Ayo-tLMr4!GGs$K4Yozkn3&a)eTDde zNRb?k<)qsN4w8cCjzb_xP(W1WIAXkT%L+b|mH>}!fkc!L5?#R2lJ(0O9fgu4mXf%Z zSzQatyjw|dOGvU{i0q`8e4vGlTp(GVy`+v$1jbv$M+WZGxk;Aa!AP^QToDEqPi0e_ z2n04pSHRSZB7NXbEmA1`&&{CJ$)y}T^b7~c;iD~3T7Y3>kRxLRkYQL}_h}az+R660 z+sB35O4!RnWn5IW$Y0vU&mqQ8%|iraT>~H;{{2cG4r(0jzsN5VTZMjm>@-OS{m&rB6w9?>d;ZMVPskQoJ5aG<& zU;!C6mJ*A6-cG1fYz9a*nx{IZ;Ylf%78=?XvW{ZLCHm;Y^VQ{kqK_J$)}n16onV{s za9fO(4v7^-7eYk5co|r7mtd?C`xVd~jt%?;BylK?_&uOJ&W2_pBxsUog-K?KqUZ6g~-b6(Y0Y*XC39bBogEZkzZIv<`vK8g=Mw0jL}e@3hL-^|Jl~$4U%f% zXyu*W8zPinZIS5NODY=N4>AR6swd>k2TbPF_^D$5_(jMR54}Ca%4DZ_-5bO}90+Dq z;pvV<)znXjoN*G&&yi=8f`$C$iFPH5Hez8nLdJ99UeBFRp(sY6)umnDWuhJEY292# zg3@*kDG5Rrk{sm4nVZ3_%m^YT&1 zXsz1nt>P-4C`}_8o}9?3Wiium@k@*Jokbi7HW^QEE)MDB%d$F~oMh)cja(@W28R5m zY!;zQzT{Y*n0E;$?Um;nEeT4*4h*Ueq}W|ho&@V8o@nq$_Vo{$|FUC-uu*0`6Hu<3 zW4R{4_1oy7guo?V4_Zm>@z}Tm=3VL(T;|#K)lZ<3Bm8`0PkmXV^5>!=ipx3Df$C4B zQ7Wi1VCsmVU(t()3Dlff8orWcomk<8mTF|)3T0l0wV7VYE}7x9-n?p6uIlX0@~r)d z=A1s-47R3e>DzeyU+#qIw$pAi9uf}b^9Bxr~r5c7-q-KTCSnOlCPs9p_lhPmh{|rFTh~dRH5_VOmR(>hz zN-ofe%vOwGFxqCuEofpYC`a^_y}aeRkYl`w^9@hDRn8fbYVM?bb*VMq%QH z;&*jlR`qQ2I`8xT(yorK_wnj#`dA{OmVnSERaix0k?S0_%xdaWhde1~h3r1{&yMkG z){;oL8Lv%#+-k}sz_rIM!fE%g)cEiReCC-oEzWK>nZQj?mN2NB z5ye1BR!kbJ6rSY)4qQpJn1n7+!k!QRI2GrTQsutSV{i)a9HdJ*?4n|!IV#PD(Z;ZqW;t9$$ix-~Aw?F*dbA_#shN$Lb!0ci{2lQp`qq@HPwO>G#J;KNZ58!<(~ ziAC4GZjzsPXam%V#c%-U$1m$%8oQmWd z_KnZ-VpSE3bI%IzI3EWGpP{6{>-W)S+V0CKw`bfS)sxXgRg3jtk{mFYu}M@-;_<@CXx_7V8tPJxj|;+YfIg`LDwOFAl=r4%6G^8* zI1}LR4(1qV!19#8c8O5dsb`wS*jR_!yZ!lHWT+S(13M)6EM&$`dX&uoIl?x3EZx=87 z`EyTC&Y?zb#f6!IsrwrDr8eGJc(+A-XdH$2yF{p^k=gF*WBd8;HP=eO6yaZWYM3JV zjhgj;6a3Q%`oc532XQdaQZGBdHLq%{84mDIkQ3%GiazyNj}&>hwTQN^ddCeRQg~iB znJvSRrN$Rek{+*$Z&wSHnUsk|ur(R=TB;tptwBhWcG;~s=pHU?J@MN5~?A^8Vedwp7L%*7n$s=q5E8ln9s zIRdEZ&UAC*I*bBD00{gR1Tc6I;lTs}7A$0FFoC{<{0dy4h|6{x;x}wIUicL4~47v}$ zzz_ngE7$NF;HwA|gOI`sExZsz3KhVIrpg?o?;_GtYw)0=0#G2s6fK-k#m5#j@x9Za zbBwVM10Ya27q61)IsaDjY_x<#ON}}Jm7AcR3F3puq{>VZvAzJDoJuq!6*MR(o^;AG zfxhNSD7DoL5=~5x#Pn)0u|8Duw8&^uj4U>(gA+}$ zNOVodHD`p6L^&f<%teA=Vl+g7GHYm}i#)P4QzHk8Zm~~a%dEjCL8VAk@}?pUv*jYP zF}?&YwJbXaDSC89jy!CYy46flNz|s2Q>Y^kqifJ6qbo&}lr<@xvM#e%Wi<&vBBML6 zy_p)yD68@cG8WdAw$+X}YY|NFB(gYqildHL+Q`R?vO9OqHcO=R!4~!1caRDdXwSn< z>m|!X4)#45!U7G z@xL(##&XM;{{qygsZxGOR8|+Y>QFb)jd8|9X>?F%nrJ>6>7qKqoe=v$9qOHL(+*sZTO=y(Map@m3wjYueO#_Vm= zDEhm*8w=`o=l4hha#L(^)YeCKt%Y;Iuwz^;Hl0mNZK1bA(@vs=3ffG}^6)O%IS+XR zm%GZF!qzk+G4p&VLEjUPGS5+UPu>6p=eXk9rG$O6pJYdKEWjehD&gcYTo~aD>G_vf zt3(6RD(Vg`xcTfErm!Z5WxSer-j-`wJei7~2s_<>WUg$}q++?C2_jGuxBd>AGNaqY z<@j&Xs-)6oFVoThangX9ASMV$PJj#{X~1|3dZ(?W<$F6pcX9_U^2snojw)HapC zV6 zXJ<^cINv#S;?q_xCD}I0rYaAf`sNX zs&G$v`(j@9D4CEuS~Pca!rmt{)JPRI3_)(vW>YeA!~o4EKkMV=8?7QRAmQdBzXFm} z;K2ZrXoPw+vB=HVmmQ%N1Y@|t7##;omYom~|9VGz=9Rp5Jao-)GL5wtHWhh6ab6a) znbquOp+-)~Dyp633~i!-y0vqPwnp#7WZ`0z)=mEMeRR8_js8fGS?Mk%E6fxUW0ggU zW@}|J#M>8f_QQ}!$2rKI$VL+=Fy>y-Zlf}%YF?6^hAzZ8OFWNp->6DpEpe}xq{sp} zk`adhXmB<%ZYK8!6U$X9SV|3F1$){q9+j~@n8H-Ma2Bt>Fc6#S4DeL9$JGq=YXJSC z2ZtP++jzz8nxYJVCml&(gc;35kS*bd%A>Ygy(){$RS9?k!!kNyrFSw%Gm5u}qDiPeK{h0!PTe{_1IL>vdJI zGAP)LgetHTRWkw4w^ZEG4)snJI*b6xJGXl!T!{%UlT5&_RuyQ=#3XIq$j8C-k=*A# zM{YWI+aMGuz+eq|BEu5to`2gfvQ|^gC88>O^Z`%eRkR~GX~6D zMs1=t+~pHSwo<@lvO9$sC1YR9J*Uo!=U!!L-x?@|r;#O$BW!0vA9sjb+)XP-M972Y z)LmsP>8VO9s@W>1i8>M^h(VlONo~ZTO}vy!K3d#4iizE_V)B~>&8brIRIdX@q#2w0 z5RVMfj_oEMb+GPl*53J*7G`#`AE}|^&Z#*-wDa}C%66`b(k5SfbgI-V_5e1tJz1&N zz#Fskw~uQ~^ShCGX$>w~2XLtRb>>5A^{E?Er(G%uN&@yhl`Y`}$bOOzKSc{qttXlA zOFq2?yOMC%zaI9nm&=t~`P)Eh_M74^JmHpWulIB=s^V!bb&$C2C~-##TO>AVT;n5L zz{7eXtzjkRDA#X4i>*Affoq&&BoX5Bl8f3eO$=+!T*GDkcVrv13bp+Zg7$NPYo4tF zpMF0os)ya93-#{K78Uvs$UsgACTOZY((c+cF1g~aZ~(26WG-TCC$F%j?DXp6Hm=?( ztJfsN?)GIvs;9*Q>$hH}?lw^Ee2sgKO@&Zs~`} z0F>nM4iHpi0^Mj$HtMV~5(=a;B}|xvwswaCSiq7xiY&C`#|SP@YHi~%4%coE*Cy+0 z6h-#F5DfprFyZFp;wYuz9;OUSubhs|Ua}1BTn+dVDMTue`%;dcJWwHyNa`elOfm;v zj;=g>iN8YT@jz^jw$9^*Z&WaaR`wJtSdm;P3n$iwc3glV8H!6cbfN`#7BL21`?(7Fvq+&nrCR^Z4GPF+tQ_4LyY13Mb z7RPUM^5?Da$acQvN8E3B)&p=XhHs`rr^e?Fu~IAT=4_y&SynFNZV!IsP8^rYR0hyx zSc;8ur`>kSd(MvEmWpSTrvwp(AM{0%BB`os$i!xCU@9xaHZa(X#(`dFD+L2cGEH!r zCAlbXe`I1RM~5s@21$7ED%n!7Z07iek&V_5)ppCG#3U%PQVwg<^9+(Ff=n;s@Uj1D z)13gUAwg0$E0Q;R(>MEqv2+vSOsLC1^5V{EBY%(KChpmc51lAzgsO>)h$RxI)7FSf za*zY2W)6Zp(KKYtUG_(~Oe3u#23t-EaKyt9--{AQC@DEiE4;-$mPj_nXvWM6kCO(4S15+$dW7x`8T#mi2LMuZmQ8e*WdT1O6ra|$G3Rg{Kd?es> zPf(I^85L95u4ERkf%KDvYvmzRD=AG5O-yaNY%MlYrYwDAA>L0CHIvg?WxuGWZ!88s&jh8+ zDo8aGp8^MTHsk#&aWY+o7{%t4T&@7Eu#GYzS{+oBNk1|hM4^9nJr3(0Bgw8>2ELQMY;R$;@542^R* zkFny;Fv;AoBO!=lzswtX&*IjUIcv4nX7!06V*QGw&KwV=o^Bz?#5ingqb4OvujwR8EUGC=si)lSvb0Se6Q0=8s^t{PS}Ux1w9%`$F(ea3Wi^6!&;uCh!)!RhU6BaH8R!INHhJ_Xm+O}z7b60m` z4^3_MvrI4Itcjg6^3i%k*r~{a7+J#6LxnOjKkO>gVRl4C^=0p zw2s$dD|N%luuV6%cxz9MohoN=wnmopJ*;C>+;#K_DN4->mGI0`iVPFQEm$bViD+$XL`)Hg&S>t4@CDgc+-hh}`f))mOM~l7G=?f1Z;GuFj&Z=0-tD`S8gVo+5b}JlALQt z3?dVfKt8g951*2be7Hit*(#e)9s|H0wB&X1Vt5U-kPY!tcxhLR8#*t(|Jb3=^uK;) zPykLjC6+nWbUBUJIaLc+9JWK*v@szT4y)5kVeLGjBQsqPkIGEBq?Jk02AUa_TtR|e zS*N_zDi_1Zrw}bxdAh$cW!6}_@OId?4iPtu3j6BjCD{c>-YdNX)FsL-y=o5I?9bYw zWB(FO&s1fv>+c$Zlhew|X% z7VC3F^{n8LNd{?j-vcxCGUEQpS<34rUedCu^dNY`U+Do$G~dr!-gmT^=vh9hw=*Uo+@67|QdS#Ai` zC}I$Am3iwlC^Tn|ROJ%cM!YXShXz3bujJx;aT^0X_^WVxCzJGyY{Y1Z+BqI=sf@_E z{!q3(u=_@<4hVC2mJ>W$%~l-=Ib42;gHw3&BKx*~+2-bvEmOl7MTi+g_=8EDs|G=m zs;cw!mafy(vQ9**NN~h2q($jwq}rug^f)?7ar8W_juu?%P^zVHwuw?>0+QgBD4dQ4 z359L7m9k?9iEU8>paF#9S;*+$ctmKLD7wtAfkoDp4w0`0{{bh4l`cY3IJ=l6xv)fl z+s)q`Va0SfrTdH@mVKA|xQjD*E%r|N~+F4?wwt9!0$vLud&aAYEO66qMxR1$+55o4XR`!+7^ykzu$uMCpE$)^_sOQ-| z6nT8DMU7e89f`rPiaT^!9u;*u88|RZgoXb$8JP{SOT5N$@H3iZ{FVeipHMj%cO0F% z2w7(!hzuzMU;+vN9{71IWNv{BB(J#RwSd|qf;l(%|KYTIa$jRztrX*}_~~SC2e!=$ zp!$j7X@a&Dz_uZJuGJVKp_7Y00?t!j<DA18qu$Ay!Ja=yxdn9^5NFRllcTae7_vZNYaaB zd#Gh^eLxj?6yKx358~g+0a_pEAK;ZrZ);=9|CVD+cJcf%3H;9{8S$*%?wS2#2!XQE z*r%^ACzKC`%u!^+6W`w|E6{s;<*VP#Tiz_1TLt}`&y#m(=$!SIbGbj-_SRG(@tk9O zFS&D*Ysv-Ip!c>4%j2~Cn)Fq<8smuGO@+#>+6xpQ0tkczkdPl>fO8dtzWWe>%MJ>|5YH~0f-eQZmGC2;)IVAFP2QXvSq{y-d=qR znBv*EV5NG7+p!^GihUJaUd_5PTHE5j~mP@tzohzHMNTsl|o$+{Rha{O$uW>u|e zA$~MD(yv>6-ch7lW_j^t;DLulM;&Q{`DNNf2u3L3gcMe2;e{AxsNsejcIe@UAciR7 zh$Qk=V2LL3MA(D12{e?8V!a4sVJyyQ<4_C9IAe~bDM;FBFV}m} zIh1lyu2&aQUy9^XO)3q<5}8~IL|jHy^%)hR045qyQX~l`rcOAeDQ8e&HQJ)^2;#CEc47X*K9M2Cf{7-jw~yv3QwpJrR42;+IpWrtzeT4cDwCrgU~a;AO!kqo7uFdidJW(O5NaFeGzax@vyB1*|)NZ>lml&xg{rBOqB{O zy&-e-@QY0CWbXt?M`$!JcY+ZYzpT?9%A2m50nOsN!YKwspgNEfjZdg&p0sGhTvIts zRK8QC1^Q}T?CFg8tW~lTRcKh&ves#IM#2)B@PsH#|Dg&glsgsnOhr}OT?~IVHKwU- zNk{8d(&ok_ry*@95$ciBmexb6iKR|xfzny56s04s$ul(RQcrnm$s{o&?RkFpKL zuC&A^e#|CZlL&y!gs6%|jyi}7koBt7KZcomOE!15Iqa`cr1dNDTLUMT{^T-F6CxJGOqBT2u-L$7n;yeE_8=50$Pw>=p%`y%!d)R z6Aotx7axMLXe`vBl7=QEj!p-O``cf=6o{T$7Scvc;SHXU@+&}A?2|)WQx{K*nM1xo*emhRKkMnxvFmMJDdgyvHgZ z8YNjWOV$M$>*Zx8EtJQ8(RL?P{U}YuZ`?3RY}fq$G9f7{Y*iIcrI1!xTDT9?oB;$f_XBObSH9_+JI<} zMV|2?-3_2R@spbJ|FT#D)AAIaFw(}964Ns2EwI>=e64~q+cqNx=|u~*W1IY%lZ_)f zdpe3a7u3uQW@%qgHXTazCeS zVpd?lb0Vj7XEH~0mBDny^jK$r7)Eh{5O5NxHF*K^Ocm!~frKdoPls!hDoP zY}W!c9$0^mSA|x1g;Vv>?< zSmQbD6fog-|10ryNlYj(IRQ8pR!_CGKRDA9gb^rxm^miMA4QQmw;uN$5!e{C@mSJ< z7*D4Vfc6j%AudOiTDwDv<-=Fj*J`%0H@JvIxdkK0B}T7@BwKPHEj1A*cT}W^hepUr zZS!_;F)ec?UT1QSi5Fe0)pC*3ZWi`LiN<>Oq-}NgBh`i)!PX#Ic##;Hks4VdV0d{4 zCSYg?|B{qvU;(ymX?Q~em25;3Qp<*4omWyP8G|A!eo5@9H1ezv4@N|s5NM0-yrI*P(sAmSgRq#e-HBFCa^FQ;cQS97csS(7z2 z+Cgb{SXsBEe_97-`{fryH;unUW=q#}rnP~ORZNNTjer&<7pR$Up(!u;FiQxPSM)rA z!9;I{5cH#IVIf8?sbLKVh&vWGb+%|MR+|)oPMKsTMP*f!(uo~rnBTQ2r6MOa@@ZqU z6jR+66BH2=)J9PjsU#a&oz^KLYX5kh>cxinHF|tWMjR!P;Aw0h*M*mdQA1*0ANQSg z*k5zgQbqVLXL(@`7C)Cl9Ro)%a~C#WAtlykLhH63A#rke@*a9)K|Ha8Q?eOaWs2{3 zS4uOFQDq-Ed7xfFQ^mO-B9Ua&fiR`maU6k6fsuZ%(uo!aNEzchDy*Sj1S3Ix3l$v1YQwk2H!|D;1;^DUA|nbi`5_4{?D5fQ=Av1{-Kv32~X< z=%fn(rQ;%wkZQb=QsvL@+znY8>Hyasi!eGbJk#pET+>BjJy4GaV8U zUJW-N3c{Z-RZoP4p~Qt7q5ruP7_m5RqaF#0E|0lG-$WSN`KXW@saTk7l1ESyDS5?) zhS4^9EHhCdIiB2E80zVsTWFGVQX6yTfdyxMGj^u$=btk2nhXYn;(d_}5LxafI5r@)!kwU9+p&R|wPd8?9(2{6&R2ES&toPTLWJf8o zSw4d1Wxh!-U?Oc@9Pn5m^AC`HcsDPmb+B{o#8vH(A|Di;o7$e?X*q@D67flsS4%dc zX^&jlAlrACwX<5OvR+>iDfiYB2C5VBlZdRR96EPYwTD4S=z3TcKeQATRhNA7q~nC|Kn+m-H;(22eo9jG<9zEJ&>cS|HiUE9Uo%hLxx{Dq8+GTHHc& z!K9hTB7n&lq#yR9{$aY77M;3OBnI2M#_}4M*`(UIuv#)MQi_3>Xf^#sx(L03tigPJTx~Z}8;jeQQNgnD+t~g=KNg65UD8P7WbG0va zF}kFyiZys@`!|{WS9G5RXsncTa5=<2hp{ipvk9{`(AXk<;Y(zEEa9jn;qs(Zny}(C zE+dgFE&nkp#sqYF5^zn3Dw2YG2|zzQ)hGGsX(_9+c57jyqm;u(!PW9yQ@b1FfGZYw%Py|<+@`7EtE$>PPPXA6b@g}@Aa%eb7&6LP7h`l%4ifK;1# zn$@X@XGA1L%nGr={Drjx>%oLUZ>y}5GqOLjrJobyzYgSJY=zD3(R`riUX*hD$t_OlXu17!WgdmmlfT+h=GjC(=3n#7*DKLaNKfSQ!0L_QbM;& z#{YM;bvFj)mbXa;}769!VU4tY!3}kl?(p2mFU0xGYIhGZxyf_^(9W*sLB#PD! ze5ws1%)+J1bY0gBEP7Pyl1x*}UpT>F=!JXDhP2GVJh8zf*}XeJOu&K3bGKtW?I_&nMwN2rp4s|QZ}G(qtAeg3JxA@@nK_pTN*R%tCx zSdxgOq%lBB$x+Nmr*b~K&B{NkX}dd!_S94ToYv?1fMQ%cb&S!1=}Y!7N|PIO2LG75 zH8^Xjq0yJ|XA66Q3;WV9jmJ!fjS1n6pJ|jZO^k!;oB1@bgvT)p(_6x-rg|iUkIf&n zgfWa{Fkgbns~pW%{c%ACD^I1ew(_;M^k&^OG3*Ck#z|>D)>^1L&7UG3Hu|*prMtR2 z+;-jJ9$uXhe4ZkuQ5qh$Fj7*+86zeTJu zX940D5#&;hzIm&~t%qddi&SMbJvT;i<`gkWUAf8Dj9=!(B1jd&eJ^Qpw~H&HNimP9 z=r#R}Zc?R_SJhSlql@Z_=5}gNgcZiWI~gp^x-6|N%}p7yi?3}}-J&>lX>_-kK}Sww zwjguNqjfI4It`BCc%=%#ncvNHOnRADYFgh|-je}!o+Ni11J#6q)F9-8x%NciRNQ*$bHT+6I5YY z2E=%ocUr#5U!m%sgcmTAiqhRK)iQxhiX)S0jY+EAh*6m2I-g@Rce29pfbK?Z*0LM! zK>?r|+P#4VSITsLY_T-!oT=pJ(q%Vxn<9dCLZ1ZfMKR zUibfYzn#}U?Q{P_VcYijO?nMnqbpzIW-#Z_Hz}(RjsnX=5drd zu_ac)SA~oycd8R_TI*5<<$w1c_MGbq92ZSKAQ3MHn4 zz0$+aGsBB@A<SN`joRO3Y27x1tx z%Rc}eZ^q)2usbh4OsB9{D${3xjpCxb<82UUAh8gD$5U#t3Mdu8(d$Mw5@=8!SxMCZ z5g-6R2muNPNU#9m0)hh-D&&`tUjT&y1Dp|9aN@xM0y_ri$Pwg1ffy$;EGUuT1dIPI zT@ExD)1}6WA3Y8{cr&HK1T$}9tSGQ&PlhEePDCiMV#AgP3j%G~5#vvtH#^$AIr1ge zs4lr)-KzDa#eZeZo<*Bh?OL{N-M)nzSMFT8b?x58+crR6zJ2}v1)SC**ui48egumZ zabc@o4;v==?cBHVQ zFBlKLc}DHNXKFA15`EV5(Q~U+FWvn=gv?QU9U#B~AQ*tT0Ryy5K#&Af@GAcY8H6xG z2`7wjkO1ikPyz5Z3@Rx7JQT>k4aag0ywgY8m#%~rycJsn z`9&j%gs{dTd0fE97ViSr)Yd*98!K1LaUF45&p!t}bkRpAT`vF5PZu;+(RO8h^ z*uqOO@UhCAih1r#4&33aa%y`_l?!eAJD%j5?!4glZmRsO*4_(wYG3Yd{=Gq*n8qk# zu>&P&LI(6$ggSP#2GvYL8cNFeD1tI;ZO23K@ecp~12gXVL~k-H3yT~Gp`sCKMg$@n zju`Z#01=2FJ~I;0c6211$*4$dGLq2PM5LkZNFNJO5|t`7p4CP0B3;r7QNH6E?kNOt zML7;p0{5t*pa4xkvcJc9F(HfOEEe0?mA!LmV|}X?#ws z%zTVTHZd8-VSp1znZ~n`uYC$dm+@4q04Xcj-Rfb=Lu4WqxkyHijxvHgo$cVpnnyAw zF_^L)=Ui7iT0Q1hxkF>8O0yl(e}n^X9f6UP-sdTzsF5@BMT zkt}f}?qU?B3g$A4Q0;y6>tns-^^%@wgn|c&6r`YdvXt?wI`K)|eXk{G6^|@L>E+~7?GwTkxOI$anP`}`@u0p8c`ef#+AL5 zHq%-U@mFE|1C{Wt>3V9LKs`jpoUq;0j9LZ)6d_lct6FijbFKtySj9S4ux|A_qMV(s(wZ4b%F3&l`6F9_RjXjZC3dKb zYgcZGI_=Q4G#IoAS;ra^cv-|{UAc)EfZzII^@+@Qi{}2rtd&CIDt{ zg8oCP5etjW?J>1K;vtd#7{)IR-o$(ag%X=;bPU@SRnNHaEAnV(M(Z2%G(*)GWw>HiuY8<+b91mvIqj<

d)gLDuU;0#K|NSH*+P(z>n{9Lh|^xR*pJL&P?&*J4e30&|wy zJ{q3Reza^=Y)@x3xH9R`^QQAVbKMw2Ckur;)RfEJ({r7(pUi;>WF2Bn|T zD1`$c8c3Xr5+h~QNj~Z{M^WOVth#Y>15odOYFVx5NJN$c*`}4D>E+Z+PKtaRrbc8b zW#LdD0z0)bQ-*964hA!{MH!}WZc>qLCT=SEBnd?SP}wl6@vmEu>m_i`Wh_?}@uwgr zaV1}6Id2}>-uK2giV2BTLy}jmkpwt$RfkE`oi%l`lhv=<8e%juaj(QPtGg|oZzPXL zBq*hxGEr0HGX|=fpY^SG9QIl^S`p0@Ms4uyTjp%mQ9oh!-xcSF6nB}C+sO3hj-zE> z4Tsrb{D~0L3?db&3@y(?&+XALjp ztdXi*8LDs($Mp|y?%*A+^|JaMP>T27_rC|-UGb{pO|B-!20uQ-XT@(1$8Is7)Um;Z zx0)-7CS6AB0wj5#SB2TvuA95p_8Q24R z7UNywu4zubB4aZ)h4>t;3JYKw3_SCQJadqXDH5E~E}bbfLSwG46A%m;0CE@vd&mzX zN)0iKI{Jwe%fh;T%CO_|x~8k4iaC&=K|2D0ngaW8c`Z4X($(h z0I0#JheC;XGB2JA0CH#m2Jk**3lXvZnZ5s*h%N%D^oyWLYeJKfLezi?U)#6B3^+rc@+0{a~!$!oT4PL&I}Ll7p1^V4n8VkHd|8ytn>=p5r*ZdKr}QE5ekD4sde`W7?qLsl#BJs`f~a{xcY8BD|?8Kb+&a zZaS83`?YbwHURREg$YKzF~>f^kvyv?R7(iE0|}^cl8OR}==v@}LqQ6GrvTAFlbI(* z1DtA;y7}jO|wH99&soe)H@Z7HG>MZ7Ro46bHNsJ5M)eDAb~3) ztR^H>LaMnM!BdE~k%~MD9x02U;EXEia5A*ato{iqVQUwIa5j3t3HWOWCRC+kSte;B zmkog#hkKmYs}wj}H}2_($5=x~7f*fgcINjt(n|O0@M|xKraWgjyXCl|0teRvQNL7%3>VQK+@F9 z(bD^zI0}W;SY0Fp#Yuj1&rDRlp+vYzD$wNfu~~t>n0e2WS|xsx)!rke$3RcPlbV72 z#+hTSVe1Z-Nt}8rkD$OZsrrrbn4Dx{9wlQm_`yYWAv|m{PXAfZ;9Jp5WhaavwFE(l zAQ})S85$y~>6sx}5G46hDq$cBoIoSO(hpj*x&%#6+{1wVAko|{bnGaD+OBmxryA)& zJ$ss;Ng*9XHFwn+AKf#HvKSm}D349d2fUe=U;uY0PO4%bMJZ3O^o>Ue4pZz%;~bmP z)1!*`wjo2C_@aO$7?<;mFSofE)3UOn0NU%+7ff}G{<12^u|mS4qvar+MS-K6V~pd0 zDnv9s_>|9nGo#}>%A{0KrA^mkHUHbQH5GbG(En7`sFbUid8>Z|P=c$Z8Dk8Ev(I6b z(DOMS3dPE+#EQd;9=k->8dX`Nuv+6HE1ay}Vx_-bEjP-&aat%SF(;!bx{L8aKNGJOypa`LyAkr&kFh`}wGK8ySPnX~g8(Lv zThw)ZMMWe>>rhx1`BIG0GyxIQ8d_c!T3&dRO&>BE2>c=03exKeuLR&&f}m4V<6h>- zD0>u{y)mwZObYdxAm{0#q=+*2uoE|_li?7PGxJ6+3K2*3NX}KIM4dcd0xp7~qCO?t zG2=Nf>AuoqCYU`rvEa~~+y6wQ6imHcNfg#7PWc z0BtK%AQ{(F2s?L!rpnsr}bUpbhLrdTtNBMfz-In<&N~grm2KK&PQOc;b(?s+%ZaA zrBAz+F}VFtZ1!5YEx5N8-N1C!Qw^+WHpG_-U0*z|^y||ZwMn$>N^C5Y3@a_c87i1; zFqM@b5|u>|nNfO~I)jyB(LrYnj*tgy683ti2CS|JOGfGXv=%B%1vJKq-mUWGWDX== z{4>cq4%g1yr*0}(VC2#X3A-+BQz7vg0U2WkT1SYgSL`(!+PW?}ouP{WB8j>#AW9N~ z$UCXflT26uF+;DE1wZ7(xHb})o{$RQ=r8<@j_`3ptgx4x=vnp%4mN7N^HE?~S~kvY zygzZzn}oJmEz8{S;OK?CaqH$ zl*y8L?%@uFLJe{bj6fvXlN}s!SIa;Wko}QFra==Dnu|%HdBm6!Y9KSdkV8x35mDc8 zg3`!66#46D@*5b0ZPs$UVh8$C6&nFConx|<;vOlj67Qr8_)kp(nT<1_B< zn7U~Ip@R}?f~jt$CZ)1ami^=_Bwcx#>Izb_2fG#A%$siUPs z3k_>M#Q%t)fGHpD9ph9~N*Z3Qs9D91E5gpzurAP?ROZ5N%A~~4yf*P;h0n32NfCcf z0To*o@7f!qTM{oK)3IAYIdNGfSk4UC^A&BeHc_jUT&hEu@esixM{NT0pv{`BBDXsF z38CaM8EC0O)1^mQn-U%j-x+kM#AwC?!PttjLF;BGXEbBVRhcx8EHj}|nMkwNQrxSW zHwA_6+8xs_O+h2o^CWS{J-;q_yw^WRD86iMLi0L#*q>sw(j_nS(1 zL;qC_-sY@=8x zfjsM|He~7SqI%+AbTe!X;KYF&|1ckZ;tDBBOIv&;30WCBHQ;s;C$(YC-X_N?SPWXx9&Z`vuC!Njpn@mOQ@fcmwQjw$SDcwm zGImYpvJs4e5ZFK_Tx&B0 z=D5o+iT?IvZjq(f9v3Ez`5o%^z1i?OFTZhx1d?!{@d(cB{gZzc6K?^$mhWWhyVff2@H@Rz=DGV3I>3{PauSX4m<-a6u7V= z#ex_uW@OlrKAW4#3`B5dxiZMeL{Acqf&YU`T^6csJC(ximhY~Gn z^eF$*q)L}E6)Msr%9l^4a+Fz;CDf~0T~^I1bz{q!Tcrwg%CITgv})I~ZR^%1NQV3t z1ejYe0bPM~@j~<{@S)$pbrA+Q^Oxa4y@Cl725fi%V1az&N~Q}q;@yFZ2j@ln7jx%_ zll{JBx;ZiClmk{@9FRaCgx3xsEI7jt!o>sw7P!91do}FU6m{3ur+a|D)ew>|hj2#U z8QiR=Q?G8_dO^bm5q6$kg>nCu z*@TcW=GbS7ZMRT|3`Ecuh!H(xRa|^&XCr%b8D?BVeGPO_L>vamV@4e!G}mpBEymtL zkmYq*kSd+W7-JiyreaF%bs17rl$rG9No2MZ*O_8k6;_#E29{=grO7!bopsuIC!RfJ z1(r#Cl36C7Fm?4Oj$saZ=u{pe8kI^v)s!cskxDx0P+lhHmt}%Y6r_tgnn+NPk&#GP zLInY~mv@nMCYg)gh3e6zCtf-cXqP@zBB$J8xoKIH>gicQbnzPBe5@7bP(mm{;LsVo z%~zak0m7(TvA89O+kU_8N1cH66~a6aCYXZB zr$Bx8{gkU_!hN8_Vpkq?Cc2_vQtteK6BMNt$apLso-jPc_`BCu(tFo)3%F7U! zi^WQwu0uAqE2~ty7}@5Sftu;&T}xz;yI*d8*5n?oYS_Bp%ETe>Bk%v~FhvypC;V~< z3rDbY#rfM>!vNz}F!J?XH;A~lvpZ$%ABHZNcHAZAZe&#YHT%1NmhX4~_)2fx!K~#c z@M`T(C!Mvr>D$`z0~YsuhXrH1Z-<%%oq`2#HWOe0JZ|G4ibYL3a0=b2Kx8DQF~l$m zSyEm+MWF;#>@{adK(}O+vsal!Lr6LZL9`<@g2W7VLK~5kqNbsWfag;aDiY%~<*Fcc zk0TM$4r+!bknV{nhiCE=+S)@lzjf(vjT21bzA_fYvF&Y;lMm&xsKqTxj%!Wiis1aj zII=Bnia88ipvH2xFSZFTj^iR6<5;`!IO!poie1=tXrq;J&Q|{nA)n3QrI%Q#YF#b4 zRIE&AyYC66M6T3)XHwL;*$O2Z$ z-Mof_FuD!KI#vKxP7Gp6O9*VB$F*Owh8zbw!#T})Fmn!o9PNZhJ6GdQa=?=u^spyA z;V}Vy@^b?I{AVU8Kmm6YU`B(ynS%_IGCQIWAwf$BsvJnQpv??t7wpSKW}+k&sf$)v zd*C1U6d1Z>4_8xCQ`u5V7BGTTrGgui+t#KyCJs)DfK&g}73HYYo$?edYlK@G50%8i zp-H4PHR=;<8dNqG^^HA!Dpc`gD$cpmRj32j<`PpgmwoT5Kq5~aGo=@PT#1oA{OVX4 zBQFr0$~%^cNFpmE$Sry#d{0wNZ$y$%BPpzX8iEZctJ$019Isi#ESUcSMn8cq7M0cs z7i-1^pTw{=SAe7{mS}jo%WbBP-MOq6V~IZXIW}Re;f*k_bDso3hc(ga4Z#A&$=Eyy zVe$)}f_Af-Y8qz&6G#F);|IU$wC|k1^hiMqpj_ofpa4lI01J}vTnm3#kT*AcinEg~!=%+FLO-x_JAnoClW zhCsBC6uHo9CfZF87sI9>9poVjl`qT&1J}F|&TNDVZK7EEQp2ruY+vMXO@*pN1qUXo zCO$EW|AFD6>Jo524dbHNh&Y(Wjj1n=i=$Gk;~g7CNGgq)G zA{k2~O{9`pRVvQb6|?FZ-7X_Z8i&SiI#R1m0SS1#D*a|I3ge_R@%Nm6-3LI1WzE6z z)2-67hI#|AGxrXW-s_uzvd0rQ7{D z7`B4r$^0}V0skt@ZoC=WxipAi_(%c+GLir42Dn4jtY!cM+yU!ZM*!9lxHYc*LIHeG zAfOcRCk5ca&wSEz0KpBAJ|KfDDjVb=fDTQc>8m81?Z}Qzs|W}k`odCa7HUOal0z{H zHI%&UzI!2vjgr(-v5izPHDjc!`o`25s}a#JEn|y2OcOzw(Zflk8&R{&bWcXs2Ws6m! z@@1uyF9pZNN6iOCGX+?q7cW&hn!7s>~xqTOgRI{Vf*oqZir*_!2&0m4-#{Q zD0UloK$iv3RW1vfi|fG;e%3NFJOlpRS&%ozT_laUpiS7{@i>^P`0a~!zN#UP)@6?z z;jdjlD7J+FskamA^aN3ILm?V!Gbsbo3MUlAOlhcyh(6-&8;8@@$N1j-4)NlC&u=kW zm^2kX|3325V&lMV7>zaV)4oT%(u8ki_lZtE_OmY^&}1guYHe$&rhH{oeeHTlz8sgJ zy**OFsvvu|D%brSNGjK?bMcL%xhCo5geb|dwLz9^Bu4?w#=#`Z{}Irnnapkl$3?7} zbU58`sn*j`N7Qu(Wqn7hWMJv6$f~G{^{k(J5E^_GpYbVLeL!H@EnROs$G$`deqT zN0c1NiAWic;m~6k1gQKBLTq6TJ%!BBN)~b6LX8^*#;oYj z|6zz96^dBB>>=cpfkc$Q) z2KY1*%8AvJR2hj?5f#yn39irmOh=j72mBmIgv4V69*2S0j{_nY(0!B(E`V|H&uv_b zZWN6D495cXkI3+YZJ>nEq{-Kf&gzhrsiX#xSm9kU6l4%s$dyisFj&-71X*EY|L0{# ze}S83g&tq65(Z`Dgk_`@wva@r+|Z9bf)nC{5b% z9FuM(&qWZ>zwqBRZH$>Ugvnr1&ne4Oc#{xi#ij_&L1@R(>;;oBp~?si|F004D;^Eh zgrU#3R z%m)kV1_J`n!IYA~n38Fo67@h9)+JK>g;bNh+y`c3XQ<`;UCxAxT~U_LOrjEy3Ky0Z zm%->3q+#i37LZ~=hY$WGMX;c2JP33Mro%YPnNrMm?&gO0-!R^o|0{7HHnND5-5039 zPC}yD(5E3*-||gg6}>7`GN`QT%~MLBd$O3BxD-(N8<|K-tj;R1 zw#B++8+w3>M0#L_atA3Qnuda9yD0`Efsn^V##VL5lzHE>^;BR~D?9R*w;BgDVN*ZO z#lL(M{|Vrj#a1Yt)(c99Y{*Yzt)ubKnJS&nx@^wvpy#%N;0Ly%hf3*tkYJTIrlVP! za_nepnU)OBo%0ye^Zek@eNu!}OdQ4*IQgGm#^J<>A+?R$|9qTA<)KH3c+|B{CtPJ6 zE5eK6l;j>wXLmx48A|I!W+$hFU(D(ZvSh?W;tY72;e)YWM*K>Ml&8!55z!LjuG(0> zovKa(qVNHqshSVU+^4IW>JqU{ux72n#b+SA`0nBCEYRq%is>V0sdbDwCKF&u|sX zZ!8a-VJUSirga1t^+cUlilb)290mpwE`s9)daiq*O|uH3XWAR&SOk6O%eG7|{3I+f z2^|FThkqDWZW3m~%+TDSR)_E-j9yCtoehV;slv?A{|t4cNL(mLbjFVmM)#zKx4|Sf3)R3?YNelrFKM(~shGd7D zB_AG|+K8$ZL1lr;=e!tEA`+Do2`B(hEu5IG0?$dC(3>UZ>Wlqv^s#DEF0ciom}I2l z`wa_)b|6Y7Yv_5!WYXc*WgB1+66sp&l>i2OP1#e`1QZ#|hENN#@N7mz$GGZXx&~kZ z8Vd`?pwazH{Iu4?7LT^2(B0f$u~$UdPJJV0ht5a!Jd6-pZ`uw~_3%nFzZ4>_RBTdkm__!WgRN@6yug z*IHP63Zf!HWxXBM)#7RtrO5^V$MRCF7}T!nl-6g%@t!R2^1!(WNX^bw1rjV>N~QqX zDl!J>h*dJbkc2+o3QMoXVd3@hl$&^2%k&m8Cdi5&F7FnQGGU8I2RU5G9t%l!q2wY4>kdD!j!3NElcv? zt}r4GZR91J-q1fvH1H+}9xJaR7Hww?m^fn3{84INzTa70GR^g}a5~PU`Q40X6PbrF%l6)j}V`S>MxMVjl`{0`oFd z`$Pm=C4QE&t74+SMYUG{r&vyc;#)3f1`?~EJ>C;%@F&uoB;6rczHckK3xG=P(CHPm zfs=T$>BTq*8V6)^I2|?OXmYq95zh}h;s?R5Mg<~L=LQQgTh6eO=wibn6K|%uh)?R4 z;{#P?XtwTf$*VL5>HLUkaQq`Q0?%~38Jh}b(lir!Ds+=X$6q4Lrx{iai;QhLh(+BO z!13OZv4?>@*rw$1$zF~Zf<(G0bAyfCA+Mo*`B;5@WJos@mpo)hZ$}JyCrz)fNBwG3 z1a~M)qLK_UUSojTLYHnk%?18wFtgrb77`q`U1)=R-Nap3NuX5($qrPtl zYaW3uxF2)GWYCutzO>GIuZc`~ay~cwDyLKt%LuNRPXe{nYS;v$cYao_c@GtvtZa>i za#i!Ve#f%c8g)h+=v7YhqEC8+1KBucsP`QyGPiAJa%x%!#$$-}4&kYZzVL>{HMxCK zZz3Q9GRSQI=-@HI1`aD0^xzLUM-!t3IW!73D%qK1s%;-dr@i(ov|`=6XnHV<7NP5S zIX;SypAvw$Py9%lv`5c!@JI1P&$kH3|5=hWP0X5Yj14{qZ$y(Z9`A;)_H9Oa%J3$g zwvbKF%*yx|cb-VPInl zo>AlK1TXv39-I?F98!<>PXXL}Q@X4!-`FCRD(f4_pL{Q0+zYi|{Be|M)OZI%Gunoa zhBEj}7m=5b`Nx}j3}0=Vc1T{r2E)uyHNCpt`5;~xmSBmR?iN$iJDufHu492?EWM0c z^>@4fQMSG!X|pzxhIZ~AdqsUyhJ08lkXNqL7qKzHR<<(?(d8~~;YPU!WScf1!z}NG z-~G>DG`sudU&Zuh5a-J)Gj*@hy|Y`&8A*JFB-5~U%ivB8xw)Gw?*y$c(_ps?wK+xH z%^SgIPzT(f_j&8-rx%sCR3?=zC%VOd7$~nYvp*2Z`zOYWJq44++*I({#{O16P4;Zz z+H%HvVwT%J@sv5+JF5!3If*#hcveT15?S-m(Dh8%wuO`0r%_XL@RkDxN46i&rJ*se z|4#%){dK5#y;d;^``Nvh4~d2#>71W?rj?BpEhqTAo$|%*=_?=0Cf4@#fXLmv3La0r~*UCc6-RCine%4DkU!JJe0Xu_&!uTrPK_9_U)Qf;!)_g$c5U0Yap%^(`|jh% zhE-n%Eu1TFYzxG-5>eK+#secu0GwpT3)U8s#x_GMfkC&1EvYw0m zYJ#cmy{DfYoqq4kY7>&aZj<3h_f`i`Nx%gna;c$@Msg`4ffktPqyf%Q$)WxdO0b}T zERv|gff7u}paJypP(%?2Nl1a|Py|Z6xq?E>sh?Ja(I@L%ypKlpa6Iom)?ReXx6L>k zttZNW>TSdk&q(OPfkO1qp#eTh$RdFlvdEAc*il z2w-w=p^p4p5J8w;I*UK7+&hmjJw>Z=KeT{yil?`{J8aNDuglXZu5`>3zMBY3i>tK6 z;x3|}tjiNA_-slqJ(3z?39kB>!!bzIIveh?(|l|dNX>-PPgT@tg;iD(;3Uh8NFiMv zE;!=U3r#Rmi7o%u*kgISwa;9c#nmTdp^aABX_KYa+Sma7j;gCvq*1G?X6&v-Q`PHH zFuCB`c1BDQ`)9sut7R=e%U;^4teFUlNu-bvY_Oy&%Nz+#h#+k6p$^eBXe5R{#Au*` zmIU$0iH~GW-0p59m&KgAGk0A1Oug98n_}c~T&7|~RbGvQZ1q{Y0#JbBBa@U8#F|AM ziOeQ7Ofty^1N;z8oM{)B7EJz>w%b|CP;lR&aAaF~mg;~qDpEhZF-QoR4% zL49SoT1hl>s+s5$ANoXH{l=E>8t6wqmpCXY)T1CZ=}A!JDpS{GI}7O3>CRAaQ<&dCtvu_kuZa7e^aW9)O)5QWl!4f|w5zSoeTF|$flav+yr z;t)u3?;}O+-~!`B$)6$?CP76cO{`M2OK~N#EK$@;^9aLrkt8j!yKG$2(jE9z7CSuQ z%B7;xqOph&Cn+I{*-T5$loGUfNZb`f-I~ONj&5ZattdmuxY65|F`~Q8WuO1Ns72w; zmbu1KB5_OFT11nK*bjO>{j;r(wFHFDMJ6W>KjNIa% zstStRUBYZjBWjWWfypFG7SLyU?Ou}}U>4+EbsJ&813BB~oKkHlt=BZ4NOe zf_;#vXqdHcf<-4%iRq0x>&J_gMW&GfnNX>u+Gq1>roHe zWCvOn$whax6Dwk3S~`;ktCY@KOQl*O6ZZt{gJtUEmKwt80GY%fK%*02i5Wgmf)CXL zJ`yR{J2L#8EyAn(iC4*G(~Sjh!tALUD#_KOSMix#!{$s}sY)?2d6-ExL*SRDM9ijY zM3WNw86-*Vl97z0A!nFVzrGEVrTs_ISo*05&x7u?RrYQ0v6p!p{1dN)ZGBwX)O3FB zou#;YhI4t_?#OGqP4NWNB(Wh$0}bZm0!z#sbzhqiYGylY)VRw6p>C0zaUs=Np%V|Y zvGJU6Kr>F{B{%uWT|?frX$w;ri?<)k3F%#@=x!^d3a>-HMoPTb3Txo>=GB@v!PFRB zAJSyl(3Y7nN-88k$&B_e1=1mjbpa#Hvn*eXQ0Q1t zV^^_@Ohk47MN~7Q9kYN<51&jAK$6Vcn&Nx7q{aW zw`fB*irb9ymZGnxv+H3s@nNI?eB_HE{_&B2e2ejpk6xt?p+NThM17rpPdPMX#Kqj5 zpP9WSxyWU^G(x>1h!>h)ou+ zBSc8u=A@|L3wb1uhFEJ)pzl1=jlA-0?CxzUmIAiK#>&#eURH>oSZF;kCQ&8}^5nt-ap#DzR+Ew>;8?TSt7isHhUh^;{Gi|}K>tm=R?DS57!=8_SS$B<&vcaiHvx4A}*{_@;%50$tEbM`+}<04}UzL_C^nkjnA)ylI?>u_N50 zXo}_{q~!ZjY_6F9gkWywBRi=;Bv1kwvA_mk5a^*9^DpecM*sfLRK!DmSfoeHCRSJq z%g8Fu0>|SnjC9^^Vp0(n@v7PSD;8OeK?v(Y3L;^&r%WPf0})S6Z1IDh$9c{}BBLhV z^r?}qOf_;Rwl0k>5G8ibYj3o}r5KGT_-Wt{qbSZ}2j^lf$7L-NU>|iu?J!KF?t=;2 zj-+M}aoUmLgex#3ZVD;xFNKA4vS_5x%P##UA00C?B{K|9CJ4JOv{O#>d`k@rJP^o(kIxVw!^HscI%+;%?cFBq(RbT`n<2+;An?Frefj>);}Y zt_#7!=3}z|Mc2Actyo4VJaI06q@lkk-DcM zo`fv+P zY%=dMI_J>}BS#ATk}!KuStdt?Vo#&au~#B=_a5}2F3yQq13`%eGC_1iNmS?b=y$%Q ze$qp;YBKC(gqAX9x(4aFq%dEor!-3wKJ6|_xa~Tz4cZE%LXb%*MMNT&MC#V>0bNa& zJ`Y?huVkt$GtY`foFdXttI8X(v1}q^9am;3KhC_GQB|LsmQCD<@#KPUC zBen`oDCqbtlFO@GW^;Knc zR#j((p3h2o6Q|IwEnrd%<#0$Ut@i5By_6!$Kr52S;>Kv~B@8b7a1kjbh&2Uqu&61R zEV4Bj(L-SE>d3|t6V1E)jaN6uQ?jK+%L=VdG?vr~jtFo&#)cg;R8w8%WzKXFhe^N+ z0%+dP6-8`&sH8RXP$M8|g8&h~=u4^oXy> zV#|WF<)qJQI;%yvBdll@ImT4@*eo3{)IlANa@KJPSr%0tr!e0!Wh=DyaBm$0!&ON% zR!jD0fwpth|Bq?XHOP)l( z`c4sp)HX$oY(>N*2LLzOwOhHa>;AD4vx5!0W?mIdK6uS3nk-0WEmzUCOchWZU1mDv zD#WbSOFq(=R%}RhM%o~xXZr3=0b~&A>u6e{{PLtrOb~{e4*XcqQ3g&crDES+D>C3? zbn%Ga5DW(&HA`!9DJCmulj9SA%eWR49yzKo`Lbqb7x&z89$}Vd*=_-~kO5@^gJ=Km z(Pv>dd6josz9UltN|u;Y<=E$rsB1-lWt}M_sS!|CzSuxbihjF zd~maNZmvqD7{Q3Mb?)f9s?2m%5@YKJQDJ8Qf$jee@Xj{X&FI!lwbN!M@UArE!&uGJ zI3mC-Bxr7NsX|bMwkJv|u!7!Zj4Lv+M94-bPwJs%HOSs8y zawaNkQk~d2GR#@F5SQW=ap>^%j2VZ8I2~y(3v-VigIH6^QK0_jcw-oed-ag&O!jUgmNfM!CPYSjd6e9#t1 z4B17?jzwe=E?TKc@#veWBY$lWtx9p@UQ{M!ci1EvIvKEDBYAx4ZgKymBw{8cNV&6; zvcyZ|93Dnq_!Lb#{y=8t{J=j!a&OI7O!_f|dljeX?Ol~Kt$ z^q4&q%&Fa0F~tgYCK~n@r=bNm*#ry`MhI460Nujpfs!O znzLOGc`tJd?l02IuHOa2$G{5;ZKC9Iv7!0i{Vbi~q74jEW>!FZR4glfC~c(x?9PX$kQ=w^r)5WO4N_ zz$QkctL{p1tCkm%Toep{pwvcK1W0^4F=(WAZgONwQAf;mkvXw{(Cd-!iXs6(eZr1f*f0t&mmk69uP^I|_>!?Li3qg+D~kf>Rfifs4xLbP4H@yvrg+qJzfhAmwVvZbtj zthLywACimP+?cv|N;J^mi0Z=@L=o%z6@|uTXwz)Tr@(M?8F7TG{5LUZi8|~p2+?Vu z#s&ZIXEO->tnyDe(GKl~tq5XJD5KsJP@` zysw(7*6(zpO$Jt9?!``0h%ja%wpe(lVf|1v7W=ewDdHyJ+^$(#vNzZocn6Kr|z@@$!&58 z0=DNndd7{NExo@`4AbphTRjB8jZ@32? z>HqJ_!#x1V0nZ$)9nTrLsatS*r3s0oSZvtHnb+!h zfBLUp;U{&4hELCgKDx_27s1Ir%TJqLoEA@ik=R%xJA1xq#$Z?sd}uRU`}vjT2RxkA zpWanc%>`UH6T#OdWB&jm0lxqN1`;e7Ai=?f5DY?CNU&jmhY%MIsCbYf#*G*+2Jq;S zV91Rl|CtmyAf?Kc10W30$1>%a0cTvc)HkzbO_(%!E=UOh=Ky{-eEqYsx5D`aEpPqgSYN(=)$|rTCo(h~mCx$5J zZJnaJ9RH!(xz?ye4YeibSn{2B>r?Xm8emdh4Rz~}RIT(8O+l>$l~PZ}C)2W8@ks3e zgREqrjVGR{s)Ga}IH9*^RutiLEFz>JT_8e-Vnr!tT{!rKKiquJohZ9qLB_-bZbkSs$8IA zk_PBOv%QLQ)>?1Pb=O{h{TN>8I)^pbxe?lLzqUQ7^2cKS8kT$T&D|2)a_N*FPe(D8 zlK;d^@#N%7KssD)!G#Y%PfGM$C?cHgihYoX1lg6^sv1!=@4Xv3_z;N{fm`ml80{rF zLnvO(IY;!`=<2_j?wjLD0;{wv;VgB`w@mc`H52ZAJ8TnrE;aeLSWtDm08bFe^<7j% z-ThsFV%}G0%+rJQQ1mdm_+aN~%ILP+F79P^`Q{(jQPQm+${nM$O1d)ptYI27&!0&a z>Csfv>X~&n&R-)wmCgozwUtJ02x$K*I})OL(G^WZ8s;TAAGmFIKU@ z9m`5S`q%}%)UkeU;^T38(kgcE=p0AdG>fK>>(tO4=ki16T>e61}Zf9qaUUu z^}3@mGErBY+7yNMs7jViITh^A{yrJILyB^gq%5T=nS;a$of0zcqu~3-k{-5QFOMHF z58LdMEW2&+40p`QmBP5YB*lj(L+Xj)j(NDmU26c@vQQG;C9WC{@gO=BT?JI!TocWr zi!ZKiad)@2xI=M=LV*@{w=S@_!{YAluEpJhk5nORS%6sbNa{rC3ESP9y%=HG6+d*TjqUu5+6gU)hAaGR+w)^X;O} zWbht_D_Rq6UBf@fR880vj!-2UB7WsnEL?M1KWPBQ==m^OTlJbhcq{DU?7?L>!6zjo z3-pf@SnTKTE*b1dPCHsSj_mUOWv4AVB4LY}?&rn)q@?N&-4*c&L6d~+1J%OEf_*hng<26Ah*04%ix98UaLuD%; zh9 z74R+c-_86nvsMa$=oA1nA5 zs-7k!8|#_CZH&aMyVgw>2RT?in3BFMxJ{A!KTbqj>%o0(Uzkpk6!_8T3AIMJYlf$# zZiBj?sC*H?yQ@gDV7Wbw%6iqD!hgnC%Z{Ynt|P4#7v@BaRnS_?OEN3HJoFFdV~WGY z5zztpp-lHM-|EG(vQ`&o;a5FGLLwS$4Sw}=Dr>7HN&I$3`u5f~5)TvgBVEe{_+B}9 zjo&DEnhPC;@=g-a4UiU-Q_(hcH4C9xZ(f86`LQg)+2R9KMyBVj6Yt&Xv2C*?gOWw_ zqW=p!-SB*jX%HRnx_1ppWr)SlsyUe&nJwe_*(cO49&W0wHV)fql`TKqe$7;vxy_+P zgdLzNIv(`$E>!wC*3*ycyh3kNG@j6X>QJsMPp>|{N>=Nf9Ju*@x!?B2m$~F%n(vcKedh*7i*!iN-%i}-f40j3QW(TLqZ}qO14hY z_vJPvkuN)&i8U=ly>y5vhDgj-LLwOW4$xiLi_Be^W!?0BaEjzUl<6-J>VIBSPuBN6 z2<7$z(UTE<_Gz;9o*=tMr@L1a^NaAkDi)7Srwxm;f_gc1lKN38N?2G&m@@g3r0ba< zgwVlx--7fGf9fJzNi=rtCQz9bK*TvVu!<-FF^Vu+Qte(ZWmX{@9O1Ce3}t^A8&WhF z=b<*TKRl-I!Le>Flx8rHQ7?>_mGU!_?nb)w$kMxqm6uuQA2HBDrM{$I0+N3%?KwJR z@`PW#!t~+Ll7JhCZWME0%G3WY^DU6|#X43`x^YEMcL}+t37Me3ly|GYc8%$zL&ucw zv6Yv(w=gZ0&j9m{d1}HdLs0e^ZncjZ^A=Eb2T2+*zKf|+GBt>w8CwekVB0RH!`~?+%T&i6hE7lvMG5;z` zT9z?k%L&Hn2x1*l&XGO4U%QL~gYGaZVjEkD$mL)UVX^PN5YtIt0`wP4(iS9^%%|FE z7Rl;fdU9U!@3TA&HsqqzVH)b-#8^e1w4+3#;s9~d47W8qNq&jt4~)Y@aAQkG!zJR` zB|nxU4(4@8o?QWjou_;x>$~BoCt_=X9~Ljz0+q&LELgD_U19fCg*Y(|B@En1-Fv8b zrQmq^I!UuAnVcXdb+Lr#_DJl-T&zX*05q|~()Yfd zc{7wn9lK;Mz9qVXhPE482r)y^vGh3p5aw#4(Ghdn3JGG5QnPhN={@K17H1K{Omd}6 z$Uvb{wh&QuL99>V(8quVD=U;327jUecvZ~52`b~Ma`0`|JMvOUjB@XH<;IcMlrS^i zj1OkgBADSuz59|ORAhKM5j4ln=&4NUtS+t?X?44{NnyUI3XxX(g!8*4K@~p7*EGSH zpPy`8e>NwUCr73dGJMPO?M{2ob}jhl(NvNu$sH@9srp?QD%bS^XDYf35*yv;c(J^X z-KuiML0msBQKX{jI80JJ)xxrTz^67Ty71?I%_4i<7kTdOJx9_}3-U-O9ax;el5mk; zD(kP8wBJ%YZ(dC|bE0_EFs$deeg9~BsAtzn7bemW+d-anlMqQ)+4uRJk*1GP}7-ES&w=XoFN{tAZ5i zS!U_R_+7YWe z7+Ul-{ZC8+VZwB0Xa-Re)g#8*MF$#@9)?$~dTrtgJx1nF(3Y^+z``MeL?dlU{h}7z z=DKW3-cy}+uYf_XSR|urX9i9)bex|h1x8rYoGBPEgxPAv7ZaikZ#?3pi+V}Hz?tML z6KW*6f5<>e&7~7lyiVVMQzQ!?=5n=E@2^Q;UTK5+eEz^;i%o6o*OC7DQ{$6+B2P*h z*@|z?QwfcmoxHeTGkb*MhWGXZLncl?jMCg%V_Mh z>zZMKuQ8*{o=8C4ldT5^N9mGa87c0us&5)r-Ks$jQHA?_nYQ$D(jg1DsaOk>Fs-yC zF)BN04~LYHlHp|br4si}R_eu)b5{3~ec*neQm5Q4$?tyef}T#KP48M$T&OZ`QVWXf^s&ZqSGH-fe6B zzjrhqO+d65ZGefNDqCmS+05Z^zr~4iWKRF2FqK%E=jHE7mNu&Sv)NC?PEslyVaztA zfBez3+g$(4Wf9tg= z@?n7~YhifNN&qJ|7Ar-yvwwpI_J<}Vt!sfY$N^49__|b%(NBrCZgD<$j$xLKxi0f< z?j(DkWX)hH;?0B`oMau_qEA=7iZEN2&A-|R!}*PWTK&e_ZL z-OJ(8b#`29T79b`25a9@&1ODo2)+<3`>?7{-l7pi<#GNNme;~jn|PEm6*0R`6|@#_ z=2)Ow@s?^T17%&oagjHWFn)GJn`Hs}7q{E?Mv3~jyt*&(c5aCWYafGFH+ntF@HT@! z^nEs!590jRG&_Q3uwEf+bl0H^2( zUlhEyV|U!So&E3c`HY$UWIoaIPS;Al>FR}}>GJu1>SR04=f3lFU$OfU6|B3&_?tBk zR4Dzk!%qEzPFrO7lq=ha)G-aOVwH+kWm5b-j{ZgY%Dr*D*#e^V-;%SUhKoFNdkkE2 z^hWY8<6(#(N74L!+58ANuQ5Q-A)9UAJTC82)&OM^H=G za7~WC@WffmUvu&$W1sahr3mHcgSv@c3wuQWwd=Q=xBroco^NLT&GW#&HTM6HsJav1 znPFokS90{CQRYUm;ZW!EPr-{?jD$OhBWryZCG*5%o}MHgztW-TU{<86CTs&FT5|8PHr0CY+s;zJ{2}R)x28LRbco%y*q%u zSMUzjXMRiY>I{~AT=Y(=b^hrKj~Dayfmp!ri=KRc<)!?#G_C*l+}nS>^Z)K# zkl66Gru3_r|NiSIfFRNE84VW@(l9Ji?&oZh-Qf@rqeiYM^n-jf8LDk>r53|T7`b!; zqfv@Akjkc4Z@sZ}sG7-VGgoS~e59T$=5=8mo$~!%A$Jf4lktk}bUbCMzixIbn@)vd znMRrM*ORX~vH|9$6N^=A2^4f~*#|Lt-wj*rzSi4puAZB8x*g8Fj!Abm?(xBk_*=a~ zzpq>Fk4|8om2qV`OxxvcwngFiVU&cR0w%d}Z8MW4&frN|oNQ zlc~SpN;L0uJ-1=jFQ=_8ugfv}oO>69zpB9B-{PXO|Ba{lIG?w6S!cC)sp{qhK6;&P zw7F#1=SjFMH5LW4{I?MBm~Z_Isi|hxdi&%1;(W#X|AN&Y@9#d%a7c(bf_YDt;|{icb`gzm1>W0 zXa&}@GUv4NtMU%dT&eO+3#Q8O>z$=k9yO|ZG>go`Jv77#V_N1|E=L4fdJ+5X>Uj4g zo|-b4D;^!s_3Z+xtshFNg>b@jJ+)Oh+DIBy-Bbj1c7?-jRnH|4J$1EJh3-};aHWly$Ar^tkm{YQ2J1ADu_r^vsB#Ao_R#+g^qr)Je%&t$wPt8xn?~$BY3PHr~ee zHBL*$j&>fc^+)hPE#uv0QXfAx#m8n6C--8fyBzc=&cVIuHFkgLW zyMGV8ydiH1p=|RS56n3{X@XLFKUhXFq(A7`+1(1SMGLI@dRZmNpJuYaiMU-Y*Uv|X|xtO-uWIMD|ih1fK_?QUa_Zcb>d!_3g=fny>9cTFl3x9kYqVQx_ zS@M!itpT|@zmlKX5@`|J?0b-aIZ4D z(Ql=vvQGbauILy9d99lG{PmhX4-fKQP_8iZaB}(7CXR}E6$GFMlL!0m#ftp%JxDbO z_B+b)`R8|Bk`e5GQq%Fz|Lkz}ujeoPt$zVmcJsl3*V7`%z}sbm&q2R7eUL#9`x&2u zpH4fF!GCVnJ|oYy$K1v6NXAuFE4sc4y zoH1mMGN4Uc#beF^Hd?pZ=P>8sklQymIu|!bS=ZL)QMY3?s!})S5N8EHT9rAO<|10*~pp=<2mYJiBiwn5CKZ<)+ z^4W}M8|ro-b-9qs9mr)agfABow*!gGh0N_h=5isBTr1cPtgKHaZb!vo)WIV_TpI#W zfs~cit=`jJYG39)ug|JdUQ%j5TcKVPLxr4I~;gCR%v_g30*;t+_#Jso@%77)-@w{r>SgFEoS;d~Abd=4J>We)dN zx%YHzfE63?ye;#%UoE>Y%e{Z@9?4LOm))IwkBID<_o04 zuwSUOR!U_mrLvRS!7Z07G)q+SRNAV)))==uP zUH5?CclmC7+n(-d`0oo;_UMILr|V8`#2vGFuNTjLjms<5RqD>>?a9nCcMF%WTBN|Xg^3RPMPr4jjPJd+ubX@*dNrzJLEkM+|aV(Xi zfX46Y$KgWBCEQHngX1c(i9_VMmjBw(PD$8%f_MFYe0PSXak%P(D@T4^?Z17UNZ$14 z_ZX9T1HRi^c_g3o_{-pqMMXr&eOV+z9hKWq#|vM9wMeH%NOU(m5~P&pMf240pLc_u z?V{_vrD%3Ib9rWVD0{ZGMJ(fk}SVM=j2LJLY&N?u^51 ztaLxdK)a>MBe>kJjrW<Rn&fB=iaLx)!-7~j4q=F6YJ1L5?xH?3ePA^&( z>H`q9-Z31D5LsMaXwRQFY!CKd`s`a@bXuSsmoyHdVA)oZ>a}qV#LLXs%k{pYq3b36 z$F>K-IX*@HbhsHZE*vGAmtbx}qt3aS;tV*wZs2+^_-c|VN7tj5fUEo@J&r;}KC$9=gSug{*8@5g5-+yQi8XxONnYVsY)#1E5so{wY&RO2 zP}lpG#Zmh`zC#a?t6inZIyvnOiWDZ^$T8{fN{+s%>GrJOj`}5Muc#qt1ZRg|oYq*y zK3Vn!uzP(!)}P&;9N9OkiTYeXeY1*X^HMT>`TdsoK<_kLP%+OliSl{JyE#PEMnm<- z$1S9}`RmgI_8A`pJBM0h*Lf`W+o6i!jyLa)FL`>_*R%`#EEv-5wd8COJ=<^HF-fxS z=A8&U*MI$Cc^MEg=j-1>{TY5Qu=MrBem?9a`lirO37rBI6hPG1eb5;h$P-e8q{n|$ z5zSC8N{UeKet#WGDq2I>5HROJcP+(Q`Cx|@$tYLsUhgBz6jT0*DA!ER94wo}cO@F? z@nI=DDh=Fx8b^OCofHIB+T9D?bHKC;D2chX} zF&fOk!TW|KSXAW+AXpaW_fOKa{~%_lhCqJ=0vadR3?T<6lQC*Vfu3vU|3SkDdlZ$n z>6+oWIi%!Sa7C;(cUSGe+*dWW@+C}TAu(bFm->`xI>H>mf%}#Wn;E=(;*2PGgk@xm z&#$m3DM-wpMSwjiRLd45ObS3TA`pp|!Mv80aps~0ol;{zwN)ApD3M-K%}P-g1EAJF z6tqr{Bb4-u8Cs@@=rPK^#qg#)oKxCAG|fl$K4LK?d{p!QzNIDpwW}UKO(*>)%{N-b^wGOrfuP^FtZ>dp2p)jIE95W1H)5Uw z&KQ7#KWu31Nl;u8X<^v~OGn{H3{(r99A!LqO|EXDtnATh^#B_s@(w*?$|Wv~tO+#j zYVEB>odEvflj#2HRV1F93OxV_%0A8tgkI561Vu^ko+Jc`>Y_qI23T|y^j3QwkbH_{-Q`E_$ioIKD%Kr5ZDjRY{uW($Y zP@|Z2gM>WVpO&1Q-Y9mj1uk{=Zr4dfxPo%*5h%v2q23XN;WD~~D8>`kfPZ%)7PuG% zHPca8h&Ln_-vY%27-UuWiGdr~oWp|;@c$M^gV;);iOxa6{D9#v+YT+_KW^+T9M*HA zA{@76G-$guWvrCy^f&{Uc5mAgSA9Ds@Jta#o3fH+J6S8^IEpbq=7z?xLrDR#C z`C%WsA79Oywm6ajp(KSH*<$fZ9vs+KKQIt8&fYhWUs@p$Phf#Q3Cc)yl_Vo7Q1!;E z)tx*bL~EUL*0%82TwrSJIpqO0#JdH9R(#ouiYd?PrE0Z$%SjVWmZ-tY|565fia)>h zi}}uU{!V@K7pFR;ik4lVIwfx}}OpaycyHH0H)JIMO2j1ZI1P>_9*@UYr;tVy>&moG7n?kqiygB5D08~Ka>79f zJDIOSk$At8CF^G&MX4HHdVMh%GfgX{5X;2le;TsB`VsRz_MxGf5F|VBt?3y2`kdXQ zdUz+)4O9GEAqt|!a{H=AVgn&Y_&(kfkUtPmu1kHC_X7YeyhXK8S~CoY0fje+-52ys zI*5w+1k$u5wXh7o0twoN2^)eQu0H%&h0wEr7MKWWebo!({3r*4q(;5zp)gbk$~Rq* zZia>F8YCqGc3}ZL=bLxYQENP4{QBf8N(7i6dyarG>A#2H@O>bFd1QFFPY*@xX!#2> zek@_~e@P5-g?uRXgo2S^hp|-!f=)JXm_)R@*gYmNg9Z>|zLZS@kgn=--=mV0#6F=r zf-EeF&;ez6y*?kU0QArzGTV|XsstOYZ)7n=hG?(LT=6VFT4EZ_kz2ZGf?ObO*_mqa z3?QkUVCUZScca=IBZ#&Ty>3yx$J?3MLWd|x7uD5}jW+mn?IX|#a2wd~gI zNQAHKssauSGz$Zv(!Jr6VFM+(tyGe_f`y$QB3iuJ08#)=)MnN|_2kYbQ^)+Ag8VO7 zC;cW-Xvk;T_7l;SGWz1j#0N5LKixxC857}9XXa4Qf|~M`DgLf)6xfAE4Eqw}1IiI3 z<;odLFJ&3&owDo=>OM3L^YWrXDDT{0V3EeqgF}@l!T?tofg6aw%Sorhut&%gA05Er z4GASrNgfQAR`iD((yUh|TZ%>yOq#EPqd*83Rb_uMtAs^i_xPgE)yz^M$0}BjZz;4S zypQ>cZbN&SBQw535r*l;E0Hd(;v1~Kao^R_2BI{^gXGpSTy|fR(qh4kz_N`op0}># z5tdrM+R>S=q*wsyf=)qcGKmo(xScR3LhJ1V)o%o#!HPl$prp`(>7TRE?Na9t@oB+X zAPYQ-D-gW_e(69A<`IE0EeaJa_{D-6ohC@hNYIrw@NhyH4h|?rr#%D*)7esO9mfYE zef4#^Y=+r<)w2?f*%&}qVT6oHm&JY+M|rA@8KE%3J5d`w49p35-rj~W*gYO`oWGPp zzdK5(%+CaD?zC&9k>N42QV>XE3A3BwO-3fQwJ;hJSb@@(R8u2Cr0g&=st6+wHza1& zC_yJ1WWJJ5ikk1vST3xX?^Z%6YXNAdr`uQYc#`HlLpQknP-#X{YG8`EMJkY?S|mAy zq;_;l)jg6;kQtO%fLwS}=Frq(Cz@Ei*$>R1x?TK$Z;);!y?ZQxeA2X;&_K$Nu%QaB%8j8~LRxM+Ys>Pb#of=a8EN@f}xUh)I{N#38N+aTny z73jlc>LxD(ctIZr)#wnpbGiw#lVvGb`tcMot_nUlx}~#|C6%3}U_M~#o0_ygNn}Oz zcSBYoMwaLJhV4`SC6)XqzkIgQz`5^%CzP2>(HxTdJbw(?a^<5=sU$~P3_ABU>7{Bz zvr~hkQ^mWa!l727yr69okTE2C$~zhrvTA}=2-14MkWh#NApm*TYCj3*9XKnZDQVG;f8KM ztLrj+?B(F*+ijSOENkdqbN{|xVz%fjOdo+P1RXwLXvW51X@V`nz2*I*-NGY6p9l=7 z8G^8Zlq&_tRkaj@gcVj4eRgQoDD4@$b}^Ie1w}(&DIu{Y7Ao-0J1UYVGm_9EH$)rZ6>9r*1>(g$vGx+fT{P&cIxqf7$DM0RsebkGXtWeEX_X+91fi0O2PB}B7+u;=P7**&; zz0Zof;gpU2Wk)7zksYWDIphYA-`aJ0%4y3XqOjBS6_3u3Q*JjW zu} z3Izg!CxwmsO^s61C8aU!1yyb)7Cev?(6n#C=hIa_Q!CK~x#@vr+5n4n$lmc69Ptn; z0EJvCtLG3x`v9s3#dUW&Lr3Bf_Vxa%nhzWAR=@y`BbgON`wzvQpBtpj)qU2TQrbrL zOO@?38S0dc322K0pQi_mH7C;pCo^y-Q6)q?yN#l9Xba9lD4T#ZL?@Nwdr*4hQ%s() zPnXS;_0a0%wVl8})UjX(!&?kHBm#d@??cbl^p>iR=vK+#S(2x!Wvs0}1c?BQdBcop ztAF#Nk$GDhyEgmF_iDEpba~bA#yB@kDHm5~!i+%5^06A>>Pk6=sG627Av~O&Kpe9X zj-tUBIpJ7!?AyQ&df=s^eFx-3tqoKE!eU_UA$O4KbtrF1sA<=&ZJ&W@&EZ4tl~Kp< z)8QvS2%hQUkV;)q`Qjzpv{JkEN^v>Sz10tC&O;$V2c>xH&VsU|D&eI^?y`-Z zS4YY641`lSE!t^@RMi$M70%P=rhHGzrg_B_M%(y{>f3U9_98Va>Uv5)yi_yhl1l?P zYrjGY=j~n;68p?YV(OFRL5TFmor}IKO)cU}>c~>r6G2=R=@#MZQN^(xVt;LM=_b`v zDmUup!fQWgOA?4MO(dqB^ z)88`^c0bK~j932kn-<8Zd8gzhtG8v4Yq5V1ne}qUAvYUBhaZf`uwj=FxtU1y8^)k- zOFrzYUVQx_T0d(&D)*1TYNAbXP%R8|0Kt|L77w{dLVM>#BqtOI3T!XRYaig*TH1})|0k38^L;bk!V)Kx}JW$C5)#TS5DVB|ewq};{!%p1i zZ-m{_nod&L!BpQ^PlOg{RWF~40mVib|7O?$=Gi8)Z|hCM_7CAvXho$D^CxX}G6mlS za}_v_g^n_Bp65ydCA3I6E5qo=Fj+~(fS!4sve*Wik7uSi8rO0W_1c7t#y^8jcoRYl zPH2m@JMEJ=H@=on^23)zlJdgXau_N|vK1Kk5(g3UjqgLhV=&`bMK?PZj;3jvCP?I$ z|8#k^J3gi?Z%WQXWMwS4ne zNU8@1IZT7@#!uT->sO_h(o@3wS2qZ|$Kz&nP}c&>71BFqx&3xBb1os^Rb;XA4DKI; zHP}kYn||g*i8KY55?7a!1rQ|!NNE%?CSgI7+gu*K(l4h=;b zP9>4oZ-UP{x>IE5!p3rZ92`%l>Tag{4Nd!sh?BGK*}C_42`ZwaKK=}&u8U?5Z^HsQ z7Oe(+{?SqZw$ACc!xJRG7t~~ZNa!dGca1iUqBn{0&b`DD2-(TT?cC zf3I|B8I2ZBfKkZf2j12D*pTmuq$68M3%F+*kpc)2|4D_Sb>PGN2bhTgw?X~Iuk^cagT}Qw5Mlwnru|~=bIAw%*=l+F=~|F~%mlmi$wz>e zo#)tZ*AkcovV%v!9v&D~DO4%JY7qSIv*Ch6ZjQz!*56)#!hizwZ zSw#c$i>*DkA7AU+Us0vxIno*?D!~`;R&~Vr} zbYNhGR5Sn&a&UnP@FQG1Apby!4oMMgTJ||2h%`<-$^tE$DEk>($m{{9Rtdr6XofAz@H@lAc`L_p;v30gOZ;jQq+Olb4 zRoc&V;55gTr~=+;0iEZYHO3J$}r~lMt&pci(yZq+p5HZCet* zkIw)``Q6N0@gEbec}IjAzde2AO1eGAWl+ZE(2>f0eeGg5hpM~K1OO}IO^MSyL^-~h z*OEq+(*C_gNYNw%C?N#n*-2r#?U;7~t%!)_tm6{$&zr`hlmt}A3I6{QY^G;#gw{Zw z;S8Y`el{+KjkIs|+>uM~SSXs}apZyqg9Z5eyf*Fvat~TMIZaNts$O1C)CXNo?|A(1 zB3Es_ZuU`Y*x*++?LlK7nn9wts-c15)U@GBSlj~W-#BIqljycWETXV|RSdiOmYv~`edC5B-&(?8=?*K#>8=oyV5 zj?3@UK=_Rwc?3980qe^gLf#3IYgUNyUZn4!*s7a&wMV+*fUP1fe`#zW8l%!bQAi6T z7B3T;+C|KWB;DbmN!L&cIH;o(QN^3${rK%>;w#rE$+Z;uF_iINNq7NgvOm?at~XgD z5gVBs$R!ET*5<_yzEi}Z2k*(CTt%F#j|9=v((hxi4N7TTejMgJ3i`D!g@N4|*Wtba z&;qZ_(js3+gc!|UWzH11E$3ioSxo-QocT8SGv~WCr1`3tu`HM2X#_h0(p?`f^1&MvBjx)`P52|d$7Q1MK0Tmz^6!m;d1!`~L1*dytz zUh6Lz#;m%;ESICYE}*@HD`&6rMsT0wUmX3I*!sKTsJ7%F_2@V8Uz1*$>YWWOAvM>q z`FLIMNR(WiF~@r|U605O(+1w@MN#hdrtUwSMp??*OA+mFOG2yd?#7>qPnsZQppOjy zvAqk%CTR4qTkD+RmpRF@23>@b_x7hL`6N`STItl;W~CbE3Tb4{$U9*El+Bn-PyXm? zQ6hmL2JAnG#EYXixO1@Xgr?VeX&(qVoGeHv7PgJP=zWaFQ20sOyDC!P`$K!ZT3=b! zTV8-R6G(LLQDQNIr+R?uDb`Na=kr$+@Pf%;i?OtTURyWEX5>-7Nj}(7XgFWi>Jw8h z;x>gM?}&+qU&sDKnaUjZkn$kPQWV|qmxjLnOtor!V%#--3w$L5kgI(C6>>WTqO^6M zbz0zEMRR5%QG|1Aaoi|dj-WUx#&kFjrT{+2Rk7f7WmFzga)&RreNvCvY~oGC5x>N$ zCj})Gg9t%2LW1EZF3kPD_bb;xO0A6vdVedLvN=j|4~3;ujjiq?508HvB>otz zkgtpW1#fPp-aA4hXx7Ze6L^82F082P-d>*D*YDV+^q(YGeg<{5^nVmfEZ3FrM+G9g zYM?l>khsbdKw~7jfN*M36`1zDHa=NGVFyegrUQILVE)`TAk^KI87&-g3t?v>l>;I0 zOsS;k6zEiq>HWV>{RvpCA%0l8b~(KDxNi64Hp*;S{E0PFTR)Jy9V8{SEN;+^5PG{> z^mgw1Tt54v$lI|JO!M!reiA=p;(giyD?W!eq|n!yKjFbFNE#EPHHX|mSd54}I%wZz zxXk>~E-G4*|6dJ32r1xV2e>SlV+t0A;4dJ5;Ju?l(OtStCAm74PgyVzEO~V|gZl5P z#K7T|O~pccaKCL#g>&7N^m)u&oWTp`#3TOP0Vh;d!(I6=fSM4NvC#c#d6MXt1C)^U z8{zMeo>Sr^3^tU@XDTSvSe&F7E!$+Aq$bcp9ArQo{5%Or!s?=cYAFF&tT3~#WL#h3 z-lufzh5RD{XwzvlM5#w+9ZNbB07IeZz*~!@bmBAIgx}bdYM>9O31>RkMe8_#(hFXH zF;qtZ!y*|J-TH;lw>UkdsD;84CM7~s?Qx+54O5&m43Z$zT#PGpY}a8C*szErfIR?U zXAVm^LNNxQk_W?YS|cR?48M&7nx$z*z`P@}x#g@s2^BVn6g)?sF|d{3Bg&;CDy<`` zBO_`na$em-;-NyXk`rz~Pz+oKEmKjnTcVvLtZr5qT2xaHD;nVh^d}aeO%#Gt7)!4d zg0|3Cd>w=SFwja3rLWA4V33i92*%k0;}Doi6lW|5#1Ke0t`yp|`?}|bP&!ryD+Mm6 zCHpOlVA!MzO;A#?H0|+5Ch%(LI`a!oO{HLWSJuW`$9>XCoOFxFb{8a!mH?57a*X+z z6AHyudes?@LOX$C0l<3EG1j3b>p(={ggSg8x^zPQk1T2w6r(Xhr4hqHRPNbGa3=G` zQdNwgvZ@-(B8PZ>5?U8{G@n*Jr3z4wR`#T(0j}OOLSSgDks?=<*x=-f;YpSzXj&2s zTS}!=4aHDaurNiUCxA$S7{B!&dyI4Yf^r9fa|hOQhn^aTu(_kS zxxLoj6B1>+?m4*;jnjjXFv! zE7s6un%O^Sv#)^Jx!d{K+L`HEwMj-z{B;ymA0X+lyc&{V@sw-3lNlSpqn%JLP>o>w_H^uwzhEN@qY!;Lr z7}q{a9QxUi*xoIYUuc;>h)5I)oEdAQx-N?dK{2A1-;B(DYhC^fR+5<2+1y%wQ@8x) z>+)OD6)7fNscoHKa=N#=D{}I>k~S+SKrCh_x^fVp5(ND3 z2n0=8R9{(BUwRqWRD0FRtj|i1vJ;B)0S&+jY|1=Vd2yRcsntiBm5dE(MTg)+lA4Ap zp}1FlTAh8~X+W$Ufp#%|&SMPYLnJQJRt)g$!ub=~Pp8dQE68|9;UCY*B1C|hr2S=! z^n6G-}IZf+MQ`*EQu7Y>-6L<7747dRPPsi)z=_{<34>IAqnDNAE( z7wI;;qf__@@d+(9FMP!bDNq#9VlUHJxE651YOz(g%?@zbpn4qO3hhecYg!n@^$ly1 z#;rjEoI+7CH#!?C(P-sJuD}Gxw24=~Nq#|HUY@F-g|^*Fzx|8evXRKLkWK6h6jk%c zvK=b_LZbFKP;b9iUk?@tszZojFY`IoykdpEUZfDT0y1o;anlnH3(cy6Dv5F$wOrxS zmeIW%;1^q?(@G#xq$DI09OCyi?vXz!MTw-c6W4eu8_XI#Jc~^$my@pqDzDVr5pDR4 zh^K9d^&C}J|1erf`{fpoJ=ho;lgvj-rV4;Vy7RQKj#ZqhMC)k{{(B+=-e8ptzDU5a zMkP!X$&?_ZKsUV|$gh+Tu}nYnfTI;M+40dAjqC@?!m(_3J*@a%n3Do~2z>K#SY5rq z5~&&19GARh#?MGj^mktj*MnHMHR~CIVU`%1(BQ(|)!9Vq2nG~6Z9_hkS6918z9xV* z^)su)%?1|3PJCrl9|7p?+r`)QjR0#YzG&25+!f~6^&TzFWXtW3hoc6lAcePnud$Ru zaF3y_Jz(5|g?!*mLe!56QB){CWX+7P;53#9bpba$6R@L#Vm$zEhuL-O*wTj4j9^}h zAFrKW5BXw*kgmjFbV5Za-ozcJ;y*xxfJ8wR1Jk9@aE2c;l>pbmf5Sn^DoWTFCB#Kw z89C(;lE!XjKa>0!yV9h-DT>g_LSn)NlGsH&vo{qu&O%)FwEQO);pffg{Bpu~U(DIm zs)ca}dlh32^~#X-C9jmW%ea+LD=q)}Zd)8>xu*z*BA~$*3kOQW$Hrx7U{C@zlu&Sw zyb4aV0!DoR@-?YNPl?%>qeNAqgWl0^enw+kEPxl}CC5(R?0%q!1GUqqDt>$8cEo)A z@m$fd9d}q*NrYI??^?Xj(d}rad_qC!cgc?xt(gFcwUe)~6XAe#o;x57e(UXZPKTY} zp@%IP)i)87id)&|lLq{_dE+w>S*%v_x$u2A1jc{yq2whxE$>B?WvU7xXzc4^<*`BI zM_zralOUlA9dk;(eAe2O-kD}r<9~t{S`7S*lSG!5*Xh%6Vh0CSo()|Go#jMo{Hu_^ zk`s{7U?c(~G~3{-`(kNjSqNmXbYk=F5n4n6P4SgS921Ye1QU!L#6f{6{1bwTGul|_ zo2p{C+o-3v>P-;M0@N;4Nx~xwkVbSDKwvlz;td0UVG8!auWs3x((VU00(BX#q`}Vv z5Ei2zI*kwrWQ}P*W$gqD&yb>FLW1Ln`tb?+pLkJop zAMKYGcNx)=@{lSR8co8gdfErQE0TgGx#9dtWa3MCCW)gEKB9yg?BZ64SjV>+os6U({l z{JA|g21cd06D;0S-ifbB`aqFVg$v8ZD@9 z;3f5754I{X1kCcd>NAW0drZ?`LG)|N<-Wg`#J!Fi_U{u5NMQk)0TKBl1uZNj2tV+2 z=9=q}8UWXmBa9IN?|Vuq9ttS(KI2Op>X>5@kk!-o6ERYK?+auJh^%G>KGTLh*rVHl zh`8z2Ufimgs4P=|@Fa&G_vjCsr-zFfdXg$xr|Tz@r->S7F&7_+8h`i_uRj+&=7~1J z?GMlv2%vZGmVhFt3pg+Uy>pZ>u7K3u3Kp~g)q`F*0Ei(Kv04h~BTshz+%$P47#55rDCeEAt2Dsxn zU}pq_?)LTE*%OXGpF(#ANNO+u(UnqJqO5B5D%Px8F;4BOb;G|1TgPS%P@rE(vL4f_ zWys(kgti~4mSroXp8>X7>FSm1x2gj{05q&rI0Qh73Mvwh5wxTC#t-^$wlYU9RTo40M>P-*Wb9=tg2 zmB`(}&RyKy^WrL@Qx6bSr9a!hlM7!dApnR411bzDXd8lo3q`b)H;)_tPHUw+iSEk| zXCDMO`3Lxe^YZCI0QU74fdv{k(SU6wcp!rfIw;#?Et!@RPL)u2A%y|l_fmfa`L`2@ zJ^2&VLH+GR6a*%A~*n1^@_C4bcH+EN%JSRD~d>1(g7h`J{|f;YOl{)7j<~ zWV=a6oj=0WM%-_F`U%}|cJ5gnbnp!~ouGCO2dHm~a;H*My&c+TcaR0(fJ)5;w-N-p z!55T2ojMg@LG<;+A5sTJa1e$t8T21aO3FH`LrD&~kgc@py6cP}JidyPcrOKuPpieg zl{Gu_KyKgYLJ&{6=V>mmW_2l*-8uiqvn$kw&_|y zdrqaOl~O6R=%Uf#sT-q^n*3bIAp_dfo6D5bW&T%F8=YNzPRvjtEIn@{-$+nz|p zNuPiv-bde?s4}&zJFEU`?1k}4J(jy9RXsJp&-f<71ftm!32l-%V zs&s+A5)^{z0EI}9fCfW^|1`iGh`rJ75N|*(f(2r#7J0{QzT96^pT>zDe6)|~=TmT7 znjD}dt9+++e4^alqlOCivhg>QEOMfFs?2hBRH=l|oW?_E9;ls4OMBbR_9XTI%0jRY z91$1=E306~)HwX{%Rm1^TXTJ(*Iv7SQ~JpM1QYqnw+|AH4}JX!U#X@QJNwa3SBz_5 z-0nsZZat8Gad`^_MN+Q#6)s3=Sy$o;#Fq?Otq>`wmj+Uh0u6wm11W&mKm1VvfUPhD z5&)A1DgcL%=r~whYA!1#DPjaLZy+EntI$debb+SXUrcetqz!{F7Sav8$ zNh*5BD;{>_v_<5Y|LjqXdRe1@(=sX=%21Yaz>`MCJoikCCkg3y8%G#~*je5`XI02m4dQG@|7 zBmza?B>)B^2#A>yQF0s^aSUZ0;shlsE4fY@b*D`s$q9H^{1eJbZWZ&io5=}?4!HU*O zA!mV2gDm<|uEB8uMzxMnZS=7P3_%M;D8djb){q5e3Oh6l0Bj=Q#p)DCQC%@7^=x*< zZN}zfdP3P6y9ylU%@b!Ja~@llIK?YI3T89IDp_^b00Y2BjeFt`bNsQJ-Vn4z2PNsN zIs_n}zyxhO>F7`+E7`x@%5G58C`}$3$w-bZ0-p6xAC-nG!)j=-DJ^WaCac;EnzA1S z$SM59g2C0oQZ1ph%56E)5Lz-2NgrD+ZcXW)MQNQtlPy3TVZYMsHOCt9=fsNHBRJGjC}UZN9EqDbqsrXrO}Ph}9KIP|e3Ey@7W z$5>244Y*T-t$*ics|iuakps|QN7J{-?OBp&X)9mR05l@^K1E}`3owQocrk5_1i%fl zsegB>wcd*4!WWqhxLzw%zl<=$8*z=sYGb8A*~va~l4tYAI=kxN=8N``&h2J&8<1g> z$kI7$pUBu_Lh6!!o4ZoO1)EfWzL5RCWmI-;dnC8jI zfGFr95)1lVQVEdms`r!~>$4z<$}W>#1~Ru+H^{oHs*u4ey|Yel)$CNYU9DSc?NL{Y zSCnTGAG0E2#mr@O9H^<@`;-VE)F*sAA7g)#Dm(wrSizE5rD@gZXqP1=f|Ln*=bV!e zwNE~HOdpsHI;y82QARPMHoC_G7KCkhlnwaDO=%kz+nQ+<%)E*g2IIDgGDOlZT5+H; zCKV?(tDM@)D>kKU&gb!%$SieYU@gp+O?ek)H9OfoiL=#BL9f>s-*I=>`rSA2Wie0k zGH`yUd8 zxzFUzE1-AYqG@Ls(T28kYBOkPL<6_BBR-*Gurh!XTJFRT@%q0PCukXUtYe^(iaE*n zS1Ot_ob5#8#oIX?*k~G5^^ER3Z@pEy%RQ{r8@2GVh)?ep-pPHs?XG!3oJRRN*xz%O zNs$7dW{>YB>+?KJ)K|Xz(mZO|iuz=!F7*ztrB#_7I?+ptUzi_#Z3o$Woo%<$fD+uq ztN(qx!18X05L$lk_QQ@;pAivbB>;sGfe7x66~WjWT?_Je^3{&vi?!Y0WJSu{@0vt; zij46MCnH-6zFe=8b9p1DHIpq4|G3Kk+GO@D9yXzacm2zFpC+I3K(SoP_1RGHS!jvP z(frS;n8?lL7fkRMhEz*6p<IAMj~}ySSMHG(b$yL?w7&?8Ssb)ftEQhlwQJvItaA z5Stwh+nxnhX_?yv&Y;v-9Y>g(1ro@+2|=O_j@GD#B>3P}EI=g_m@Rm#rBO8QBx58QB(=Q5hm$ z`6ZDpL56JL103-QbVN#f1sm>})Fzo!`E(wLm>Ee4P^%c2^f}N379VTTpo8R!&vl@M zSV#!Q#HircM`;zf8I(v_Udy@vAj`?1BaY&=2pBY?z4%>`*U? zA-u?9lSsh6frLaPA-4RA76cI_6h2{qK%tdn%w086oJ33W)K$z-msaYX@MxDD0osr) z*Hylib)_TlNRgu$-oY_f#^G4-^xZzm#sb{ob_moSUS8Xv&wPnqQs5V{`NRM{;6i~7 zA~B_5ilRV@jWtP zB;o=@Q!W~%M4+7_nhS#xpL7D$bS@*9Mj?~QQMAxb%FL7ErQumZ+3T> zSvjU9l06w)-su$K9X++-`^_O8cFlopnQ5KY?!d_k&c~^U2n+rXWsaUt_>n`E9!}VV zLwd`CZmMA(2osiQiVD}8J?2Bo1ZB=1A5kWWpcH+qR8$ZY?&VI-fF!6UpM#<%)u?6w z9(35&T;bs89twNN&ar5!U8w)@Bj?0}=ki2c(6W06^Kv#sKtEmo&s>^wP3U zDMR#95Mt?3a#&$zpDaRJ>~vogj_JA%8C~&C^*mV`{fXgakyw7G7TNzDr;$;|;7S*9 zl{u=_G+tad#Tr^V-Z_QZHy-STv4#WOr$?q-aTwHRl8-%NWPCZ)2`C86C6K>`3uZX?YSz4HS7s(TLWn&;rSc zB8h9L$q+cmi*mrE<%IWp3;^V0 z>q2P2KYR!ARRmJTON-f7y9HOeifMd^Y*Q}YTfzs>=qXyZ;k?SxS#qVoCf*oM5z0D8 zSsqu&<=CW%UygMl7+Rd;nPGBxT00VI9OcPIMg_&*R|*Cqr0V~dA3@+vSZW8pPslFf zm{OI=GG(X+Ek&qonz1aL9oPV%mjhI2sS4&%@E1y%6n#L;QXp!8e4=T|qr`pc(837q zQWUPPMbDMh^D4yBPF*FCf(U2;F&XQ;c?8&208lxf2qb_%C`2J7fU~lNU?f1hKt`3= zjRn*T+5UrS5M}*X1Tp?sb6OiCj+lUu?3W7g6ONaYImgh{>lv;S^utK_7s#oDqCNw6y^!gAN3C=G20-mZtt}2g8Fg} zQ*rFdZbFu*?ZSlaI;11f1V@2jM{yhQej==Z2z?2~EL&^>5zPux5j-s~^H#CWiDdL< zvA$@4j;59cc5zS*K?vX|v_0Z26o5@;gaPzY1z4zw!bG1!xN+3}!wFTN^CbWP)bF`u z&Vam5Ka=YODK!6j(Ca?sL9%Ny;w?l^VOV(~$fQWaxhdf#8GIVv0^QIRF--98WEQA>b>4#o)L~uoCxYhC+>b=@tLO)!kA6 zynzU0fY)*jwKSSKdn%J`Z}&dP7W~5&lrcxBhP}K*1z@WWR)G34=|9km2(XE2j6xrm zQt1GI=r{yIw**FLX+xJ1L4v8DnRS~{bTTfGRNBs0o@H|^-ohPT;C^z&8E{%A+`F!0 zNgwW7re&JiNt@Q6I_Yb`mbMtN%x~}rWHf*sCa@Ft(Ng5z#R5qB3}Um9-eZ|=PGqW4 zbDwqh@}>gw|IRK`7xM)kGfaqwRT}{BVyd=%R8nkWA0|a+N>a<+qtB#Ng%Qm*1yYkNyK36UP4bqlQx@UtOFNIlmFWe0biU#r zdBz|)#f&=6t2nY@Z#Uk)UK8c+ouX7vLTc`lke7UCl7snYMAF=DTx6C1;|v$E)FgE^ z-Eej{B~|Gr>{^V}eK|~cAWclDgB7ULh;`T#T&if~oBkPj_YiS9c(+tLdGj*y(iHcfne6 zcCwVFT$ytP-lx$vb8Q!^4ep(`A;pm&mw`BB%w;8fk8haDu<7iWrAU4)R_HCDq!uD! z2C{ZPbTFf=-IQ#!*{)+!^~=WWGC!n90U}`b#9yWloNS&a>Rz1aj(LPicuujO|GC2N z3V;7aWXN&s)FSm_#9r^Uj9S3UkN069Z@Gapj$Oj&z+eoSzu5e%3}e z$*EbEKo_dl&vs2g9Ovs#^g*AM1x5BTztU76hdR^Wno-+Ux0060Kk+$#(#_s zwh}lRAE{DiM5}AqiQl^220sB4GTnCta)(OAiFD)gdXpJB8wNYDGv3m(q4)Wp)yLo9 z5_VnvJp7bP-OIAFkn0r(6!sKjxA1Cl07k{n=Q zrAn7CVY(Zz4~{>MG$YUqFkmN5n>!=ybhmE+oSjDp_%q6LpSz&}jP48I&Yx3%K?6u_ zh>>eouV2B26+4z}S+i%+rd4}(!@mY=y-La;Ld(klXtx%GxaA*TxLgaR05F&^01aCH za^>)kg4msc3yY92su0h+|3m`4UF0cOTLvLc+ zV2>iSy!)W#+#7rAK6sm8Zr{cex|eQs77Qw~0X z4kGWm@t!l*SoPdPkJ$jJ{qMPB1BCZkaiJ?NHjfr?;5nWuoY2ZNec}iLnJOeH;3}(p z5~h)C9ZT37dyF{P9C5WVMu#NM_}hl(lnvwFIPwUjB00im$(K-4SY?i6vZ*GL!kqF- zqbjT^D4Qpw`KYCiat|x$vdfF(p(g^B=%bPUPTJPC-kLK65#|DbQUEFljktz5C}2>f z9c4_cH$>3j0@wlOpb$7@MsELbKxSYADuZxJ0|^R1 z5e9Ps13ttcdlS-FXj3Z*2FZHJk>27I0<0W~30t+h+bf~mN1tz>3K7ZPsVD5BkEC4d(`8e_H^|tKicLcB0Ca`TB1Hl-lUC15mE32 zHWc86NhnPLSao=)6bbDwO;0fjg&?@dp{-Drt#sv^nD)xG2#uAptcVA^78XIwoO^k8a%b#y_=uLy{ zr-%E2-F_$swD*8fa$`0_;-a)*Cm;4~ai9HB&pYQC zo#EI^AnvGy1M=lbsZ>ZQl{Az6hG#tFm1rkLVOf?l|XN`wfSN|)jZn@e@-Q@O%GsFI~E zegf1~Ct?J*h-C;GkeUX-s(}>n(rN?Ig47IvG0y-XnXM7Q0-6dg&w!v-ah*sEvC5T+ zeUF=j#GP$^gQIOhOP%QdqSor-sl{n+i(LbqTw={BL?H$gii$y_(t zRlV(XX)N8#m03a4UO5b$dWEpnu($&Qk_Z7|M34eW6e5ETDTFcK0IC6yrmX+)TY|Z= z;I#d9HnoYbHjCR$iV1R|E*hsnPI)@=)TdgVr4JDutE?|-Hgbl&qUs_m;vZVZTl(zW zfo7MUegchkFqR$v7vTvn*>zY!scq1Mg8I9ieOF4iO_7IS=24OI(YL|9;Yqs}G2*V2 z%NKTtiW=mkFjp6+9EF1(gC`<4Zb?d=Z0%6RYsxcOCqhgamb2{Z<@M^B&rX$UpC?jP z3=h!4PQ_&f!6K6d41v)@V31TEG5{@HK!#=wunGwx8TwwN5F!YHx5_fBN&}jO!4Xqm z6L+y6jgq_v`r+oz2}J;1&WJzeF%lokVEAF@%zB0Nq=4I!8 zBqV$^Ns_3_OJ+*iHr%welDDyIQSVZPH&ihSbb{iKfl0Q;H6}Evfdz4i`w_kqCy1ai zl3w;6Fu#bl<;Ep4IE(Knd+0v3H`&y`!^&%*|GXWJSx0O1pwo8CK5=9j%R1Kuag?X5 zaeRP0ulE?#$DqZWXn{!2f{qK=WkhWNtnHwpf@ygD8y>ikVv{F}yP_xBt+`uiv3GMB zrCJ6qaHF!H zjNO9}QRdTxNcShN=~dYvb5`{7alL?6|E_~msMG@yNwx98MpSC|Dn50|%d$;LDCyBj z_wM5>vCN~#X}{!F#@?2HizClJb>1WaNlWt0$GLClWp>o|m3FFkq;Pwu6B;~H2PVpJ z{a$F9kGTH`q10@8jT20=dtI)UD*e4~M4k@LVdpI)Wrf=Aj}r?l&Mkm6s&E9*)rLL`bg*l<9`?*TuL4(-rQ6maAU@GQ~`^9sig^lJeb5F)PW z4#5T9Hm{_@$%}-jT@Yk*6idY>=UI*oiUMl&4(j%Zt;KFIURKbDaLt`gEU|h|)(Feh zOsrUL2iT&c_yoe*DnYf%t0uUu&3J_IqG%3e#<+z{!tR z2xPKQ3y(x4OhTr@kR)>AWvq*he5bXPB1^t0s6?qI_NOX{#o*wOUHUM34AK7p({UYb zV*p1E@*Zdo3#U#tjik~-^4Zu$C!;?gl+?8hdLbbcF0Nephch#Dh3aW zplB@EAkbVE^5|C1orDY_5zDiFtYXHi)MCRyLINN7M;rmBOPFUT+UL6HjvAA2lCnu+ zIL~{IvHP;`x2UiQC8!?{2@6T)BRGN!M?xrDf<^M}rig-INFwVJCZxQ?ew56+R1$x- zq-XX4KLm)V97*HJxH~x?WZsk zyJl+u=nwij${JIoet@w>V$uo8&g{nShWu*#vd@8VM^{3|{NQe;o>2+s?lVyY`l{s2 zhGHoS#$W2McY&V&sUNwLIi?sbBQ>EdYwneJVZvAzUi z2czym#7j&x>dm0C-|EQjqGV;7^vFgec`MC{yMr zSB7~u1SQYxcaq|g_=QU_NhOB|LX7h}%A}2kY%2}WI_*$LpL3fX$I>Erv~bE2P7W|m zuru+rvq#15I~MZBTrbwX)LGlZk8)$7Jw5 zaqU;tsm7wBD*oqfI;BP-0^> z^L|v~hV=gitsg0tAd)ok3bA5kb(>~VhyJnyWsWapkY2LrvzXOgE>R%~@`?D*OCKxG zQcy9~Db~u=J&Q=5Hj+PB@TW*lG!`HhJK~MLE$i%bL}dTsyYOg1jj;Mi6GGutV^}i^ zZ8BFNCwvyldN@>H(+9egWV-m3++1e1p6n?$iQIbACg{(#`sGW4f+`lS&ic>8L{^*j zwr>XlRV7YU3&;xnuq+Coa3}6#E4Ez|2S|eK=1?(;G}54m^=8$SuxxPUK+ppB$rDXW zLuu_wLx*x-4l!pfa<`>_nuWqTj0d+hIAf-wY)g}5_g{I+To)!Nudg)sF#_9F{1#`+ zxVG$+Drx5RP_yu%8bBpXX3V&dM9EBGqwHIMs_uy51jY3SVORNvYFLib@S0R({TA`Y zcW)W!YSA}+$*!u@w|(6=VhW%D-gkcKmmyrWZ@K@In{;RsT}%}t2Rx@matzWG%PA6- z=pTt_N@K5z)MQ@&vDRwGa;?aS3ePcbmRX1eTc-$L@}*cX6;BH${-}{|m=%j^9jn&rHb36r@=1{hBm`yxM@H&`u zWO3|0?a43O1$68~b!I0a1t^&SR3Lcpp^*QU??mJ&osaKS;%NnD+(xs(cvqYxG<`V_ zx&Aa%nM&9;)B(cGUWbw-!tD5XOLwP{M4EC%xkR_VElVoum}4tAhj8j*)Zv~uPC7R8 zs(Fep><9W`dF`kOk;j|o_?r=imB+c?%K4nhIi1Z}ozZ!n-MO9ZXaE?X0Nl8nzxkfQ z*`D(mpY^$)#d(eaI-mtQpzXMz>lgx)TKc6qz@=lF18BOYZJGnN0Hd!pkP6?mJWf%~>?o1qhe1BAc` zT-v2snx%h02uk|7Lpr1%K)P??0SdsnB|5t$nxVHlwRwA?eL%H4I=l;@qr02C$Gf3B zTBFT7yH#7eX&a(TTdLvXtbu!FA@4&3Czk{SHhC7Q~oyvm_G%cK0sv3$$DT+6}y%f&p)Z^5I@Jj|hd%+(ytoxIGg`xXx1A1+~@ z-~7qT+{_=~&GWp>tJ}=)oTH(daCP*ZGz0=rcqI(L8rs+z;J5+Wc#YGT8_al&!08~{WLP7IplFs6)3XsN3-mCH zIjj`;Jn@KdFFLj;pFmaBsHg><)f02QSvZ$~MQokYWUfU6qDCh9^jevEJm3Z1MK%C%7_SHZ!Qjo3C((W2t#&wa#BrlFKG(D(b1zLBQNJO)*6&je%etT> zOFmJKd(0DZX|A6VbI39rUERh+Q5Otili zMu75sjI!v3xv^hD{dQ}a?au2wq_$>ar`GJQ$4e@E=ij~7;kkZ%qk03>QqNWh;YV%N zsJiQSR7VK+fqn>AHTjYaTy}!}X5s#2m0do35ASU@_NpVq@^5<(V#G2rBTrVxG7B;< z{#u9y+6MxS3_!J(Vkt7@%y6Q8I;xIRgf~U%+rj_+-Oe2uTlh3rb?9N_@mHeS?$xnN z=CNHurg~UqbP-{8VkyXtGkt>c&kQL@B&WQ$wX`0A8XoP1pE(&I>_-ma6`n}XUaDRG z$a84<&tibPT3QM6KWt9sVs7SS$M9gTiMnA+Qm?yVXCP}a=^Z_D(K!F1W5*%3YphANT8!lv+5TU_{|0-I%h%uwa zjT|pB{O1s%0|WR5kQ^|eBubPH22^TkV5Q3eEoIugiBo0-IQwh{Kp6q$Pk%f2xhoiS zsJov2;Q0F)z^K%v?i_Z!iZ!d&tr!Dv{R;m!tk|(+%bGolHm%yVY}>kh3wA>#xOCNe z<=EpNoVs5P^8E|g;@!Z63zz)}a&TRN`vSQ0_qZzL$x#_Qz8u*xWX+EoKdv14>F3O$ ztBw{}6}0F8n_G`w-C8qhfes9_kP+K^W>_*JrCb(T6N{iy!|CCsBz%M zmkk^&zy}9H&wV{nLKxsYXhD=mdy3DA9;IQ!44_Gi0 zO(xYu(n$dt;6NodDQHr94i2#001+JZ6G~2v=iPYwz%k*575=21b`*|h)qa2}W@3se zuE=7GF1`rkMGROIV__&_l-FK0B3A!nk2m%RBw7vW_9I)SQ5G6!vn5#^XGnr(TV_oz zx6g3DT{fF&TFTa2m&#qXT516}6+i@IN|%_EzHP=^YK>XhoMdvs8I@^OCTG-T&}HP8 zZVNbo4}c3$H2_sc2~{Xc-8Ge{q7J?!Q;>Gyw-9}lx_4=&?dfM?fAYPgU`zNIa8d&< z!PH<&39@wRgRX)H2YC>#>Eivd|O&&Yi9eA|*Jgb?zZaAUO_98l!unvRV z7t>BZ4gYo2P|qlJ(;el=Y_(OhW$e~44w)ET??MSBZPPQyN%~gWooKsb8lK;pYo|be{~evf0sL8ApcVHt=04x;q+z2n z?Q>qv>Uphg$|j$DpM9HRD&Nv4q7G_JDmh^4NeU_mV5l*#g!9gy1I1yixb{5aQ9zH_ zt5Xn|nJd_&W7YNZ(oau)^;~hy1;Z&PRV3b4tnvZgNt}##?aXntwOm zapygE{O{ZSesE9O4R3No4&*Dz=lrBr6m!&tesaX|*h^@i;ZgYb^u#Diaf&DU3)QH2HN8O1by)NW`MTJxVNFmi zq+tw_4AvzyrjJgHvy7a)WH$H3OEUAa+uHO7mCsP-O5n*)n%bm5%q%dFMuCj_9J9cv z7?3!dxldIjIKe;t$#OhNRD^~Hp`rxEIX!#TT&Q<4O|39xp`1_m;8IGMLFy$myWy#D zw#o&eq<25OBUbUx;917YX-^0k~ab%rzu6rhsoS;&MT z5;^|JQ)a+akbOY%BLAtx0t|2+2wCHuyslS`R*=R9z`mrm5FS$7Lh~P)Sy$ z5&)s3B)S_ZPE-<6n{=>f9~5E+8>O8OGVxG9kqSGZNla@_bEiD*DMzk2&Fx7KrzZ3% zd#a^VtmMypYU~oY2xuiIVMcINUCkZgwJv(yrd@k92n6t`G<)uANs3Hl;)X*o=Aa9a zXG5#`8WYe)7O7f*0@Oh(Iigvi1fsA4-lJGmLK9k)J}5llrc8#|`0yo^tp6+tf@U%j z9@dbDkc!DHX=2$PrqpNMsTB}sMW`}0g*!*7STKE>RHLePwUSD$Se6++@(FdKW1OCA zSv!{T-Ih=O10te+{>`^aMVET z;jlOn#TJRGZ`4$-fIaoN;Dn=W@VCdhAs2m1x^Q^^SzH-&$t!d-TvAin_AbvPPE70ea9OTk%=23F8J%RbYX;mus~2 z#^QFhiQg35mN*?wz1?QH`(v(V*qJ8I%#$+-OCQ1r=oxeA(|`9Fz_g~Um@7}<0;hn4 z0>p!{o`ev>D+x*`Z-TSfnpwYNZlRb@Su7aFr$7q&D5OHlyBWd~Cktk+~vZ5A!h+FPkAegCn|CXfE*$i!>V93&v!RNu?F85kVOD@Gsen$7a=?(R$6Q%tUCPCVp8XA>h`v@$FDI8y;norhNo2{ zkb$t?=OE9@YC{PW*B&W3Bp^`)OH5!V5s1*RG32rNmiXB-cXsP)p7q@Ng`**%2`g8% zs+8Ceq%v9M2y;?W-xclgJ{eDkFgLQTXMXFnuf0bI({5v49kowijO`cGbg>ARaQ?dA zoT8MaeJ9S2ih6z2_t%1(5AVVjw6%Gp#aoZboNCArrea6iLT9;uU;Q z*L%Mwff+V+pA=O8aBG1EI6e}A6IeQ9lv@2$K=N``ndUxnCw={5f_uYNViGsihgC{5 zCEm1EPQ@?U(zj{QYda1Z&JsHe+Z2pNOi=Nhv)W;jTVjl)r|Eqe2CRs&J1Hp6ybc1SwgJ*+X=K>|<@*4zW6s9;v)&VgH zbwRLb6E624$rgnarI7Y#f0TlSErTBdB2wBmQV8W{TqtMhQGmWvT0b)@nnN@xla1Gy zjV1rNb-4#hT2YOxMLr}M9VVH74>(LABYj(fTr*f)apY-fr;jJ-j?(u=|0XOaczob; zM&_e%a$|5tRycJsRkDUKR8c2V0TsFOE3NoJ{)iDIafAzCPzYEnvBD@D#3;w6LmxAL zmWNR?w>p;ckop%A)lpCd7AmY{6DZ|Mab`1P7*TvxQ=qksXI6)P2$O{glhY_nhc;-| zM407cn7HUdfcSP`*Ns>eb_Ew?=c7k$cbSYRYIGDEhZs%}SBN|&c8NF;4xpN=$(pU{ zny>kq6R?`HshYAWo0eb+7hntih$0AuUe|#{9K$hQ_Csk&N@Zq+p+pv76Ok18bJzbC zjIkqtvvWI_Rg84D6350~7xtLYLvJqGolu98iTQ5P*qBxmo_*LB4KX*Lrd3?xO^TQ# z@=1s^7+f~^f@B4sk0>`=Qi<6o5b{tD0!p9*S`P+FatWHCBR6si%Af}dp%IFp1}YB+ z&=IavS6(SeM8P>Nhbyike_%#)mB*0np?Po_650lOrneGYSb7A)ZF9ydDK(a{LLrqT zm?jCH>B*x;g_vNplB%_6^R}bE_jC=BV_YIGpLUs-SwGZgn!*wradT;ud3-nqj^mOv zvEp(q0SBsS5DFlsV@jrFYNls8roN`83UGLAdU%I-3UPW0mcW}^c^v?G9UT8sAxkJB zLGy$_w-WXxg)>*8C!}JCfgc8NUOwb`jap#Q8F~XCm$QSNH?cuowm~vknCID}(bTDJ zF`htLp5X~%MQVG!#}H)qpW|qWa}+R^7>RbbHkcVz*5_1gVkk?Yj7Nb>vEroy(wY;1 zo49EUr+@@)ssIOY00LSM4sl_$7!vP;eHq9nt|Lj6^CwPtFc;I zXxkzm1a@r}33`l5odc43G7%HB)NLL^VVmSLntHEcE4IHEs$CSMsI?u4xGlkg>Qqr^U*#AbWW9Fs3W3tSAei4LYC|in0}Y04ysHEBm-5 z+qf(1vXy&y+7PD~Fsu{M4qjTb4dI4n0!&D8CJDiG_L4M9K}dB%A$5uo3m_0jswE+iOhwibZG3y__nx*BAh^iLr_|th#9l9eWCx`=%pHvdX%w5jt`Nx~z*U zza@*ZEGxPETe&a$vT6FUm@B92ySJ8rv90-7(aI1>$r=nNv$UanwZSe9go2nUCG_x1jRCpdyFDBB!j+W3w+Tu35pH5osy_vZ$j{P>w1yqarF6 z`6rN4y96dEUFow%d#6Pk#I$R>4&cDusl9W^z2RB5TVqY&yHihWicu`XCo-V-OQ026 z4<$>uk;}g=3&3XjrtMq6wxGGgdcK{znwGUZ@$sx3$DXy}!11AQ28T!v3?7IBSH2>Z zqnJqlaRrJ5l)(#QEGRljl2i$mP_oH-0Ld!AX)LF_mYbTZx%EZG!%Kfum_qgHMUoRA zuq&JRFadv(%1fNew`;q&E3Cq*$+?-Er@(8yE2r)Yyln~z0sOM^d%p~7!X<~j+l$3o zjLc8ty;W1i7$|7RoXjdl7y$qYm`lL6V89tGtPao)v>6kz0s(of$Ct6k13`E9rEua5 zaC{6KRpCzF@?)3DFE`sS3+F#)l8?dFgM`At4iPaRd>zHcq2aL|-0^bEEQR&AkeHX8 z+$cuzv8FCdxs0H!6fMjqx1b}O(GQBk9c^+f%)%C}vZOH5D@zKI48V8;lMx8b(d^Ry z&B6c?z*AQI(k~sbA(6+3<5iidDK7XPrtwvFCxUHvpJd}W{Mm>Oks6i)E(MV!fBa zrf&Mazw5iaHp`YU1d0dCb~TsTX)9G?E5%HE#xAw7TtPNn#iuSy%^E*;PdvC3G=J*~Bf}>8;-D&EDX*VTs)2add^rDl<`?y(4EE={wzlIm2y@DKgpP&EqYy%u^&SCNAVd zPUJ;y5EF?ACjE)xr&z6ZRw37ofi3!4tWrEmb~fiCDUgC6Ma0O(_4 zFNdD!XR_$_QsPOjSST(-KrDJU>}IRN;;hoV*U4b9LgIfWK7-lg{}UL~RR$}dh%YA$WS88<(PC_|}vpPRl#h(L?qb5g%+F=A|>nGqhMcQi@BVUtG9`Tc8qpvIoknz^t-n zT&5$-%kP`U)*Pp)rqps(A7sMsv8^Ysc1TX`6$1%n<-zLEx*JsfATIL^Dr;- zHUILe+0Cq(^A)S|JCE}{5A=O5^g~bdMQ`*+kMv2e^h?k5|4r}oPY?A>pQD(Gk2e|Z z$`^P4HriX_lSDo1t-)~56<%~xiFOn0{i8SBM_rl`98R%=XZ2l&!Z2)Zr#%}eJ-YyW z^(r;70NCuia$2nR{;}^{*D#BwXWRq548Uc~_>IrMjGJmKzw$?q@+!}omhbXEFZBG; zQ3+2fHDtr8vcs$bDf5*>^Tn4HZx9)K=Ld|<>I?g?kIi(7b;ttpBG2RbTHa5SS_*)m z%$m{cUc&3Hpv(&V!at$NfBe5Mp~kQL>R!KxJE74(0A@g$zx=>Y{RwLQ$*-X5&i&o* z{ofD%;V=H979k@BGX^p}-IS#NYk}Di1O&cmG!+?e2N3=O zxD)6uz`%q15(ZG{&!EEq5F;8ic+lcShXN5wR7en^LWT$}?sG`c9Z7d4J8Fb8P-RAg z9YgwqBk_@_2Sl}Tet39yLt2C#k*IpT)T1m($#sD8}i-@ZqC`0?e>r(fUxef;_LfA=1q|9}4i1n)Zn!PCv6iabgXL5W0q=)r;xst6AefS@E5lZ6SQ2m%=GwjOixDJTbg>UIF5j9e|wu z)8!U2%M@$i%rVQnOiTkhi;P^d&6E>srHBkm603R?7#Z8W`}X^9zyrUS0c!;331q`9 zMLcoFAzmESVGT3s)tn>WaKd3hqHuGWdv(yTnm&ps)m}|%)pK8yoe8U?SuA?lkgC15 z>4)(-+gsZOrduhkz4rb0c)`YrT{Fr4viPCib}Fnkk7xU+d;!KRG~Fj-3gNp0C=Kw( z8~0xE@5BEWzf$YQH~)O})7R-@Rs*PZX3S+}dDad`?3L%web#?+j55UEqm(^FB%Ip^ zL0H5(7*Ql>H98v6Qc}RGVGDcz5Yp|S<|eE)U@49RUI)46n|K{=UQ&ymzUWn^GdaaE z0}u-f&lD%!fr%-)v)iPswLOt_PklS&VGn(XDNZ#`hd(qG5siqQA(BdRt6JIVTm~%D zRSrY-gW~;yH9FWmu|clli07155fN3ZB?p|5v`l9Z*)6VyIU!278nA%d0R>#*BG<<} zc*k|YD}?{hf*#eC802}0Gx?Gmo0eAzGD%NOond5V+{DPb36D|(`3J)&mc%1wQj?qH zByRsQ)=5x?Qj`wMA%jvED*&cyLAhcjt}HabpQZA1BD#u(^tUW0PO%~~a?uqJ@;;Xw zjf`jH<grgw{2Yf?6-5*EOnxWY6^u?4zg1q5SGHQ+ zumqB$5uwCaE{T>wMpLb0?hz+M`4SxQ;YK(-6Plr#W=Vqwwm$k%OZH;h+q_9OvMG;W z1}l?N47Mhv$Z#nW)D#@ji7I;rRj5N1T=^1*PkSPjs6JFGYk!M{rQB6cETCBdzmMfv$vCw#s&>78DlOv<9YBD>jMD`!H^T|hZr$KRH z5LA4{qaSgSQg#XQnlgM*H#_y6Z;fsLeB-Ia z6TUd+agRMk&}P+Yw476=&LRYp%OS+9{>$GJe|2Qa8Ac(a3Ee~`+9HQY)LL7{D21g` z+-^1IUU(Hv9W&Ko;5qju&<*i*@loCA6;H(9lpCE>Ofb)Akf#zur;L64XFvzqQbrZ@ z-e_FtiXD2dCd;oBVfpBP{&z!I>>tdyib{jJYM`gY$W<0xK&6Qe%N`lc9C0ID!3s$z zQL}4k63IJEq1kI{7NNQ*Gp2rpWioM_=~28Xr{yi>rEvnUMm^SN=qMV|%Vu^vHyb!* zYckr+F4eT{OHf>i;XIV2viByImGBSyw>+B0WyH!@XDT zyoFqGT}swnqcxk?Rqy1*bh>3y3?U1?>u^rbdWtQ!wIPLU<}Aii;|XPw$Q<{pnVuJxg-PjaO#QT${A zk=O#OK_bdogF+s1(^6=Quo~!YR|FH8RI6GA1g_4*TkpZDT8;%1dU=VS$9@m*-?t9D zAR9YK4T6cAfmJb?5=pMA_wDNUtoYZ#{`v6{`m>-XJ&b!OX>XUc)-Vm4XzIOc5eO(q zwg~OeYB9H80W$DcF^8Q@l=j=cOXV<8QMIbc_TA9f@bKv}y5emuM_iN1F)|yCX=`C*GT(JSuP^JKFMat1_iXcXJpJoGHAZeX5+U6E46ixao?i=P?@=k|Q~ao6Eq$qbNc68i;y0 zfEbiPG)zNL;W`&oLv(RN`KmO2;*}?2D6k^C{E4kqshlA*C4NFJE2}6?E45NfLy>XW>ig0;8cp#VS)bb`Yf6h$GL6wxEaC`3gy zT%!HyzYMXNlKZC;xk3AZozJm7etNkB`l4WJ5rt?8PV_ruTEd>|mIi#lpqQ~n{2HDj zp27N(>WjXxxc@GtE3wLG6HRo0EIB>XfI?OLMsVC1^b5!4I74#8EE>d|E|R~}xj&Vg zA^{UJtMbFtL#>#|GKN4q4bi(TGel#%388qS3zM3ku&JCvA}OT6=W;%=$v_Lk7s|*$ zgljBXx1Zv(SJF>1afi621{)sb|clYD6RxinB8j3*~A*!AYV3=th&g zO029DHq1)lFh@x7O0Gbh*}H7WsjooYaw>>=;d^JfJw6;BZWJA`ZjofdeSE%WVHn=j@Gfe9qvKP8$*`R;8%=FXCHSeup6RV&qPdf}8UUrw zFKZ)#{K>3=Pg$eDj&Vi>OpNL)q3ZIIi<~^jFck?kJd4m9=7dlnb$%r83TU z3NU#LLazF%Y%?(au{4r<8TvuFFCs!4F~IH_zNlzG2IM&hv`yoP&%_hV!{{jMlf=k) zGlNSkKtfE$)KQ?y4hLw532^^_+OPz*Xon&VrS0fbBPB=63{pVV$V@8KjX4b8JFV@^ zKV1nqD%Gd)j7dlZ(WDts2nt4FJ{Ap!kUTmrRGVy!XP28V?<8o!dMHuK*Ojvay3^WX(4G86B3&YWn{#2 zfdfdeRn@SBP#mflbXl7JWxqi^TeU3?vz<^W12QREm3cf%txCycRo2$wG9tk_hXq%u z_>tlx$d9sElfqaK)7XzX9ui`yY}!Br;Ml%hj*`{LQ>YEpSlJY8Tj$hVcluQa<=k@w z-9)9qy1kWwm8xZBsCE2V1rjR*j5oX7vnE`?h;4x4EZm};PcuDJHrAs*9$EcTETe@uwcHXq5PSFkB9wySay+!;2PlCb_{39?Td!QPjkpbY_ z$eqBb0MnjhLa2GoG|Sz?MY_PVpxG!z5+1;dHSD|0QE-iU2iUV>WJMH-2L{ zj$=8VV>+JWtH9wLR$I?a)cN{hIDAn*E~Wkv7TMZEI-I`*TFWL9GTnm+Y7M?k1sN~( zxt{>x-|<`B-Q9~oQwv*+m_o`HA}M*h8zcAu2H1cNIOSAc6C!mgQXVW`JD2g-f^v9{^@sE&v7yW(FW;WIkqPPG)9a=3@@#0)S>{j^2dxCV!j1j=4D>q>05Y! zpAPB=*nk4~X`(*r0zhh|PHLuJYNu{$sD5gxj%uo&YNO6(Z~SL{{}!|ZmQJk(N0GL% zbm>_}{mz9VE&T#ABr7#{(2*fB+NemrNai;Y&SY>cVFg5CHtkKI;2?NW93dzG3cyIq zOR>T}Y{O1$#4c=WNHJ@O0Lz1H7Sa(Nfq)3mFv|`p2pE9N-iOU@Yt446%Wh%}zAFrN ztIgKZ1rxp>=!Z^`wheIY0p4nR-C^3M?XVVT&V)ZkwJNgYoJh+yr7@jS!;y-e>!~2N zjPxMGecFt+h=@#{&>P5 z=^2-~&?Iv^)wY&>cx?^-Tc*fZy5>^nW*5K{ljxzwG9gFVbEYLLwB>Qz)zNzc};3v3!cRH9yR^?HB@&u1TZP}@%P947K2-TxK5SCcYvMq88anOf2J~Wpe%NtoJ7z~mb2gVw^A7S!ms72i zrb#7YTU2ePv1G*Ap#8>ZcIha{NS?iBVImzmBPak24M4AICUYVR0BeqPNx%Q}%cStj z#BN+S#Lpwsi%6L*(Q_-$35)!ox(@V)Ok9puK(R0%a|{3uFmUvGfC8}fS#R(H7~5O# zb;$(sAa`?b|1i??)^>3ejQ)pXXBSM?@58vAE*4&~vlI{b2MR#1*v57or}ch793da~ zrCRe#>SJ*qcV5r*zVdT?I?#g{fC>0>K+j?<|0v*%8bsrOe^B(6wPtS5c5C)`fWOR2 zA9#WfdEhws^1Eb(XVHUL04OeYDsTBOnXY@GHjMB0mUZxdA5eY=d6Sn+U4PDkH+V7J zxFI-tF}-pwMZDW^?!!UxgBSv#OVR`=h6Tyr*lw3(%a<0&{_Qn19%uz)w|gwg$&}X(N4Ua|dhYo76W+#ZN;t zCdxp4{Xjd6qmTT&mwO$RBAmhuLHgCF&*^eepm>_#5+{n{r%mOBd0!^xP zDbuD-p8`b+bt=`WP&ZVvs&y;Zo)o_h_$qWP*|KKOG8~|GE!(zk-@5-rD|as4x_0m4 z&8v4W+O1@}0uKE00)-(L5MB}}5U7^JM>VcpKw-+PB>tJ;LcU0YAp)dGwFx{ zWLixd_US^kqgO8MXY_ID+O}`whM9XeZ(P03Q3y^Ze{MtMH+dAjhcN)MQ>dy;D0Z;*J z3jiPjXE^Ow00SrCm!E?V^41`P6FP+zg%^HjTybD=xS@v~f;gRsZEcs9bNCoAfCH7N z$l{7E!Z@Og$0b#wjW^;r7KS^vv{7S<#mAs|mStvHPDS#Vq<@DpNvI@CvCTFWLl*#$ zz(`N+BLM(!col(c3_2O6m}8Qe+izu}*-~*D!bQ=CZ-TXEh;z>QCUW}>u#Y==>WQbD zZSwWpjML#0-JpkFNmP!D_LZiiLP8LL0Sy4D(vS)T*^)>lH6YoepFVV2l%PJzpAY^8 z03Zlb7V=LDO_G%0g0zuZtF8aHI%*-Vq`LWMh-|L8)}I~*+pC7g%6Y7^A-b~<9R3W@ zEF2U$8-PF4UTf{M#laB)wcCCR?Vfl>yAQd-(&d~;D5{uZiipA+?}&;rT5n#vB6z7u z21texM2mrm8A3yR6stj7+=8#8NMib>!bb(*u#}P(=oo7Qh>*d0H3bksf4V&!vdANw z2^`5}k_arE#>E+DvdF@$Y_iE-i|4rIu6ZrA9d{g71jRuR!L>g3Dl{A(?mVrY8$Rm} z1R@3ihq+E49l<^kNS&u^Lcv>!0|OXPsG#zaUA9^;*_-xuJN|XjdknLN=}EV2ak7&7 z7DZa4p2>QXTpBG@L4p57mK7^^m;yKof^hV+fsl4X9=YU9={h-&9lo6M%q`DMY|NX> z>?Q=x?i_8N-~zogMG?4D`koz=KJA8BLkq#4(6X-j0MPaswYI-!D|B(q{vMX^oWo1} z07thaHoWRa=bYKo$J?XY*VkL4j!(jUWJG`$jt~cyQ`k0?dLPcW$D>-`S8J*g-nM+< zD!XxW8R>K1sCZZ zN+e+u6cDLr#6kaA3VTYVOA;@0kd0xgd)zCMPzrI2wiRLv79${i7SaF;cxxm4nN??6 zGeqd&uPJ>)+eW}vC^WVv0T~bwriA#$wXlaGXM_$-Bmw}9`3(Y%LkWJ;$i+h<5<`_s zWEU5Bz(*!9b1|D_=qgFNa*bEu;fbINL9B6H; zbhZmiwN`ehNDYsJ0nn4RZ0L@K>|-IV+mAnpNlXgbY-+0X2cEj+EUP&)n$Y~&5vys< z8FdN+6qw8wE+T{<{9psOxWx~IAOs;TF*}CSm;c%a5xCvr4}g=~5QeaT1vG$>T;U4` ze6RrwM2i0*5-Z~y0ASFA8uUPA^ismuNRdJS2%!c|zC%_(hyeyjCrxXbP6IBBWsuZq19jTD2ZFAJc+%4irS>$6 zG>d8bII4#t=&hsGtXaB4Dkk3~F4Y|ksoq-Z?oQ~+j*${hVlkfa!uTv=u9Ab#@+#K$ z0j-(6wPu*qQv^}@6>R;|A9>xYwelLZp_-6_NtL86uW49B^+_$vgVv-Lpv^6G)0^NN zCpgV{5e`6hn-u6&13KG)0esdcX)8i`sJK0P{bQu2>D~zoVYBYFCM<6Yz$`4V4V&UN zx7Pm|-#-fQ129Id0H@Mh1w>$6;~G>1suhe43*b05!a;B*MQ(CaFoD1oC{Y?s!EveU z(S$&txZO?e1S~TEEh&yD0~lmG*O@7p!Z*IV*}zP7Yv21~XQw_4mgiK4yh<{#l5j1T zCSjOVX;H*=HhU$3RS1@=deXq=+7^W&{4CZPn7~M#YM)RSV9q&MEgkMqh`&3w{)v-=Btx;|3TI2fGx~{cNQhn-Q z`#RVW0QRtjP3&VEJK4ov_OXZU>}Nwe+AH*Le*|Tm-vDq?YF2BoNVAFR58jvD?PwL-V_BX*n8}QB^oZy~a zIKvz6@P~UA1D(Zk10-(o&Pu%E8vmlkCBE5@b9}SftoX<=pv{w00OcsxEX!N&@|VAS z3y5m7iVXX+eK^2eZ&mDCCa%rNT=ttk7^jq{d=V>eld}l_>}TIKq6v;fF$SZ`Sng?c ztG5Rq|7*VI09e!(2m}r65fD4s!*2GmpS|p8U%T4dp7yu19qw+AJKg7Qce#gs?8A;X z-}nA(Ycv4xf)7052XA=Z%0woO+ce)8V!!h04FW4g8yp-h0s{mz0`P;(q$8bz419N; z0}xyamK!yYGMXp`U_8C0!VEx}je>vk1e{D;5bDY(TNiVjvxt=;02l>1!5oru%HUM;07=N3{s%) z(O~ZB9s({P0_tD^4xs<_pAS|R5DFatLf`z|-~E--5#}HMS=RlXU;KH15v*SnvLE}U z-xXd~7HZ)ZdJ`9VA^Le?7>Z$#Nk`79h`Bh+^1y-8fYv=RnUitAAN;^J5M2;b8PYjc zmZ6PCOw4641ZG`?11x|bqR7`Yz}FNax>QyXGGaM3qVz%HBTAwpQeq@l;v`yPCSu|y zUScO|q9;<{Ct4y5`Tz@-qA9W<1Ds+hvZ5;1S>h#LL-pB048i9~LT&ijbLB(?|DawQ zDVj%7ga#x)=ta>0pq_Sl*##KaEk;BHDB7zKg{JAEUzA9`u#@azqpd`MHE^RfpjsuI zTHtwOC5)ptZsV|#BRZO6Iz|9Hy5l;+qdE>-IXas++T%H%Bdq0Ps@>x@s>MA5WIhUH zKoaCY8e~BtMwK2aQJwIsxO)=Mf~mMz?*G~B`2qygY0!3Eq->ZDKpWWX8RmiZe|3fxfw zTu|zyH7sRQ1|>bxV>*%}JqDgYN~Jx*W2`+JIJ(+au9~#9BeAUoi>Mkn|Nb3A4x3V{ zkTkJhxI zWKhOlq*iE{#LHvaMu25Qo@Ho)=4hfNY2u@4nj>mLTWXeOYo4ZS zre;D0WCT!v5IEg!s)f_7g#mCt;>m_g+=6i7VrziFqCo&IdJGv^kNwP+186`7Fe62v z*GDV_1YDE|v=X!DxpO_0{ir9Oi;acpR zv*Av+*czT#O>qc~v~^6e#oNzxORYKExjYTIS|8&aLk@JN!19kAv2K@i9U_A@txSIPzVVR1UQ(` z;HZu|m`>JOxfq9_(kuN^R|OMY^-fi(=v3v{ zz@pWD5Ny8s>$?3IlQzv#bx^=|NWfYw(Z~*oy=azR(C$Qyy>hI=>S@#PjMDhdQ|Xmn zF-yPF(#am|i4l+N=-9dz)5NN11U+eWvTVcNShLuxx$M%r4(%@$>(n?Vj6>3TdE0Itwp$dgNY*rB!%GY?!A%<<6|A_>lxoJT01-@Krrk)3<`W6K| zsc@oC05~TA5F_6hz~mjtd2y5oa0#MKC*kY^=`EL+RYD2?fFR@#>iNxjl_&XJ43+{sGy~ggkDbJ662$0s5?;Ne|>`d=$ zY_-yr&P?xyJgZ!a(LlF|0>^y-O=|2A*PUQ7Jog^{>K;pUOw{NV+3N^W*t zn{DQ!vdfEn&5GFA*`h6Eeo6pfYL9Th?{%g^NZ#Xq2XQW1sUaupS<2|)>KkZ)2&8}@ zTtFWUfG|=T;b?(LG2?1w1VcE$KLCIRTrFfgW2WJ-?4^@vRjuV}@fM#1^9&W%#L$}D z?$8v?>|{*`neNqOkh%^{^nwnxn9k~W?klZvRTZyXjZ1~X>%Za{!4A)fsnVNFRhB|* zQ;kktA?(GPF7cf2U(K(>B5%TGEQcr$(IRgmSL^Q7EVsUF_HxMTy6XtZDULb~_v+P? zsxOGRm7K;dDdXtD@(j_g@{JKyF-bDZh0+qxCNKiyFT>jF0562n5YI5vasi8S`v&kY zN$}cyMoTz=*6Hd)Z~zA26WBWHW!9WEcd(0SCZXk6FvIByr)`)705`BLAz1_5MuZ?K zXJ9ad7G!`3d;qe#vr4ryY*<==gpak5LV+wX4R_~H2z1B5vpoADMF5Bf_zih-@v&a9 z5@-K$LrXMFfyL@7map*1g?$d`W)gfL9!m_D<_dyw^eNO2CwS`k1D6^jzl_jGUv|Jz<~=3iSgi=fDC*_Z;~$b|#|2Anh6NFM*pu&t_UM5hL^q7C0lz!0d# zXSqf}3_ubrngL*n4yQnAaYSs5#6m!J2wVzk_we0531v@oALbTae_CwYHogqmea8P# z>2$7zt`Z$5(}sK%t&vi;x=GT2&eD*vCgVxzT$RAytLJ2C3xS1(>ah$#cT>rbgzA-G z>50c8jSCTVS|N>BVe-SqD~e6`%YMjrqc8osTbwV?&)jaRrP3-PGU^DGMZ=gX!F26< za)x0pCy96WKG@2P^`e6nA4@t>H!oD1TGK&$D z&IIk8Ov?ZRcmQXMaAP&>{_Kc_5$m)yTEp(r^6wJ$ELulz%m(l162xi{4t1IZ2B*(G z0Xd5A^&x^yH*Yf{s>mUpMU0C@HR48%C&j0h#t?@T037Z-5Xebc zhNFd)fuOcQr5?G5)GZ3rbz1O~G=Pww`GmX~vzPh6BYdwF_Y3`sTB!f4u@F^PnGi`& zTU?dP4T(@-m9%TLn5B~#x^Y$1$dJ)s_e|@wD~U_RcN-*}ATCw#Hq*VL{4>mNlRPuRUV(duvNEl$PTzY6A7i- z#;*y5!Ev~pj)D>IETIrpozhARZ(U_C=FYCBSIgHU@>ol{>9(vMXFBYNIx6w>QG0Kv z!+6Z@%*=NwP?r)=OET7f^~xexfpf@Ls1Sj}HzT9fcsnqis!hxT#6EZX1QHy0a9{v| z?i2<%=nqZ+e-Rw+3qWx|fB^&iZPXYbz(0}yK$a9)66DB}C|QP7`En#mm@8Y>w3%{d zPMtajIM|4IVGCKG-ZukUa#!3&6ez6B4K)fgFU* z!Gj?12e$b@8o-?oBeF<72OA=gvleaQ4@UVEVn{xO0(57?hxoflA_BQvtwic#LaC@B zKQq!MzgkiYCAwTZ(lU*X0&1xNs#MA+q!@C(jJC7qPgO5;2e zQz$Lnl+#W<_0&7p27n_1)ksZ^x#_}-V7mXp)2O!rAW#j*QA3?>9|EsqPE?HwiZxc( za_wl=SRo(|SrUK!NVZp%CDFCsFq$e>X5DkuwfG>^k5&J?O_8E$l|9Zo{P2s<+ZgBT zZdw-eGw`E=nB{RI9=DB4~>-E{r4F<^u^vCS=3}Cq^mw_@09|MY#imIA&E;DD7 zmpm%~H75&;flrNAY|gY0O&|ja3RwVZEkKa#4V4C%8idYrB20u7Fc?fD3jmNHvbB=V zO6#N5PAu)U-B$lQ)3@cGo9?>6*Y{QA)l`bc&gZlk68B@72**4?q}q_M%UfZ z5#k50`^SYHQ{J7^_ZVZTpx3kKlxeIcVLca$r=@ zY+R!p@mxefLF5(cprtxGQtqRzqLn~q1%ccoZaaeP8-?ogs&I`bHt^|>=zJG2%7Leq z&RSf0NcqS9R7aK8liiH)RXhsCi;h!)%^`aSn_ynDr-dZx^)f^wh6Oc0;5#KA1LnMl zsj{j_6r_oMw<{6aGEBGp7%mlhx8=Y@Gcm~~PKb9>)$HSDh}A=WBI>K?Pk z2m+?VV}F82$)UOuS+1L|jrN1c9!1Dq$J63bA6dy@PIqDbqS*N4RZ0+5>qHjw$Mf1d z$WxNfl+T;5LmFb$tMZ3dBT9(7Kzcs5Iu@Au%jMk|tjoD#^CSm`Vjqvw;HOGfGChfz zmVB}loAEDz3#gd^b6MiXLWP0VG%tclBD4hOZzq&3ty#o5W3mLG0cYE#jd|Q-AO9H0 zIc)GmZnHV@7&lz86;FHKqc>rlaioGf;^2hKIOd3prCClQ9fb0S+*1C3}Wgon0k?R%w6-G$rb0%Q(n65n&CZ9g_$Y zfZCp^_O`j*ZEt_uEbJMT#&U(RCkkhkY7Wk zEDw%rHd&q4Mbc-dQVg7}gGrgc?$4P*Ii^x_{iT{2J3=S6=3=QcYKEvOx-4!wORoJ* zaQ|c4Z)cQ_ZTNP(-yQFH53Q)`csSqGW^j!&PjgrrI9xq6a-EJ%ohKox|(=pWZRlSw}>dPqXW9&0F$4)oCP9511#aG}<`! zGU|sWy%6^PzIiOl9*Q2l$?xTHUKB}}gjRHM<~hH7=K~4DIsIN8gNTYX_0$YUz{3!> zmRiE9zFWtfeUo4tcCK(`Nq~gTedL%3IC_)+>TVqw0XC)Az0P`52smu8r&#PzY_Ti? zBn*H98X$UFZ6+X~02pAcf~>~C%^Lopv(QHGF8>e%Gf)HThF6+{Ko0IYI-*)^>+tL) zxSU8_cFsBWCOEikBM78e(&z(4(BV|dU%F_`Tq6c;1&XZ8xV}h1$}EhGYq`GYbS%}#$MRZfa8M6943DP%tu<~#HUuwn zNGgpMhmLq7@)Aefutm$f2=KZiIeyS{LU7F(gjQz9aPElCe9F_h>p?I~t z=dMchjw7U2hq$y)fq*I@B+N``@#y%D5I4iHgsrd^k*#QP)P{$oxKGlurTY{LqpoNZ z6QfFMhD!X$D0(d@&o8eOYsI2MKT0BcN`w^~0Gc@NDsG_wuIaJVZlG>a0x7Vcf{`n` z(ks7mml$Uwc2J=Vt_6RERcJ#7iU07nSg?++%sb98S7v3+Fv3Kx2pz#AE!o98aD~>O zrCfFk^Ri0n9`0L2s`T8*k9;OEab#M`O8{|%)EJ~XM1(r(<>T1wtK`s>ASsn3=3mrB zUwCIT+p8iVO(J!q4Jqa{w~9m%2wjHDSx&GcMFr>-tL(TCLrm>}OmY}i4JM@`*cQ>S zM9cwB=)lxZDz;D}z|iA(<5emoh%{=i9z(4Fo(d?*6HEf=>EEa#Ix>oWrxOr{p^H&#QpxC2*=tgg~bSwv?#*Mo6dMJ?Uz z$v84PCdu;0^mamax~54Eg6hCs?Iv` zA^*0nur}gL3dl;0vS)@TdLRbJ4lwN)@;h51B0L~HC-W$+^2KT~hp-YpwbHZfQ&K0D zQnyW^((<6h?X~8O2E_d>+OjrV!?+BQ;CONZ+ukw(34o@pr7tFa52!(vm%L#SgC{yr|-<^29O7 zP9sb*OY3P-twO|X#yA(z0ooHQ#*`(}?$ovlO~X!h8fQ7+^es4SPDk;>l4)Rz0!J3B zMER>E$nz=7&MJhU0JtFmr8H6NM`0bPQykUX;xke!m1b+!WTe2X8FO+x zGE_D`M~N(QivW}GJR(E=W*Jv!Xpwd~S_wiFl=O^e{(g&D1n-PKq|u&aYGse(sDd7y zN|U0iH?k)oAO8)~m^BZRZ+IB8d4h^~(6o3sgdus)_7>@F`LvM8l}9ne<9?JP7vhv0 zwnr24NWtg#)b@_{G~y~}Jj%)%T@P`#6g+q@H)GBvMi#@2PNQ0uNQ4b)M0XKEj(pPXKv_O~q{(-_(*4W9yj7OGHuYvz{Bn%Y; zfLD=@ZAx1tU|F_O-ZN$))dFu8d$U)2c?d7TNN2&Q-CzldNTmjiD_02O`s5O|?utOH zOfU_kyljQvrecmBBygm~2$3sRiiq7@DZE_dztoU3Md{;Uu3spkR_FzP*{eA}^QRJb za90j{G^a`EdSfypqBC*TyRxu1sEb|RHR@bzihe123$gcLP9XV`#5ATYi74skpBmnkUeE+pBB`3)*Bh>$e{BQ`v#;84}5QG@}i^j{DzU? zjf-(Aklqz>;)BXugdTWxOO%=`y>9cBgx7Sc+K-n z$JYCJ(@Om{OOx0tij#Cj*NwXt!zKV~c*&eML(=|%Y+u)&1yH=Wm}Gs%puPl4!dTd< zgeXNfJi~K%(U_O<)~G5ePdK0-C_p@q3TL2_b4iwt{n%u@38OdqsYIv(cdU>@TBJw1 zG=wWyh^4}ai&P+I-~TvrmDbYms^|sNl7?I3S%w3Lcu0HXRXFK6 z`|GJ^xilj<7!?~h36_Pb%g8laz~6*We0B)fr@33$P12=M?9T{3cVjok9E!qRR?1w>%^c0uyv;FjbBv75G4ag9+|I*%JR86o^t=K1 ze9!&7&;R%Q&jmfu`P|SO@W~B*&x>i$6TO%o-O(c*(kETgE1l9W-O@81(>FcSIZHl~ z8`MKx)K4N>N`-u%%v4g)@s6vaT91j;7p4=gUYY|`=S^e&@XjnGTPHK*B$`~&C?Fdc zU>Ntnb96;)+4#WffCmPK`wJY&=X>;vi;nuD?F)ese7(p}-g~{^y5&1A1QQg?{K6zy~%! z>3iM>ZsF;pp67jl0SW->OQIk8!3VNl>$$$`xqj=xKI;ph9}2+ixt{F5zU;Gp>)XET z-Ja~XKJM3^?CYNH@80h79`E;F@B5zb|Gw|B?bAn{@C(0?pTjtWqc)NRTHe1nj&}EM)y&1*DLnQTQ^S= z^&dQgV|gJ=q@MK~NZ&j4iyiv3iqEpTlXmCFyKh7pW@6+~hKmC}{OEIjRvTf>@Bb3d z=jzoax>mb?)Jzsc(xMAP_7ykrSxdjKoToSd7_lYQ`IkpTJsKXzMPef?3KqH0HSuR$ zXpTGM9GphSZfI4V?WO`=4LN>zG` zXV3>&p)!pcHK@Q1KQ@3J8@8<2up4fXUAxw-Sha7(jy)^atlhQ%S3s!0c<<`X%XhC| zx?(+491L-=;lqd%D_+dFvE#>(BTJr4xw7TUm@{kM%(=7Y&!9t7_E!+$fCvNJMR5PO zkMsc25%#f$v--5_gR@+ zKk08c%a*;h%SNd@KzNqd!;4=ZzPoH{TbvBb^h&Fok957rxJG9wO7zW_6;DLinArRPl~)@ z$XaRclm@|zv8AS>eYM>4eDOcQ!6_&Q!b1OAh-98cxxzLFPP2`YiSSDnp zlnX6@j{^&YDdqwxiHD&;6$VGqZx>qF(234vXP$>Da%ty6#Z`!4oaHG;r9l6GCg-O@ z1SMoBp?9j;*hXJ+R7s-_g!CwwUM>`qOD@T2z^0rUfKjIn46tbf)`U9h0Hq>iK&cUM znkuRdpeCv{Sgwky0Tgf$s%f+#1c9#u{7Rd!5fqDBu>-&XfwIb;w(PSI445cGqE%}x zMTK3f?Y7)@>+QGThAZy4$?)=)FZ$n0?oPFXM6v2|o z)o2`i?HMFneD!vRl6tN6i`{eR;aA;v@b%kU#3N=r=bYcg^q;{7Ke-+RC=KUe!{N=T zAHynJ{4mENv-h5U9h2Cggd)2aXG|o^S&+*G_lKQ*46gg;z5DL7B1@(A>UdyxeyaN# zyuJxsAHeS}jc#}UNZArgDjqzjg$PCsB5*=p37(%l1n{4|p* z4Y_7QdEyT>R+0e@;NuZT11t~{M_z`x zl7T9rM4U{l4^$kQ52?Por7sO6ZR^DOgHi*!E+p-M!22#x@R-Y4yrSkN-?j40H}CxO z|IkM-{q)p_OVF>rRsCYz&;i#_e(nqhsD*=vPnW^MVCGwEU$cM9=sb@j|na_42X0s=O>}L}ym;oQuo(wh+ zH%k-RoJfQ*`LyXah|61zvUfhx_{eg-!I0EQlOi3VkT*y=O^Q%+5Z|P2Nc~Yxdg5d> zeJR8{M2phf1cx}p&21oA%F+|L*w}p9_^xg9q`)7ggB58AbCz^>Gi4UW1x@{mMMQnvq2jQX|zE30@u@fZKvIO%c+l zHSz3M`C@{gY^e>3b=>2d3?)S+(yb(6!XlZr=%LqnkCU03NV#5TcCz`{K%I05&f{u>x z#V&$is5!Jr5e{_mbc7h#Uu(rASD^Ug!{e=Gv`9|L-FPKM7fYT*h2|3=zazSKm}HCfKLjj z1q+9q`k)hKCXAp70jj_TCzD@6dnlb3h}sBo@X;BBU@=v*5_gKsa4aj)pc{r;3x3vN z0`=aMI+_}L!OKtLc}+r2y}}wH&1w|=jgsoK%cp8u8)DhE*1Y+QLD+Q3fEtId4-2GgBz|L5 z;mB^Rvipv4%cCk4Q;7wPBy@e`$VhMs)Sf*upKV2LCljeUzz*cD20vvPzlho-UlU4enFaF z4uKN_*m=O5pqMGGo4?IZa>FUJev$UmyG{B48;nTQ3Qc#{ozTmY6d*}6eVzlnAOodK zz}os8KOL%&JW<=ehUlz+F|;qcR;VMZL1%`3LK1D}|AiqOJ;#RieHUqp?U40R+NUc& z5O@;EANY0lIqzw99?N5pWJ58E2$|xL#>;Pu$6xHW0^4&OGj){QKY!zm>!<@;k}C#5RBEpx7K9dwo5tMO~>u zmm_Dc2h{W zx3gEA7g(Ic4O@U$;sYEpu}zBPQ^+S|65@OjS8)875{=l0$0rjN7krN>aq{+I&EtLK z7mA`dilj(Ai6V!tS7}PcBz^`R6B9pAB2ZN4XGf=s2+~p~k|*HgA(+RD6GL2#5H8pwROJu!ZrR8XmA#cEMtsbmvlu3T|IOqxgl$=7B(1C zTr5NyDAaa4QYbW}Q0fI8?6pET5^pQCLwu(;uVp5w2ybi_U-Gp?tM@oc2zilrkCTUB zeq=fX0zMm)T@A^HmNI1vL1Cs<&>icVcT8hXLt^`$&6- z(K#$BI$#1;de{*EM15@)e3H03y+b6AXp>}!W7?Ni9oBqqG9q$9lt$Ty0%&my!w?`j zRoll{-~x)JIF(fYS(R3~8K;qT^#65X?Bg9)^p_aTjrr> zqNYMLxJxexH5alESwk;0k~OWSG_z(Texi;kco3yQUJ>w*DYt$EL`nzcqplEDmfUNQxctnZ@U*J+Ba})M17oOe8)FRlt_t4RZs@PNtxn^ zZjwoqh)75Oc8T^`loUrwXXZHA32)nXEmC=v4BDU$YL(N1isu)5@PlGb5;;QWXFC^q zXEq+*v0w2qi^)WtA#+@VCv@-j$=HupGt-g%oe)i#OZl462;W#T{+Q$Z&}LiuSt9p*O_27jw_T?Zq2 zz9^lUXESF4SXHB49LI(e3OV38k|73%;kli3Xc)Ztl68YqrcK}&9L6^w_nC-$ zDs4FbW}i!`s$_^hUn8IeB~0!^iJI7_lQ}q+3PkS4ah$QB5BjUX8m#8xtBzrQDcOdu zM``O5I3~3qY*IcWT2JSPd6+11al#x+C1=taZn~u()f8^LHGwN?jhJRZuYp{_DS&_$ zfNL2{n# zbAc(EA^mwNhJ}2d_j$_%A0SnMGFh&A%54PUY7b{?+xL3yCZ1Z^dRA4E6sjBzumxNH z5GR*f5uJmF9^tYIDJRm$hf#}HoP>y=+DSi&v;OIms5)^PcApB@V>;=4&gUULbFwdm zvPei`=0+LBDy)3lw|)y6ds`7%DV`E)si>$h7v^VcmVSu~LejxNk1L`o;Uhy+M|t8S zJr`RA@|fU5uM8P#$wi}e=|4b7Pu)m?r3XM~bAh{vbr7Pb{pz|)cduLrm%*VuPE#7L zNM5|bcJqQHXC{s>vOPPJBajLB)lswHr9vFNC^49cWsV2(9G*p+=tQ zUyoC%Pb;afhoFReSN0=z>r=HB0VW~gB?*~e^SP7|fWT(!e1cVpkI1%Nn~0jYWW85< zspB2Q1v1FtzIw&ANVqn*`g_>vlux-Bf%~^KT*Ef}84;S2jw+i?n@~qooLMC!=@&H4 z8XtdhX2yA#R>Y_pVq8lie#E4i3xR?AotFPf}bU9AK$aF>=O@lR;UY9{YdAb^u zfc@$o=<2UmGQ4evf~+RI;{}5&QiJxfHK&HVTzYp>bDd~-FB)}fQZqFNyLUsPIxy0e zOj2e%`%PyPGLPmaVcJ($)kHF-Y`8he4N-4?L@8uqG8FSM>f0Nsp>SRQ+rBRhmDwhu zNz<(k@lJlpX2SLz+ynu6l7VxmV%*ELCArJUijuKcaTTm9?*UD3*Gpl@C02`v=Gn}q z!$}6vJE!Ww#|Kmf*TJXy!GF~^a)l{xHJ>0%O8-e%#$&et{3b@#dfSy^c^f4+{Lb(k z&zh0Lgg_EKWf}%qjq}_UH z*VRV&s*Iw09b(5Js_0Kw_h|QZfnJocSAy`8BJ3e|WrMqP#~ZmDL-1%K*J)lu zdm72>8*P^(4uyB-wRbFQI~~=y)Jq-KYmvJ_rXY(Q+e=kRdv1#Vm&uykC1!>^|QBJR0p=YKr^HidKm&%vQ!;=aqyiAhbskjDX zeDvdt{qa8SW+xPp5f7YG>A3(=%MD{2tM5CYJ2{{FsaO?3MH>B6u(OtGHBwRN@)|*&J)9OpFd-vmT!wwu2rFpw*c(rQli@OJj4HDK8o}l+j))VuRW#~ zqQJ>?AL-L1gkwS%Hn}IUXabEvS7RYFvoYucBN!Bo38cB23&onNbRbRAMcY7TjOdQ+ zOdwOQh4#e!*MT1WXpD-aDJ?amY^6WtA_&oK z4S^vra*SX6KG02d_|dpd(plLXA8RV(k?KTI4Zi*ur=kPF2ex348-PJ~r!VtY+Xj^2 zT-tl2f>{;fIrFmsXi&N+A<*riws=C;#^Nn*l09yccU?zGt>eb09J|5FX*BL^DC-<9 z1%9(UmRNWp=T|6Tm0Ih+J0#fa!Ye;HLnK8GZcdmMUSvf-{v8MP_w0X z98oZIg5YJE?wB=WR$;pUAl- z4@ZYB}+?<{Q89_}rZw1GysTawlZe=yM}p zq#>(2L4pT>f?1(dXLw3QvH-2}LT~8+rIAV>(M5Xm0O4+*Kz9NI_!|%a&cT5L2{wFK z&|$-g0rmw*n9vrQN3yl(lz{TKLbUzh(0VP16`Ah2MwUL6lud{(l{1^5gsz`Vc# zz5oU1R@~T9wBC_O^PJXI}e#`Nv4Lx8;<`V2v~#TE|fI0 zLW<1ZZfzWQ>j$V%lu8fY#W++@J00s9d$;dL$x=pV4YRfta55d%UNkU zD)`%cbt)q`h(+l5h+@x;vKe&}0_~WJH2?p&nMYvN&En zW+{1T6}TqjlRm}_A}x$twEluAUzrRe*g~Z@F{P_R$&YF)+L}wsWv+8U#+;K0q7aQ3 zCqaz{U$C=hTZ|SbrCsPkThyXiB*eUd6s$~`j2*YWd6v?VGLUki=9VDxB^XkLl_I4dM%_v^~0ueg1I2kyGFl$##suQnbMygiTs#nEoR<)WVG`g*WZUpPqzB)&` z(QSqCw+8o$Z%SfyGfQeL%{qCBN8G26;#KPy_z)dhM> z8A?-zlB%y9%eB*+x!i(%}ifH#nQKNi!{%2%5PG3k{!vDLh*tUzPza~ z)I@SZunS+PqLs(yY^OU_4IQ28)Fu3b4d?b4t^RG;pKf~gKC6|rs20^W%QP697pUJb z?eHv-!cLh7iKmK`^vM=8@O?n&(LP3WkLSW+0CM}Fq6oyufq|0Wi)Ke3X`c!Udj z)#L6#<~3Ra;l>syFqxg@RGvS55PKOk)R$;T#vi=V#dnNj6fMP8~@}Otuhw8BSaI=$Adas3x1C{sWTFDHO z^(3VnNevIHC{i&U&uXUR=h&Q324+2>A&y!07FTtxtS=_b%Sz*(LRt=H@3No%wDL%c z5x#I`w%kK4+xg-YVa~F4pWfZ{j)xJs3k0}Jf)?=jR^K*%6NifPrjyizKx-~{;9>!G zSuZZ#_yHS!!oHYh_EQ=294=pqWh|Y;TgN2&)HU?^r+b;o|7&$$yL{L*7}H~lsc%nS zIpHmAfWzyp1k2Iv9$e>!akCz}Dlpz6ZGcB;Xk1H5Fp?)>&y}xVS0))=xsiKwtS?9b z4h+sX@N~EkfyP9mF^H`&G0==*jHB5<_2X0X^?SP(RTH7y|NjTT032w5Q(Unv$w8-QZE!kGX=zXKkb|Jb_@bRK^gHZB^Dg8C?LdI&Nj zGj4*lY-uE*o2!p%64rR9aCsZ70g>u@pL5y6voNs*xD5X=k9(3ZI*X*fnv1cBzn*fC zrFk~#nj$1Qpq9x%(}BUG+Y&UJy|4R|@Ut9|c{wFRw{$BPG7&qz8k86^ka=6AIm{AR zQ=O18pGKU(!O<|WATuUeDX&|+R@(>i(~CKwC2Lni~vC|eSryN)DwU@ zzV^uxj#Dat3b|^Vv40Ujs?x@BG{qnpz>ufGs1 z`?`?axDfHVyr$!@C)pD8n3*dBuS9A(^dLK||9gu&ij;!juNVZr9Spzg!=O@xEUbHo zY?B>{Y{A81tNs$MqX;G~DT>vyI}VB-R0<{Df(n@ILY|l&nn0~*+6rY_E#i3`;&Hs@ zX$nXKI|pi<`)^ImjDG>(iBx7^l79L~(kOT@x!lj4q4-#G|;bG?a|(3L<={ zDW4<7o3S&fTMfI=kjZ(CREjoeGa~!DqArpz$vX*&n#w*28GA7cw1g{i`$)fmkFfKY zdCaJEYaa=NjmOik0U#OikU*Om$n#o@3j9RJkvl<;T67+ieM=d&m%*!R+NQ%0P#bKzA?2SDu zK?Q2ZC3_>p6sX#eD70WT6C@cC|D40C$d_QmNYJsxYGN7$;uJv>u>48WE!hGN2noje z4UsSq*zt^)ATjlWAZk1|=gcu{>k;`ok=wM*0@^Xb0M15r)JGjT0=!Z;g2z*&xh8YY z=0witOwLJF9Kg9FVPvbETMwH-AeA`CsEkPf6Q5UdGw?%8%7V7iQK<1^#J^M|NkS|Z z-Kq=Yu_o6_CZf2bXOcU%V=b$o z3g4ST-kGhs$Sq>gPqM0#m12uFv7lSKJ^B){oZBAz8m@iFJO!*x8%s>%zlnY5AKgFz5{4%`a zs>Op!zs!iZg{UcYoXsG~KSOmiGflUVdYhR1lA(#z=hpU&mkrH6q5*S zkG~-tnn})jOxYy;2QdLt72=~3M8OUOu*RY!FTs$M(M))QtMP0TnsGOR7)>TUJwLm_ z^>w9*=+hHRjTTE(Y;2k#5mcw5w&}`Rtu^5j|3+aMDPiZWKd-8}7d}pO%~a(KVXfiR zv+~q;e6o%Bi=Q1VgK?9)-76EyTjGjHKe=G=@+a7XvvM;fP&vOfj1&0eSkv>C#1dRr zTTk^IMaadiaT+|Js7&Pn&@tRQGLGEvVHV!;iq?{?y;CL#U6m-jJ5d> zR@szSYY{D}1lMee%7Prxh)LJ6s3hi1jHKzqDvHs;=t_0Tj0U*TV`Ipp(a{-g}laS;XPy7Png8^Cgg+C>3w9rYA?&zq<(U7KVxx+ND33-pl z6eZB0)f&n$IuptHE##$KERP_Mw-N|r|9s2N$vu0@Pm=IechIT-h>c-|TudyI`a|V_ z`ok19=W|Bq$Y5c~NU9OS(grL{Pwmv^{93V%=NOh}kNLM+e8fB(6#!PH_dH_PkRdPA ztNEN#Pja<4TPs_>&WHU~DMi?6VNb*j!Ft*xizXWTz#a8XqlY;(%U!KxX(p0($y!1d z(M=Ws{X$*=uD$C}F>D?I^`)<%X_ua!n#Ry9oY15A2o?_B$qnRgIh>_jUx=y`H{B*X zM6PS$95H#P6(MME>_%PNwdP%C%2-YR@Q7LD+;}nAvn(AQ%1)P|z@X|W8-$&Wz8)?* z*NU}`^7WJRy<+o>o&A~=jm#Sp|6N9dU?k`aD+Em07U2K`xJ9NWM)7orrvqGWX_%YY zMShuLBW%O-2-<3XGh1X_e@Y#)n^l2$xDoEPxBeQ=BdT#JmvmO`)#lA4mCdsqTNp-V z;>6TT&9yAG?Ho2h2ZSRTDy}wU;@EL3#)jp~+z9U(S>dq2B`%YSyd+&Etci#zlU)tW z5$j&1I*$z2^i`L?Q!Pc?XIOCt-%Gn~GR|$%iJc_-W2{oSGaXQK52_H%+ zo@hnyzjG@!vR%H2R=;2-c72lLSSdyBOegt3B$FXe6xMsS68`nWakgstaSX0JUhT4s z{b&&VA)3O`Gb%2n_iM_e|M9j2S?hf{?uetB+1oL?{0D=A!C}^7?TKX|Y+p7bA?pjE zv0#s7PVSxU<-rDK+&B)ly4$uyJ>qqxyuGaPxFEw(4bucj(tvU0em!G6YYJx`nkqI@NZ&?Nmgg78$c0`XkX9 zw;v`Xhl!(q##qfHl*^7tdxuO@t)K(`6hLc;*LiBx;FcxnV59=)u%Xy;cJq~Id8>kR zU`}Nnez_r+xw5tS8LsE)qV4D0KR>rC1d_Zc-6~pQmib%q%2tO?%X=o2(^liF0I``J8vDloG{koDZ^4nd(?_h!c(UB{%GGi zNuA`%v2goV|KXre)zI8(3e+lf=tzsG%!{^*LL{qZeyi2-tREh%pH-%+eS5B#9s0iItN>pft&kLhJtk73kn6ZJuZ`j&UXQ%} zQn{~#@#bkUeY|NpF==HHYdiJWCgs}M2KVKI2WAVR@t!?&!HkxhmCE{XuB^*!j?l9K zkmkJ2>rtF)->!7%7ceU(dMh5Pb$o5wgYlHB7+`Ms_?Lf_&p2W??HjJ)c4WZV&U5?) zhyeiv3LI##AVGr)5e7Ku@FBy53L!RRxX&E~f9?WAP}uRJ#)}u-i3~t8z`l?H;Go<% z668sk{|4dsTd9$xyOaLll%#1;CeWNOh2kuE5+~7=?rQoRpwS`Hp(2Y4Z5dKyNUBQr zndEwuYs0W&#}2qymM2w$PI*EYuy$WrcM*7&jazr1)42j8xFxu+E#8Ap-5!M7mt$SH zBoDjw*O8}QxQ5&A?CUOZUA>mwDy};~?tr<5$BvDFbFaV8e+M5V33%{jhY4A}T#3@= z$%F!7a!uG#w``6z_Xhv>kMzWf9L07FfOVtC9#b>w3gEO3CY`*)@WoBRBY zROMHk%5VPl_tfnBur-xFuj$hH{rgV~hUKqZEO5wCXPjGFEoGorQz=MPZY>SP9#SVE z|2LIEI6c)-1Wdhm;8!-;wi{8tjdvSM#9b8Jf7+=y)_?=z7E@F(ZdjB;7iEOfjR}#Y z5?2%oL>rB8P{2V}Jtk#ggH8pIqi`<8*ke{*Wfdchz(ph&SpcS}U5j0M`6ZZM1|Vjb zWtw>=nrW)JCYxlo`6irk$~h;Ub=o;+Lu#%V+?4@Z`JY4#Eku!^feu6ID@}OMTwOt5>0&>_E~|d!NJ!+k_~j(WoC8uS!$|X?iTH~ zjSVm@US|nbVz$thTW)H-Vk^?Pj#UdSwq>O@?>=xe3$0{6tw<1pX~}jPq`vOe>v&83 zR{BvvWqs}D5I77sc52QQ~W1IhoXM; zq^>(0sp_i%Em=X))Z6Mw@vsL%~IHHf2igaU!!yjp&fmcP?BZ*UGq&9;? zd3{syM%I>+*^==a8)S5WMn1fY!PV|QdacCSTZGY-nrNP(^yB(!ar7-g-RhN^WYR^L zXvqZ*@LLRF_|li1h;J>|;R$4}QyCRK=4w$ZiC>a*5Wu;LCOD}e^v3qKnKOqWhDpO1CAm zWzkEc3f9^zHY4b@q(fDjk&Plifi3WASX7bTjXa1E`*^QYBH>;mMYW%X^zI?9;*)Op zBPhsK?sJ{{Bq&2E%2ATClxyx6)7h?-tXrM!40Srv#nN@Nd|mAt*Sgug5p@If zWrYm+5}Iu4iw1gC2#@(PRt1xa!Kw;LTIC#?d1Y|vc@S@^(h*dRkvH8V92b2@N6+MI zUf+U?X>8T3eEBb1%_i=CUP%hks9o77CJUi<78zVz8o za1qcgbLo~tHyAH|t>|7L=^w!VY-J#6wT4-M6A^}Tw!bd2#5`jY99I^`k`#`JRX9YE zLxy%QmfUP38-XMoTm+!&V5f#F;tqFu$ek$Oba>9$W`2s4wj)C4XVpBBl6Lk)tQd8w z%tV}RGUXhaezb~nDRAGn`Rk+65&<8iuUYSEbU?R0L8aX1Whcu-$riM=W94o~mpX-bu!6YZ z(Kq>Xt3KkPw|H8nf`6%0wKL7tMx^@JHU^n{TsK*A%8q820Mj za!Tf%lKIePXd$7D;`WvJxT-ev-LEreJT#*skykQaQcX0%u}BS9lgVLGSU-a#QH?dN zT#hV^lDF1{h^M%?nc}51bmZwe1W0%FuEQoJTEDuMI!!d3@0e!PLT+em%Sp=}mLG)1Gcorop7j*t#;bro*jn zYnj_yruw$NUA3uC4NKcnq*VGORfgq8BZ$Q-MBHw1PQcbDi6lpEoKpu5_|n9c$V;{-IZ8Dx@kYaco7`01nU@ z%VKJzMW49bXHMnoQX0wt@YXUs`r2zw0yD7-9ofEN_=93rxUOE z#WP-%uqC|Uc`N7JTA7x$rM#*)$?DyTuGNT2H5NJl>Kdu1Rccn`NP@@GTj5!{O@geY zyyr#ki8zmYmv++=t-P&o5O_wyJyAIM%ebmne0a$pejS)meS1bV^$Qt&sl~4d|I;tf zVjzTnX~4t$7r(J#4E8(Iz{Aj`G27R7_Z0|0Y)N>+u{G%hv4j@gd~~i#EeLhvxcfpj zNdI8}t4Yc^hZRv$&D0F9h=*{5MN?%=$(73u1>l&7MRx?%cZ9{^xQ?$84tG5V0WKE_ z(FT1r#G^6MR_qycCoS9tc z+M-O7GR91s6#&x3^7-h9~MU|1t+7Suou+_)V}UDoq#qBmVyb%e}93UwSv6K842q+NQKJRn z=8<3+b&1Xx_0KV(7;-$0rE!l-Xvju1M{_XbeSFML@e~A(#SZPz0s!5H3=PMm<1=wc zvnj=GkeO|m#facXuEp4L#zakA4++`Kjr`n63EAwS)_|U^~JP=hJ!Vqx0wrOZiX|ST)R0` zw*;eIK-BlC2utl%!zsvqa+JZ*-=EOn%|INCGNMo*B|)6yq@fPqY{W`k-JfOMiAbl( zsRfRu*aEN|J~%)F-~&;em4$%C@o>noF&k60(l(Wo#vtZFJ{Hh15zlcc;!Ilso}hI2 zA+~wh7j;=Si3Au0rV%}*0*Yz(RE1Fn9;6f*E&uwLiK%1>VWUAbz!p$|tBf0^Afeer zaIQw|KADF}r4jfsepM|zMX<|VbIlzMH%_2A`> z)FsWpY>fD&h)P}GP>GF*hguoloZwF|US{;oS7U01U=&bZLf00&S2H!>xLMrA&7pzcORgiT>bf*GfHsr9a%6Xxug5>cOV%-sqmqoEO$f$Sd| z>EMx?jb>=Yt}pwxuTF{VN|tZ@&S)79mvA-Z>JsVbiDe)1Z!T%94w{gs;OhY*CAL<@ zA>u2+3dMsc?4k8%HZ@FJeaf|}oxna;1uL+Z7Hu6>4*RSVFVY)+(VGL#3q1u?Jkpba zU8Y+EBjbW5xZP8s7GubKU%yBnGylS-0EI@qXyu~3t=n43elksPmQYdnA1GqnY><$r zBC&4Zm~u1+ea#Hjm_f%oQ$nHK*iWx8*So>TJK2Gq&TIj{yoLafdx+% zkrTz(5{*zl{+#{F4#e8%KddPe+FW48ptc4_DgtEgny4R-$f_tY2~lG(X$_`KFt@P8 z*fl_=5?&r74fA4(H}zw)#@$PL6W!gNc(vU>E^uHD4pHrm>@Cbpsp>W{a{a;@{>rQS zo|gL7GA_%B(Xb_xE+xZOnf`JtEsd=I{xbdAR#@)up_~qjJ;sR)=pv=*uC<3vM2Uo; z9-wWqQW#G+&dr@=uSzrz|NjxDDSPw4ddgyMPFe5;`_&uS_y%LFOJgWdJmXKjd0PXi z7qp0`g)tvq9GKT0DnnUDTNp-z^;>0NpARRmg8|L~4{^Y}pQ+%6+oo);HIM!UuR)ON zM0n7Y)n|;&keay%0n(5SVaGh`(2niQ6cy1DHcfnD?&V%Zml8+ka_J+0x%u2P8#Mi zVHY;11*tAIELra1kpIe>tL?JJf-En?Z>gPZQRmi~amdoXKL?(cnWrwA6Grd}yi>ryVIPOia1TT{^Vlh|Gj! zNzu?u3sMgiNwKR8hZa8|$=IV%5tZ(wo{IKavAi)6iQqwQszlBmKW=B4;tc9q8wESj zc#dUJe^qV&LupsIMnZJKiV(si90!Lnpc!!1?Jd}ytHCA7Bp;Tky)cE|#=TW+0>I7kvt^KpSMd+3^s3FQ#2#eDcUZ13GOm*j^MEjkI~_EoKE&~syk%mAroXlx&Pl?4Q_MSwBwgTa@CU9AGasbo+F zYtDsr|M0`L#t?q5rN*wq&D3o=49|*AC6`zLB4GcZ4x}Z==ei7ZG>1c)^mI1vPxX+? z4Hc6W3wJFQh4@a+3T4fRbIkw^z+4JeNs;LGdJ)A3Tcc>uY)OEQ<$W%lwqCfXhZC8@ z*&EH3J^~*oQB8Me6bpI+`D8}c9(wZaFn-ySpBj)mr7c5_*&C_+5M~hAYx_e2T zmMX5yTPO@ z25>+Fbbg$PxKY#5Yk|asvVESB;D779VV&wu7aH){-Lt_iNy-g4YCIkfv%@CK#s_G` zKR@*U-^r|fImiFlt&r&aRx zxLL7YO&4Wayk^tBbR|a?i`TkbpykWg?W$9~^9T)u7^GgaeAe~r?E@#yoxkzp|IsqQ zmo31O?77k0Va-6dCQC51206pbGutjxFv8E2i!Q+cNkF#0uvXfzrm^VyEw~Yjx=+90 zN+eFd{p{=NKAQlHQN|f*tkK3Bam-Q29eM20#~*augtntr^Id= zXfD;h@@p>e0E?@pF<%=qI78za>o~(uDoibA16Tl`1{^TAySUiB%|Q$a-7H>r|1@wm z(BQfVD5wlHug%re>@#23JR4Xt*DQ0h+HDISsJDY9G!)tX0Q}0=i|fP~Lq6rZ_11|? z)yYohii0Xyi{q+~*l3+KP$v%41BwC;RDPLFVu#JSKmKmKmR~-P1>tgE@q`bA>HWbwqTy}W7BvB@sm?6c8cG|{xt*7-=Tlf*K` zF9kC4>n^S2nr_JlyOem@IBrs_ENi z!`(Rm$I^97al9A|ooLx!TXX$+|5u$31%)=_;~cvX2(WAsY+$%yP@yg}|2}Ksi!Bx# zj&CjqD;WJpK2Lehbl_FJJ#9x@pY-0#USgI}>F^ zXK|BX>JW9b{Mn0twt3CL9HbfjA#I73D^$=plc2wd=p%(QO_=tE#VrADZd!v9-ZscO zF^-XpWi;a%&E}-PROxP1n%l6x)HN@%v2SmCBbfdsI5&bVOzwKzKyC#ZteK^0`nnZR zEadLBBCh4VXkvxC7F*cW9f9EniQswPaN^gW1uVb-<}!dy%%T~l1x;QaV$jmu zvcGsyW-AfqT2%U^sG=;;` zSNcL39kvvwO`l1VG@A!eV!>=VPMI0QIFzfP)Dmad^x+T1W}})!F_c97pyvu1)S$)2 zXpGF?gBDaFz?o+*Tco3x;^x%1$x(Y+bDt@(npLfCm8(ILn;kQ_$FH`Lj=#(!Rr46t zvbOP#U?po8n;N17S;c~6`eYQ15<|+7B{EUG=yw2FPmC6;|D;X4SM>_IM6Q4cb06}D z6GtheQg(Glv(x3NHUg4YJ>(!WJqhe$*SoqAWl~2OW~V0Ak%GW3cS~*7rgoAgVI`At zAjyd`5wa8!S`V7Psu|`=xY+FIwJT*YCrQ_+mSj0LeCVqU0&`<65d8;SCjsACZbCoD zEWky0HR`+wx+c>h=yHcMOQ1>v9)a2`paVk~1En~j%Y`#>5iD54Oyd`bok@k`I*vei zi`b26)ROaQ($r#D$vS;!ABI}tnc}3l1f92iMKLZ4M`uH2skCJ_?O}degwvb`QLst9 z*Av0J8Otaw#eSU5YJlPw%1Kv5Si++oajQqQN_EFN{~j@AfgEHZ54mi6>@ir;I@Yh& z@s410@*MdXI3>evk4=e|b)*I!3VZHqMWYRI92o$KO0KAzL)Pi;DM@4f=~tesCxU!6 zIk_nDagiO{WQmLxD}%OCwi=bAT6I~4s7fOMMv1B-bvu}lcF`gcbZRflAG|p)OB*iA zn1$sNKTornyY-c&wTIubah8t zB7s^Gh2|!D`t-Z`T%grpgK5d2sh{lX=^~ldz;U{IsBRH+|0D>Q2tqHqgV`y?juT+s z9(LTl`s8(*v@3CAPo>3KtZ%yPmJYMgxHj9^{{UV}L%V$Pw>HfZHbwkn8QDazz;@z^ zjq0?7AI)C(g-*NwMVz?CQLSrTdVzQVhPZD;|Je8x~=R62V-wbt;bQKXHH}oLftb4wynN5D7Xy=cQ zd`7xdrDlV75MBC2BZ59jpCPhZ9704UteqW6T!oR53Wuh&I!Z!7P3hdiT_MsNoTqz> zchS_hl|1uWF}Y27_m?YdRXE))NaLaP&#gIDF`pxoOAmt zPE^DWZ>eV#{&I)Hx&hIFSupaXGuP)X{7GUm-9 zQVLD>B$Ny(%qVFyitfegqTEPLqOir-f-hhaOWZidTKwybKrZA+OH?4GRRV2z9Kuz6 z>CixH(OhMCV4_t{;zo23o%q9C>gQUhrp`7;LN@0Mk0_`l>a zFDuGNF*-2E(r3>s2rIUL1OM_R%*w+p{%)rJ%@bL%`#>%=*3dZ+E_Bdui9C-Mt-_OT zrfR^0GE7m*P7X>8Z~*fL0NW~z^e-8eaT#&s<(iBbTf`W}%HLM5|E6({y2x$POJ$4> zJOZU87y^l=CUU+5Yhk3=JTr}&cTl_-EfTILk%1j}lQ6+x^O1FmOavEh75#ei*IY_X6gqiXc!FVOPn z)bA~<1y+QSi?GoDR8E;T?)}ON8JRINH531?(c-?Q00GYZ9+TxNb27ik$678MSMFaX zCf1a~HzH#>s-^-Fj7}UU5}C+lPDpbyk$pqC-14n9B!_frZxwIG4s%YYy+uKFE(E9xz-DPM8slX>~eO) ziWuf717=3`5hh%0J1{04N61VltYv1%E93&`z@jSp4NSvoJh+pv5=U1AWD7WHJu1h0 zgf#o^(o{9F`ZVZ=l60rmu*7`tFGEKw(j-~AC)RdzF}d_KwCTmgCjwS*CWu06ZRu5DM^s?P(HyNf;*KICKq~rT?f|bjUUet9E-2EI z376tk^G@%Ya0#)}J?+C_DbFN1k>^^?H}+%Vs#RcqbquXboz5j)9H2WM2wQ55Ne$Eu zdD7_~R5^T3H{1+cole@8CW#mdYL?FRrpSO6=4Jk3T99UZA_&~R1tFMj4&Bzv=vCb= zLuJmBziL-HZ01%Tus`H&&O$OT)GU1(U<)V!(}Gkxd~j2$HaSmq`x17Vs&8v>>S4vp zaijD?O~WiW@O%b_zlg(=+QczmYsq5PWhFBJgHZqlP#;ycecg9#dQ9ccSAI*@W&gYM z$)@Z}!Gz*imgm3`>7ec@3`+PeM=MYUiR{r`gwJrgg26H-^OAC-;!%MYrv#UAOhv9a zUkAW~M|TcVv((OZRO_=`g$F;x5A^LPw&FWxRF$rhpQf+K6b&BAiIN=h;xuAycj`cLW>67Z~O#j77jR>&* zDw9q3RZVFMGv9ZWSvg05G38+S#$-87@z-PHS7hbXW^t^zC?w1*Nj-#8&V)@i0m*oS zugr=iuHM!+7!-7f>Nm*=A2kz$Gx#K?GiwuqC^|)(KTA5LlPGvtabpmBHVyE`qH3+D zoQ;sSicmf{a?IEkJ+CLgl(3$Kg-qUUcgNx@N>Uid*H|$KIFxAU@MS*MtR9J2ChJC6 zZIZhZHNPAtVjZ_>k1CVAtWFs4pua7jB-BE(4L>2*qsMslG_NVcsdP4sTPWjm9crYM z$AXc=H-QTIM2I&tkqU9wcR9q9DDY?b!`F&}+x#v78h`^ZK$AR@dqOPnY}suoaw~1y zZig0GJPBeH>%?{zSzmc(l2Zf4ELmweWTKia{RlL|I#D(CR(Nol{Z#t3PXNwh@kWL2lTu*i~-?YK9qiC?_Nnr8hXV_59X6nqof-S8Q&r z0Um>y=`ozP+hD-Z{3hDi_6KR9N6kcch}37K@2gs@ax@;uqa#$F3VGiQx?3ew{haNx zVLHxScO#WN6{|_zW_ovXm#O8fVQ<3DucP_gp}M`OxLy-6#MYGjhi-xpn<7RP}7Bu)5YiJ|!?HBZLvI?F{a zgh9(vS1Zw2i95W*RkH^PXNvBisXX(ww)!saJPo%^KAnMOMQ4?g*Z7M_@XN~he-3f! zIvFqsotU^Ii5&_LqYlh2sk@5^Fq?AW+!g&y=;-Vbp`ObguL zAS&YwMDXDV!HKvSDUa1Eodl*-^vIyl?Gtybfwg%8*w(BI@Kjhj#GWET)?4$ z(XKQHaEvgPXV9B)qVi;lm8aCW1L($OYSp9J23sgl9iUY2Sgdu^#?^|rWz5BlwI(bW z5&}n(96zqqX>_DpcPmMvy$o7p)~sn+k`A3Yv})0%SGQ)p`Y>JCtZC1lt(dXy-MoAI z{tZ01@ZrRZ8$XUbdGge+M>A)R+O_o8&1qYA4*k0A>an#`4~>?jRkR~#MqoJ8quPRP zo0j(pwCO_4Pt(&(%5kE*ig%6D{GXRr$_+?hfxrb|V1f$%{{t8tbv@YNgNRLa9vpeO z6G4OLb+!*-QBkOtSynB@5Cjc&I95$SN#&1)YvFWZj4-}9V_r1|b)$?$v525cA&FNX zNg&lG9!DafcV7SyfV5Fn4C!YO1R)6&Q%evI@K0g^G~mDj_$;s_R2!k^Q+c83honG6 zJvI=0CaI_1ehj_1-#$+UiC#|^&H2!iWL^YiMS>z}ke!uvMCg8fVl>f3g^J`EYlhBb zk)D&*M_O(%4G>_Ab>_ELsOo*j)193eg_dBmkvdg-`UTjQtMpx#m`xW3KtWgng*RtT z1$AYVtw6Eb;;)7sd)u*|78HR2eYL78VXDm~mt2`@|EXzjTDdr?NNiqq9zj1wHspM0 zg>;#*rDeulcHKD_-FER#7w>oO-V5(w?25EtzyAIUaKHi!r(JjL9*eJcvLT$Tz12N8 zUBL?9Yi@ZAIpvRJj{(>aRaW_vYGrqp1(L>ZaxAP>V+tACM@D8laLfWDD09t>MIclJ zaAb(3Of0!IKwDVV*cJprfxMnnoC)<7XG%pCR9k{cb{o`7rMT>}bZX6Y*INVCVvkmq z2kDU|zf7KF2dxw~leTsg(VzOwSEXqXW|w*~Hsg*Lz9&YGDn6$}jk^ixKo~g`s78e<|9((OA(dTuNuI-vl9r6U=y!Z6f9(@e zAcmZip2=Q4TTW=PN|jr+u7v>yfKj**Qb>7JBz&GCua{CnBTGG2Qb)aISVBi-^2GpfCKQsrI1;UP3qAMO4{82^RuCBI2bNUTDsb7|6x0%?yJ^JPnhNwZ%0Yghf&^+5cA zFPQ!NAHie@&v?rJ(>D@~U;V&ozX+1^e)#0)I_tSl1t!p93eg>{6xJ3Jbvre_urO;3oyl*mw$$jsvzW^quY zolR|Y)0*bQB@pb}LY5RPNA(0H8N1?qg*Ms#X(ulAS}7+TrZco zMw!iZF<~O?K6cfi&aQU#$;t9Q#X;Qdq(7>Z9r!%MLYFLL0J#Dh^-|V6P$`j}&#Yc8 zF~t^tNV_g-YIM5P)u=erIWK@(r@7gsAa@&>-G+5{G#qq?l$4}0%%rR%hkI38E_AU{ z4$&yR0wI2wYLakPcfSYXZ-QXMM+=!qhxPf4QexUYy0l29D>2c6u@=)I8jGFWx~bQ2 zdf}YvNHA=q&#su1lFQ6dj0_Dcq9zG9LrF4*Jbh4yTjBsNeQ74=rOZL~3fwFHNGD5b zkDn$f(I1a>cI@roeLg2B*1dI6aE(Y_@#DDsND@hr%+L9FDN-eMFeRL<2-!|`t=uvQ zRHDlNAzpU$-p@`}g8^dadS`=2q=ZX&h{Qw)W)1M}F zrbBHoflh3o%H`j}s&lc#+Q(0dB^}Cka@AmiMA$fqRD`x&E+#iXcGFlh}7J)LUUiBU8x;WXUgo+k2QYrPzN{#0qx z1xk+}X{yhjS{TKkZR>X@>a)wdC4ZM7i}<{GQWtiBSVj%1WW*A&!!$LS?n4r4JFF%V`kb`RBSD#-82_8D=`_`gd#Sm$%3zMNyVx)W2>uh zW;0E44xrman3l*|)e(7^s{x#AC{Pg*XPEBjVJ^CzxE&u)bLbR2`Nwf#Bag%Cd9dWl zI=tS^v^YyGuecj*jg67E-S5sA1hzfang;c#_wL<(db)mcTrXvuXWzf630)zX`WNq1 zr|!}$y|E4)k;9nUEi>dccm<7cd(+c4ZSx6yO7ZrIm+_9W}D zd52_n(LIJxW!xlsL~>p`#!bpwWU<|#iWo1&*a0O$^B zRfaU-<~E2XjBFCHNQR^%$4LA~LTX$d?iJjI|0kmw)#j%j^Q|b3O^ky%NSVB|FcU2+ zAhJB~g=QC&QXMon$g&04(kiLsH&5|J*kmWR^G{o+bfd)(QAbRt*JmH16pcn(AgE{} z7)~(;Q)EXrV>d~~Cq{vDew5>gE+%jyu>e!m zCBi3K?#GElij)^K-BpAoiEWWS>SVoV> z!j6~OWyE43M&u%R@@MPxHCw;|zcXJIB6>sgOj<{Rwl#u^wlDHnO^MMmU4%!uxli^~ zgR{1T_$YX-rcdBiffeYEV7QXcDKiC$gSX~tMi`L$xP$rsFN8f;;#z zQB#O%zlc^L6gIplW-1AtLinD$0Yp71BGw~4QQ?Ox^-{HDGzSfNTZk#=9ngvJ2Y_~zcYs@HB%Qd0Mv-3 z1A07?fpxQan^^~sbeeUKc7o)yRW{dYt%fjaN1V$KC3nJ^kKYAo5IJ|w8J~^+N+5k_ zozxkT|F}O%C_jjboPbIe=XEZ1!d~dnsgIL*)u#~b5t)^j8Kh`-2U&NH3ZJgJDb3_G z$Mi${00E>zL>@6<9%ZYlC!pW>hH+7vL2+iYw}&)$tbABUUTQ`}dV7s`K6`a0dNfAz z_e!tWMgN8%YV;B?wg8n-RUzYrUkRd?<8X#jMq}wm_>~uN`6g9ZHXKr_BQ}*Q$~qo8 ze3MeIWO;}=Iyqhi5grDAw^tJADn@3fR+mA6^>>){)n){Eb4Src5&M4z$a>6_f6FO= zTgr5<^Fy;?N&q+`^jSSVQp}e;;CO|Tn#jx z5t*C|d8qFQciq*qM%aVb0;@@D9EA#N4>^#v_OtxRcfqNTvbGlMu~2PBg&PT_JMmKs z5rLknmnUgMN^7>L#)c=$U}fQ&$J4S{lTt29GQP@=Af+TisjPV$lzaO`7I-TAh_NvxdUM-oFDtYz zOM(&-f?@W%HiQ@)L3I4HY3C%VlWLs)v}v`*yhGcN|C=diE5J7Nr>Ft7`$(yxRS2CHgf-T`biP0FoLOaxR$W1G(vy?-S1ktKkql|6|Es8{3_rYl%|Y0}OQ_a>7pZ_+zf5?XB6WKUcZovCf@B*;5PVRM{G$#T70b~U~<6IYWzlr%oTpo@Mix%9ddeHTX{Z!=gxEFL*;(R(nw@4%%ol!JjH=k07kCS7gC>5Q zN;oZkm%wY{r#GhC@YQb(@yzjY*f9gbFK*=hB2spkQtC)G2R3@AHCu8vQw{QTGpww^ zOQ0LZM0vPVfIBz;nz-+GM_y%!D8W@lJXlRxp{&iVTA`uLhgFJ0mtMIsmiv{L$Q+&s zR?ODWax#|8u~g!Rxq@DzWR%=!vSb?n*Q_P#GIHES^Fd!9T5mr-Mh+pQ5bZu;!V;?r zGe&2Y!@Gap*tVnirr=X}qQ#^y-C6$(5^QrSPNA(@@!t(V0T@8w6tD#b&YB6X;2iMa z9`FOfz5%zu0JpFK!S3Kf-~qhO0o3RVzChp@Anmgr0oDGbKB>KSdet(^oZFt&Rt-(z zo^_3((((!7lX}4Eo}3U#gMW(Sx2($q36C3m;4Q{?r|@Bx8wD^gmn+38G_D^W#nLPorIii$M0 zAGW(7N+><4Wj$e`|JK^HQbh~7FaT6FBsL+&T)~Q6R+c+&NqD5NR2HS5a$`l0LiW*P zCjFMwt;VVwyBkNwyeBu6#CwZkmr037IQnC!7@_O;tOT3U5K)4R;Uaw~)1I+oSZ8Ky zPrcRQ$g#BJ-l!)TqJg5G-r=Z%Mq`ay@!&?l>(rR*eG%;e{@+b)A~vxfv)z1&Z(dBS zC_)N>HtFpZkN0l9PFl_F>YL%FhRS0|Snzx9_gGK2~cHg`xO`3m=`YZ~O|frb6@Z<$F?2o?s?*Z6v$G z$=dPN|20PWC;r;ey4UEm@#v}YC^H(c9D0h53%LWQq0q=xn`ny>oY0xu<6g!#AQ^>* zvqHb-Dfa=`kWqE_3=r-Dxbr7)-#~){{T)nrkYPcE4g>5HSaIRNfebBHoJg=@LxlnZ zLIk^c}N1lYUN zAIYK+zV0J@6HWw*g((^?T6DnS0fr}XU3wBu;KV|;x{ExZ=;Z-MMT<6>l__b|rJaU8 z9Xjf3*0D=B<>~XE-I%s*;|5ytWbfa>f%_&-+v#!I!mSP$J{&poDB*iejWOB=*i<}f4;pw{PxWKy@W{evb(*$I;tb3WFl%P zyd3fIlju1=_5jf*1<(K#%bH@+1I(63D-dW(2d$jOsj)syl^(2uvuy46{zWq*U{! zK0hLpP(uHdXwi^@9MsW~l0;}Fl$6BCGvf9m3CNQEGzcQ&bh1q`Lap+T&i|yNDk?fL z{fZ=#qyj3|r&>(VomfG2iq;4W*n)u#ZrR`mVudKy4JIe@1#jlTJ|OL$<1v%3;E4lB0!Vv7%^PC11I&i@bLjYHP>V}cWY*s@5q z3cw!-5?h%dm~7o=uLe6>NXA5W!Z^bTclP;bpo11VJr7y*EGy7VLh5J;2%15rLb?F?x0N8c>+-$3rVT#&XRbf66rm19Nc~HJ^BWOP)@p31; zYDQ{}17hZk0$NzWU?O|(;-t^P=x8LvntrFsp zK@OPY_c-p~IsbtSfd4a?dmMJY^#zAD(HmL=9jGCa@o!-x8{on?Cc*r{r(p!zPl*at zAj>c=U2a;Ofxsmc%8BZM5$f3rUHC#6#_&5RObi4pL!y0{t0^T?kx4=nE2AN(UD@N> zO^SF#s~xdHC%H{bIHEQ>8Od|58x(+;H^nD5s!u1`k5=~NlibkFE(@r{0t8Z|BHg4+ zX$(q;DitO~xomI@Qj@DX_>`LYEsl2bNC&%EtERX_jeQ)XpPpn8v1Md+I9U`#K9w^; zt}%;Ya-`?ZWH~WWr;jG|)FBhH9G&@Oce5gw*?bDP!2o8^WDJWKIonyC1zKr=-IQTHSJ*LwdF(#v4Cns#c}@%7 z&t?0OQ(qY8MpV|yjcmG^gMMkx4nb6+6Q!tzd^i9Xu`DX!+XraK)tOnXiztC~+VvEJ z#2sx-YbZsKk62PU#VPJ`At9USe5bMl(KL%I>&xW)0mdm=b4qs05yQR2He1Kz7f}GM) z@HmRhvedFr45T`e);^kC? zdQfxCdtL{&5WN-~pSpHP5fLdRzD7}31nA4d5Sj=MM>C>IXVkU+ru1qx66Q_>nG!-W za&lTc@RCZhr;SAMk#!~0gu*yao&VtDZhHYMCB4K{(Fp`5a#YaZ;0VV`#?mP@8>E>e zc}Yl;uv9NS)Y>GcR5fX&t~#a3nH~M>J{DV9d5qU9={S>}dXgYNDXviVA6ZygAs~SEL+;ET~ua0D$ifLlA`Kljmyn+(Pvqil@xi8 zS0Qq(`rsGMKhK7Jv^9x^{`%0}18U>NSsdOe+4 zfZ}hsllCp7A$=wJunZ_m+3ZpyS7PmAS$wZk@24|sYh360g|4Iw_i~lHta#5n+W`~b%q|>*zgR|P3*Kc6|GN7FB-Y;L}xjRTh8NJI_m{g z-)utFbY1_icYvm5G^wK=>Pn-1)gQO^R?TGDZvBV6wGR8O>wb5;Q%QX`15IuoMC-i8VV$r2bG>UP$pu**Ap zk~;rzil^wh9}H>^N5eWsBevKxkaF{`TJas}vp@+^!LXw{79<+_iVbEm39eY0rO1__ zi$7E0IH_SjA51m^!y284E3^6%MByANlD5!c9@X(2)Jq80$%%v5Ja5y91y~DkaUFbf zJuZB^*$I?8A-A{7lgMJdZ|N?^u{Y8CkEQsreN#d~A|0Aq8=6QfJzB7sQaFTDFq(jy z)uA>UbB8++w)(X5+lo<9=#rE+(rr?ki1f~Jx7f!^$QVS@4a;M-Tn5h$|OM5O|495T1 zf-S0(CyRT*UDG<_$i?HLpt9RVWQYQhI~Fs&OhiU)+(sANM)F`R+G?nzz@f;1 z6;?SH01HC@N<29;FiOF(ka)a8GPpFEDT25Pi;%*~bHW9|2r8t&D;xlP*c&hGLZyne zoFNGWb3MNaLlP6Qm6Hh1U_BdaNLgzYKAF6nIY=nsJr8RVc?>>Bk;4X4q#T3CLg2ikc57;5V0%WvPtGY*f#t#(6-ulI~J0SmeX{Yuu z#;^3opSi{S(YgkV4hMWdXp}iFYMK~1o*uKTu9Qo{RKad6On;H38Y~r_o0^o_x$KCV zAB0MsW5*l`y(Tdnsj5dIn<>qSqz8bhEd>K0V$9*9x2T-+xYzTwfLJ+G*s(^`- zicO6eKhYCJhHAH0>n_c$1E;I+0>iDH(H_nhB0J8;rNX4FOxW z2D!fV1HyHwjF!2K2skaH8BnC7CVQzSYAOwk8BCbFEv5@g5DYbF?5+O@b<;4_b!TKydI+M8@Q5luX!<<-doYo>BqQ;?Mv^H9VK_ld2oUK<4`Y&el&N33u%yhAEr~sPgw3@o zn~9W&nCT+l+oOt!)3mt_}RaUL26)xkr3U!&rP)Z~a!ptPTM+BAzov0j08GGt$XS zyqa;Mw(`9+n$ytv5i8Z!iYrNatuTfFDku z?A6draYulgY4X-#}HH(hB81ew^{Z5Ka zvrMI4Nj$$c%bwF%ihiL?A>Fe3%dZ(hM-~B*803r|s=jr}v<1CNLJJze-H@-1&}d~w zvo*kBWk3#nz=*M5wtZXt&9toSP7xLV+?mZFwWTft%+P|V+o9d0wAob%?qCnLz{$|s zn_CxHnZX#mrXjUVBX!q4YRJqUMFtxpH zUEQ*gtYV8dckABS!IOnlz1WSB_9P0u@k6zWh-}+fB_kav>OJ5C#D9}bjrGVpEmWI$ zS9%l_zJlD|2+d7YAuvLj72(;c<-y?jo$%#eIgw((xn7d8CEVe-u3fnhMhwmRudQjC zbamXP!7~|bJXa!y_o)0KWCKStGm#=%}cawTTw$+2z1-`ao`7jR=TWUV>MuF zElc9K$&Mr351wVQqh;OrwStKM!4!F~n?n)$MPy(OX2k0eN%5K?8`;a%sgaeE%;kvW zT%C8kTq~^&E#-+Sxx%Be6Vh#6icGgF>SBu&PPjYbj4(49qao5FJqepQzVson`K<5Fpul2M(FJh^i3(E#2UfFv;9(y`MM}$YZ=tC)vcvu4sKAH zm7BALGU=Mz>N>lzomQaE+5~P}KdX-_JVsQ$+ZDxNyA>IB3bp0&%bK=jucd0MPDUOq zPm`EcA4M|(AuPvjT*>vvJy8L= zTLQYvvn4g?zH0S0uk;RY9}*GsJYoMxxh-Rgu)gm)b0Uq*3E+#;IdzFRpb>_&~v}nXm(=&e2f5Ys*L0-?cM?Ucs z2J0LLSH5T_GII0mLsFD!E91|$u@-M7D-D(ClfK}w)W8B#k;|mP0G@7STGU4?p5n0j7_QsH@9-2$OpQfH*vs`KvCCgw1 zu43Kp`^~>$WL69_Wo!1vRu(ZJctScr3t3Rb1> zM_;KJVPVaV!Xzzb%l+LowpWfgYrN9e1%qZD<~(aI34lEzIog{WGaPO<=HU_T*28F( z+9aHkD#0HAKf&(Jy8=xkd(DxhVPD5re=F3&*5S-ln;B=MDXLB*Y)*2ctI7+xuwB~j zytqs4xO5gl95nKfA}0FiW3S?o>20Iio^>~38JLUW!G@hG?`5e4t)FwaaSd+#Jw*cT zA<}@MF@J99HuHcXTVhRRpO)4>ALWiIy5`nyVtkhdhTBnRK|S9<@iyB{1|{_x^gxHX zjNka5sUb6UCS7_7sr0#~jdWmUzprVwaS;j3Wlqk$$1vVU2*WFVbiz=_QXW2P&}+Q6 zGO;Y|=9aP|xS3tr_0w*K-85XB(Ho?iNF#$r-ZU*fIXprc{~Q$KN9^QGC5`bCSMg(x zO?uw{L&?=5zMgh!7biIGHchG2j()`1zN0MjGLn-uQ<1QNQ8pqM6P#5E^Dwb*;b@2Y z?NnX#l9!QMU5z5|j9I;*p9`PEaHV*Op?#P9+p0zDQffbUYBi_sw1soe$4UgoN(Xj2 z5KhsFr&bIy?;6E;j&J>Y%J_~5x)p20sQC-IFMM@1d1mtwXwxZ-Men(Vy=w& za^}l+;eKu#Il$!7rca|zt$H=<)~;W}jxBrgRFM+*jVv1VB*dULe;);Il4;L@#{C(u znA{ytmnEM&cg`Fka^wa}TA$urIrQ$&k260`8YbAp|9U!D&@3 z6|^Ku^ZA9)MrsKt&_{9cb`etkjOUe&^#NzpSwBgoRs`>T6#raqaCGz#Q-!hhltVS` z!_;h(L3XE}R%n|I>KUMi5~f$8ff~9fUvN4KsbY`+c*rlv~L|X39p-SK9tf_59wBvK(juEb!h^(a4HmQ1x3bvuKdV06ugA-o3;fJUCCQ1^G*tuqIDxj%iVLxGM8<5BWkDcb(@E-?6&_vbskR!;8U%~8>cv6f#%u=FTDDh zXkdS5QpBaa2+}treaxLnA#e@W_b~A{A_oDN1v58Sc*ZxF@bv^%*gcIHw`bqR&Z}t9 zauf5bZp7G;E8Vr>Gd#M-;PHf{PnPwFvY%!)rMrdo7aqp zrjvO+NIwo#i&%o?KcmQpZUey81|yfI0N#d!jI#|Krcy!*9?ouvD-G9R^|euntuJ`- z+u6>hwz0KMZjs92qSW$r)RDINVVX1O;aEOjGWUF*EH0P;*sN&~cAxXg#U zhYe&R?h#%e|96m`tqXuwTBJwZqpS42OKB>Z4*iV!!^?I@`L{BAVs2*z+<`$B@bDl3au!a76M6$pn9R-+|svC z$&iQg6y~FR_?I7EDsFFE8$S1mPO}V-iFp$gJ&$P4gCaDc3jNALu~rlW#))cd`XWX9 zvn-I&PkzcNR<)ipBGYXzM%K#Kvpnj?HbSO6JZgXgx|A^euupyD;-2;})(~}tWLozT zNWmniyCStuJ5b02h&)uO6~k69l|hh=$1});_y?LMcWM-#X}g@(`-5f&1m7qPag&mmp>w6 zIXk69&oWVO?z{+PYirxv;x@NzUR>m!hkwJRYkH6);3SN$YBf$NwpO=Z z`6r*|23yYzl(Y0CXgisl*?I2JZEWLbrB2&Ve6DqfMNV0&;wf7vi%Q619y6I;Gubk; zhHAZHS_mTt#y0t6!l2uZRcUlO@tkx!aLO%`&CjT4~n+YoY%%&vp)r>fk;gVJ#natO0{RdMZx=Td%CNBBe>#l@Css_evW)#5*Q$#EpvCs=m zHKy4G(S+e2Sy0Ao3E`P0st_Y{o*B(`v+SR2*yKim*2_>{GCc1&WqDG{%2qaQzWFUg zu6_U8weTMJEhD7vgCjiQ3U?J>XvXOr`$~+KG!}KlgIolsE2D9&R)jn1=gx&FEB zyQd)g@})q|3qkwESdqes_`iRY5aIsHnNQ={L2xtxgrH?O!Wl z)kU6+F0W_0^;pI+LotKf!?VQ+!7^(2@+uu8sQPDOkPw} zn4AT&IAJC^he#cl$2r$1bq>W66fW;B#yb^m@jvqakboo)tv6AjZl8TrM)UikH zY?6bdi-dtv1|dv$lo!eXpxALr1W8~-%oPE77Xey?MaZ4nAX0e=47J^%&J>o*ARC<| z(*@Gq1^J6b?3J()(d+5hBemX3C{zg|;h$(3pS;t%*%D^W8@V>rgdnmtAAX;ViLPmG92pWWHW*q_FYl^Tu6#^D)v zw3?u?PGc;<;jNo>s9b*NSMN;NQF(}|<;+^(1gG^*crncbPSt>^&olpFA0IWH`b`z| zjn!0XlAraVpLvY!=^yTGr2NU>qWxENpyXGD5B7i$fJmP~^wewJ&#!4CmH-pn<;J8b z(~Aud0S3t8AWxh95+l)-K*$}JoMN&u8*NF`jJ3=!wFH)&TUSUF0)_--jbk_xnUW>Z z+E5-e9-|A6-ZNUBGxnQVR%12BO%P#Yj|AK~31L=}V_n*1;cTT{Dwcx0SpoHgb^M_` zb`Eou)F1X)B(}@Q?V52xOFb?}$?aoLpxnCUlrvq%Q9TcMMMMA;4AJS=tvt|S70_{{ z1e#b-MZOm?rRMC7mA!CXvaksAkR+&$7a(OEfJs*?eUgH$&m{j!)%IXq8KKX_m{Lbk zM2H9sOod%qVHm6sjrCzAz3JrIH6;S&oZ5imLIl%aHN;Zt*IY;wE|MLMspe4R#Z}wFR5# z7j$~scs3mgX`5`eoQw`wM9OAtmY+Jd59xT76nd4j99VRr(SdYKCB{!H%@oD>A&)K{ zAI6mUK*Z);$4v?3CgP6|I;TR0;3*ERh%3Gh-n>AXBSmNM*_GjkJ;Gkj|V;e6g=`-$$eq{3>l8R=(>Q(AHCeQ@Pr^4 zOTFX_nv9IIAw*w&NqLqh95SDf;Gwc$9l}6_kxnGWDV=VT742aX`Sn+?aU`ulom5q# zs<{&SsSzS7i@@wl{>-GXSs0hn1R|;3vf9;FMRQ4Spi5p)MI=M49GM8Gr{;s)HaLZ-&HhNWgK zI#P>sbm*K_+#nVm7DAz(Q7Gtep>)9`#*7CxHAd__>YwNi^swBlt!QRSiLFG-mS`Hk z+@gnENtHAbgCMPp0#ZaG6(l{CI?~=M&D0^*nJkr`iawR@9Vt{v4E$jnwn9sj!e&ju zV{IH~1RdvE1z_L#2?at>Z}bJ3^3PtyqLp~3{ZL!Y6s>p)5HtNNdqt--8LNE-47>sm zN}Q0#F4Vwg?0Y(1pek%xI$3=ZYUFO7pz^1Frlk!kYQ^?oT%I1|a_s1Ktmxh)oj~YL z#F?JGV~0N16i(M-4r1!OE~j2xsa_#i#B5ur$7XbeW`_S}Bwl2)@K;-$(J^60;r3s6 z{T%b0Cd_GDD-JHyl9JZKrt^3bk`kw?>6GmuX3S;Z)?N&F{HTIupCjVlRasOr!c<#W z)N-=#mJVBVIuqTA;&iSU0*Y9vl^ER7OWW;LDJIY_rQ+D^gEScvdEP7D8tz=Eo$2Bw z=>j1N9+A6gmIYfb+nSH!jzX_FY;X)(Sk|tKwn5uAV4;Uu!2wbooLS` z8q@yE5kDgH0H(QQ2aBL2@$6T}F9W?XbXwDq^iP=Pgk4dIQYzE2A;-0)#ZdAcJt<&Sl9>yS?mxX? z1~2U7vS25#r3cp_#I~hdBI;tm&0C@%3$t=7yO1mYE-#*h>&$S;TBw0d*Rr^57G9$6 zdB?(tD5(wvtMcx!FvRZ~U1(wygA7P`#iVyW(hI5hV!eLaXF`wuEpPxS`|ulZz&<2I0u+0nIEx&49vZY`K{P5 zIa4XtbEtK5J+CIrMC5zXp~PjJS&uUyO)HZU@3vYDZ>t(0J!z4uQX5Zkl48^(9*ca> z^K}$N==`r3BXTlwqioIZj0tpfifdC0I0NIvnTi-hiI{m3-gJH#10kE=ycdoN#TL!> zrJ@Wm4w2--U`Q_`D1-QwWiDGrqrGWvf0}a5UigZW<4nu+S`USYT=q$%k|QA2%0y@e+d=zIa-Xg=w5fY!J^kx{gvYkHs$Qz-HH*TLOSDNGS z$i{9C>-4O9s#DeV#)T+y1+nYI&L}ETEJvU3ENx9)Z?XPVitt|<=9oj?s6`^0^rp4= z$ey}?SG)h?GJtt`k9yLF@L?FUkARURqS>RbqcuUsqxeBLPGtrm5!H+MZIV@({Q8OF z&}|h@=i5<)%=~*!?4)goShh)aQr118M<>e6fB*vm{4;nEVM2xf1U78QkYL1t6DLZvSP|nzjTk8s?3fYcMv);0juaUY zWl4bpG@4Y&vLZ~GFIlFn8S|!0i3@r5^!XELP@zMK7BzYlX;P(2nKpI$^r=9oQ>j)J z>JV!HcmCW3*vFNtz^z@!1^`PoELyZ$y=o21b{|}UanGiG8+U71x@GmUO)Iq^MK}}V z4)7NMPFR9I%M$Fm&mCULTEDs*%T;pZ$X+pf)f_f)fC!!;M=pIjGS|nR+ip(%IxghS zkWtfi9sBZTvt--G?!DTw>ej%8ua<4Rb#J%4#~Noo7Wd!aWGl0Mz5FlUv(_Myk4|c2pm0vq=*%v#L*iEj}ASUp#BacXut9{BM>o=03>a* z^U6c4yZBr}@V@~`@@O%uG~Cdsf;jvT#1Ip5sKczJlIX$|QCv~Q6z!YvrI=)d@uCbi_O<(ON} ztt+d<3(P8sE2}KJm>g&UtOjFiG>Uv$={GXPyYsQFXrl}|J~f+e&jN4LO3y<1tj#ji z{u}&*d&XE4t<^_3}IAJhhb5$&UXMb<*Sf z!Lzwp4aM`V_WDb(P=DqlC^J;)k6O>3wkf=p4 zqHMF>s4$EQ+{hg#&9#cjbJax>zp0>fkw%g}ikIFNO$yRS9%l^6rWsHA7bkyp9AMpo z4L-Q3bP-+{DXAEKxTkmT`wPsjG8J<-xwL!?EbBJ*XS-D`Uac>+!2H-^gG?OILc<`O zXwEAK%X2c##+z<4MvwCiSo+*EEz($TzPVIaIgVA*+46jgGw*!EZRk(?tc=W@zYCAb z-?EOaxTDb|i#M)66&q4mt4otXqFF0#?LH$gFXN_{u5~M3!!@=c6z%_e2wDBQ8wn2E z*t0jmw;^9{bf-C(}#H zX>}k6R_XHCWuKk)CKaxI-FG3dIC8VLzH&8>$Mm>vwK`r)JL0z#>-a2(M}AF*{E=ur zu-e<*VV)*xui3e^!&T=;bjTw)AtejeVrYcXaAPC!Mq*tEH~Gucxge4*%(f zD_d%+nH8b9<0<~B3T&sk-*;{$KKU`QZEdTSu7)KM@-fI(vhmewC>Q~0RnQ@N%bv>I zN0K;o%W)L^R(w3TkHi#+eJ$EkM7FY^xY+POxYOYXb=X4^iRc+!pkoo?j8()I5fO0xzkTGHNOCp|zDPwfQ4!o( zNk{e@iFUG6mt@XMp|+_t0>_L>Bh)Ryh9^l{=1;bAUZoRJZL67fV%u9jg$Ivf*`={o6DOn2 zjZtGjP3tt$+3ITA2NA#|238Y!T3lsK=$2@^nEsmKAS!K6`&^WrdpORfm zSuBc3JrQcVDY6ixgz8(t0S7lM_06uVS}W)Z*?lKnluHK{sXF4)RzH)}rV3RWrKxXK zRpYJzS=nQ>yfU^`#^pwX*3DYp`IiaatAW+2;0Z$!Uk-lZ5LXPM;Bi(USoZSH*UF-THktB(0Hxp{wZP?&T(W*AFf?&cuI@0CjE4@&n zI@Cq@oP=d!e-qyEsvz9sK>@jiB~}&|es;MGZ8gTY!<*yf^q`k96RKV8vAWEB|CF(l z>T{9Cpj~=!Ht=q?v4e&hovKgY<)Wi+;nF8{_v$mY!Qi=T_U-t!E!Zg?DV3vIKAcq@ z&C5|UtFRQ&H$l{QgGqPIV(CJ&0NmbqyN~w5wN|1-rf1k{PrXJH@`L+bLkquLR`AL> zJ5RdsmX4yuaWu|LN7(Tg-x1&`UwHzDyyVNPpBiH}#Sod4>1mWKD|4n~X60LK>gAlKj4wnDoFMZca z`<4Ff-1LM0Q?h>l4~6yqk0BCnb5PEDd}rqRN9AN~fKV?tMh^mOZH=7Apho8O$RsRi zEdVjhxl9VBTtjKRtTYZHpD^m$Y()U&N;D#4J-W(fre#vH<*UXD{ZbIoHt_!j%!e9o z{yOKtD9zzo!r}sp@%Uw&Ql}(rPzje%;XthbnI}x@#&=HX*8Jj3OsvHoPzzb^dBWtl zN+#Ao3kL;|=x%QenJ}I@L+z4mhK{LaFaiy)DGo(W@U(~r!wJ%Q@cu?eUYZE<7Hsh_ zPDbWX3D1xaRqzS5q&@0pSEj_X?4s5{WjjhHONz%XPOa83%b@lGJ5J^hKM@omg00vv zsAx$4Ny5U(i_8{$oTDY6_w z?c%15_TnyDtP>eo}+-8&eS*>+r33Fs}?x7VYox z^l*ymuSagtb@uNY_1 zf2vO)(Iz2bt$##J3q>pCEb&W93yyHDB!AN2|Kf@UZZyJ|wNO^k?Rl#}ZBL0a0ZOWv^YP}aJnCsz&w zb#nApZYP_u3RAK$JG0R|^S?M#(T?)dF!B}8u^dH*9al3KZ}G1vj^jY{B1N+znLE zM+~bu5s@KjGf8gq{Tc!$$FgEjrXMR1w74>3lySxmQWLWV3{B2MqO&{i6F;F76z#Jj zO>-&#GyilE5Or}i^K+i~b3d<$wGgwth9y3SCwa`I5(Bc

eUub9g`tdd`#dB5*tC zh%^mUL`T%`z*E!^v_z*9K;EIUDT^@w28$+gf<&zeMH6&~SWAS$Xjl&8Exx3sUQ8M< z6K#;wJxA+|GBKwz@CsLtjzF|VXB0}GBuWeBNl`LH{SPrAEe{JX9b5A*ner(^64FA= zMEi3}m!tr7G>qi#P4p+GL`I^Thf5N$AX~2HD*$mrFwP)U2VM@h>HF{To^ z(B>fXN`|YRu9Q+^RaOTt6fYG~he%7&QSqemBlQnP|CeZ1-7i?tB@t^Rn3#%p2x50& ztWLXdNc9vTxgBu^6;PiAXO2lQkjMW@i>z-1TSqXk&-PXf?GyMzv)*?_@g5D?bK3Ik5_3GHU;o zYU|SwVHRjXb7+ILYvJi@$1$t=P*>4)iAJ(lxczEt4b@%gX+b@fV;K~xVvGvs)KppZ zV)FD0=q(6*OqV#m$7O!8(mRONKj&3^I zH}^rr2ogC@L-n-^c~egbax!I-xHNHjPH%82cV;hlR<+hj6IbqdqFg}|2;FI&7R+_M z2zF02chZy`k5=w97kegC#lq5VH4{lil^RVA=L)hW_jUAIw|6b}cdr-ovKMUcqhd8| zS8o?P{r{4Rv^PY!7b&|oUWXTMIoBhCXN(pz0^2in9n&$l(4S1sH$oTHIO|HR_i)YE zQU&g5EhJi60-76`U@`hjZ4_h;fG1YPv_Au3y zc->cnp%-I{ba`EFCN(C#oRwLdR%HpY#3WR33-)}kxQv}Jkejd<0Xc?uA}S&^cN`H@#ShPT)- z!?8ZB!Vme6mX&B7%a@fK#+4VANZuB!*jJ4+n5MAMZmSUzH?b_UMt)-~c|Gq;?zgi3 z_fHolP;;38t@*KbSuT0e6*WhE8JSo85}V;wn-5flh4+%jHkgA6zf2}3KNo7}*yUyo zU@Mc=0`?jI5!O<8i>i5?|9Ku4QkA81aYMw0e-&OH|G1o?bfBU0P=R2gSkod`Mf7|ineIZU$gStMdq(X*7gPcd{ZQWeF1^JGwZBK zIj8wXzgnxed9XDMD0iCSNK#!S?x0iRB?@~X&N>GJ@27{DqTBa9d})o^!jF_TaI>Zg zFH5BFcOXT!Wj(8)8sd}@dlmf{vS0fKEtR$1!`8LEC}+W$bx?8+B^ygq@~4e;Y>gI! zU2PH3l6QVnu2w-FG)PB@S#Jz#rLeTD+78XeSy<-q%Vf3&k*bRafpAuh6vZ6(?7>vzoC~ujDP>0_AS=xtpTd=nX za>ZD5$Qtv|u)NKi{Vdo&aW*O2_(xqsSiT21t>g;p)zzR?dO4cr=yvl)xMic{ziAY^ z2OO>j{J|}6wu4u(4=#)$|CK4X`^8~AVHPw;?{H(&S=Ws7Ufr4WFpl7~0V zIhLs`T0J?s$Uj#a{k6lFIj%LjNJ~7L|2AVZn;QAqHV+)toBf`YQ_<~7&__hUbSHhoRq*)hk(r@ebW zG24M?B+Ydm@x7O`e8U>OV~GVfXPqK2Fdw0qdLxrmi(ETA55!CPs-k#gTlH^GoOc8_ z#l@A`>mB6572v^E-$y>OKj+_l(d3JATh}{o(|ny>Uefbc-YnCTdwr?n+JC!HPP4?; zObygGo>b=*YL)#eKR)EYn8jDVg;!l%|KnxMv7PDv?k9XXh+|%j*-N!3^T~?&zVmyb zLYopvJOkhR3ho%rm&#aq2A$ zW5y^(WZxM{3CddasqCdt<0Y#~n8w(hv2G5d<4+Xs<=(J0e=zAj-`|po^q$cRQS=9o zi-0)mDY>nU=+XDMsv&g4A@dSXH>o?(jU66-quIpe`?w+Bpf2C;GoSNMz4Yy{0PICHwJ7(l1bpFo2O z9ZIyQ(W6L{DqYI7sne%WoA!)KwW`&jEU#+aN^>gLuVBM+%__ERQmph=ajx9Ecn{j;xpyxdf)zDR{HPEiLc%BwI{fGmVZ?)g8;|s85+q8Lj5C%D3^H@) zlRYc09nDrW>C>oFt6t5zwQIJoV}B(pJ2vgwxO3;cty?YH-oSCPqzgB%-s5rY?&Zte zcwe3oHV-y@@nYeJf;%>rER1n-#gC9Zj>JqEVq}pgt#kAY-d#)H!J+buPrttX`}l+Z z&d-15{{CV82mj!JV!=n?fe2znTyw=4x8PjQImciC(IvK#WRERWSaq8%rrAZ4u~*nf z>T&4VWfGZpnPw=RGMvfgeX?Kc==i+81 zN~dCHYm!8srHF~gubo<|b#XW6etIs#Muwx|>&YHToo^!$R62U7i*($vp zcieHt9uoyJt05C8Z>}So+t!lHwQMDn4FcSp%TxN3qGJSg+Fs7n8ul=@4Yf%UN$Rc1 z(Zc1Shvs*lh8J4Oc5Ynq)Kt3)b=CETd@|OrZ5^xE@UF}+%`flE+`s&$jTg}%Is4(6 zno1|?!WUA?S+%CFc(=9GK5H??^h(Y3;DrBpb>aSLeg8P(N-lmU*yxIV@7au|E$pK* zM`lA_Mt+*Tu&iL)P=N_oz8tYygd*;C!$N@8|lCK9zVIhXls5m=UU2EhDcQ@^6;!wdQ%UZPE{>~QH)fl8ZSm5^uf8VH9&6F%hV(M}lUQ z!V@Gbr};5zmd2X@r-Wsaw3#DWrm$GXyJa}X#-QlL&{++;)-W5kGtFFbZl^j?8}&9i zb+R*^1$^4=xQWevo=ch?^rs*B8B2g7$eU(cr2S;bzp^cAi=^Tfn=WF*ErIfAeY0eU z;@LNeHSIgz3Db+l6GwkQGoS)ZsY=gh(3SpkYAqem@E%0Vgwhm+54~3|0{{VIUaM#I zq^O(BgGm-4(`U%EWbC%Jk(-4Pft&Q45eFAjms-_GRTbV;vY5@RW;e^(&U*H4whX4Qo|NsB@_xJDb@8;&_-rn9qbH&!y&c)X6y~Vxn_x}(O5a#CZ*5>a@N=i~% zTCKh2gq+U3-mP*DYu@+&_mr*H=GKh8)>>MG#qXtBr5WaX z8RpIz=1PQG?_;f{V@gU&-tX2KN~Ia^V|&G8t$Sli#bbMfoDfRR-b&6|&SOd$Qc6nB zoJ!6arJSW}wMuhqrG(ZoDV*LJrCJ%K85!^18Sh%}V;LFmoEh&K88Ms)oZcA_oG~dn zwbqpHAxfN_l<(GBQk>@IrM0C><`7aDQW+WVz2@enjC)Gvd)BSaO6Gf<-dc>sgnN5Z zA!|}n8I--=?-}M9W33sSt+kZqy`_|EjLtDq85v`Y8Jvu@bLL8=T1tBv|M&Mw@7_x9 zTJK{@8B%LX@0?2S8A>snN)Vi-t*t^jLcP7c#n$F?&RUd=jEwhddwcJU_aU6lt!raz zN@J~i&gP{M8G8^J&JY>J#pY6L-b$3Mwa(s3l#J$-=I=`885!@rr722djOKIZ85z#K zr3l{lYu@)Et+m#BI<2J;O79S@dpdjX5PN$Nt$Pqk#SpD~5PQWCd(IF_l$=_v_q|FP z@4eP3d-uIc8P2`d2%KxRr7;;9O7HKbDJdaqrE?j3F&TSv8O|{ol-3X#gliduDH+c1 z8P03Ya~T=Vtr^ZK5FtW))`ag68JugB8A{IYr3eTiYo$^ddr}#5)({zlN*RO*8P3ib z&PvWw85z!u8O{ji_qCfFh*r_Y~2g9;r=6e!M{MUyIB%CxD|r%*v2ol3Q;)rV14Dol8_tJkk! zt!5pIaNz*6XceASn>N5&w{73bjZ2rUUATAW<_!y$tlz(Y0}D3Dx3J;EVh1Z;%($`R z$AJ+`p3IkPWXhN`Yu-##FW%3dL)Qg8TJ+x2q}iTb4VSg+*05j8j!m02L&&&u>mJOx zx8bG>cMBg*ytwh>M15zrjC^N+fy$#xpSsp|b-dTF!^X~idv?3lRfEj$9{%&)>2I5> zXF0vS`}ZUrKi3&tApHCI^XuPVa=rim9eu^sd;p$AAaDFB_ZCA7w#DE>4W8%Vgc4S0 z9$j!EiJ>UtSO@wQ(N;5Rgkz2`p|>4v z;$=vhZ9o?JT99ZFd1QwaisV^eJVt3Bj8vK=rEc}DSml;ncKPC!U?!L1Z99fZSdC_4 zX(ap-27aIz`qd2@Csh$rW1p&$N$Zuj${JaO zqxK5nuMYVdEU*nNXl!a2mR1{(mcprPb6GwsEm^sSR9}44T04?Zz&7ixhu|{#t)|QF zX>Pgbo~v%U%T}9htYwZFT)beq`z}=CA*rvCKQ8HSzW^&ajD~{Mk3|)UdEo|Dj9EaO6lbUUe^TRL8TlCRRg;;UXF!eMn&Cuvz5RYFuya^#EOa z7p{|aecWuRE>D|wR5ubWm9V^15eG-gdKki_Sct+E4o{$9xkitNn05u;fW7L>7B2~zVE-6 zi;DTw20tmd&wTfeRqF@>FT9T0&bM^jO*{89&QI2SEW$kp{`}pChu7@E9ur*W^`o=C zdg9(+_Ir-u4<9-1R9h%4`b}QFJij~t?R)$V@^1cR_vfFH(v7!{TD0sLT;>Egzuwht zZaEv+<{lTizGaV7on!x&L-ZG>@`-PPGqNBB16VuR@vDQg3m5}=_a?E)&tQ{F4fHlh zBMq{UX$0XP%Yetev+$&Tf`b#lqDQ@$A<$V1^qFb22d?Wy(0w-iM*#}(LMfq+ZK@m5 z6Pvg|$sz2DL}Z{Av-iJ$G0$q)6C&=|1VAb(#f4^snDOS-zu`e-LCVve^+uSm%k67- z161MRqUXTYl*fpj8k^0QxWoZ0QETJ_UvXwwped$te<<{nBz^bC2D(XTPdgyFpa;D? zUGg)J1dby^Svoc{(v9guBZ-LUKGe)HYID+y&g7S#jU-E!JDjB0cIi2W5R#CFEC3)r zz%Yu8k~iQx=2QPpxgKSbvUr_rAF)!ozOmI!lQ+Aa9Z{%5RX#H%W&CEu+Qv*&a?+VG z+@T0_sL4v=4R+kbPJl>Qo)qeFejiiFFMnxBKnzm^$16)SlX;wSA_tw~EJ&t4I52oJ z)Q7knn)B#4OR0q}jNweE+z_f!6xqt4_rq9^^47_S4p5?di>B@1Xv-pA3YQYZBFF-; z&tJNu03Yz@Q5M(Hm2p&*<5SO1JBmZc9}I5hYPESH|V-B5daCaPxZqfu;P zxad<+UG7Y(A;g>*VYx?6jn17fEg((#S;zvk!yP$=lC^|0RVOwxtBb5_LZ`N*qZ&1- z@3dQ2W5+J5qE_vF7R{(tN4nV4nG&y56e&2dcFv?C@v8)k=SjVp#j}bof(@x@0fHG! zE&n~LD#p~SOH`Z4yMFYbto^1dp$5W-dN8xoBdhFeC%JV#)til-Sa2DY6OA_Mm|y*E zZXqgIjDo0-d<)_xx#-!MigvWvJ)~&^irR?)S4@tJZBjsr8}U95fEHLO-LH>zJmEN=H!y-!xxY@xlbA%n@p{H}5;-Zk(32x_XJ zPU^lZL2pbOI7N0;L_q^}*=K4tPs5u*azaMpX<2?BE49chGb*pp=Z^^PmFPh%Gw$PJhOiD0@L@t^u!s9!G%o7Eh27;Ez` zSI9IV#Ew8$rKPtO{!mjw1 zZe_?9GrCrhp3dZwjY1=abOKp9M!0Owoo*lY)fdiiLl##)MeaJ|(YAWmS=>>L_eg)i zm4Qqrz%E=_%f)&rHkfys<(*bYKlA|zDqP?Sad$}I`QGW4cX|bU=ljh8fQYjwpwk=} zfIkY^K`dVZ1t>rSA|62gK~SF2=Z}59k)11|rxAnnX|Bn(c~O|s7NZi%admrK^|k`q z>OH4uvd0u$S9z`0dW(cBuNN8WCwrLZ5GVlu0DrJ)Y}Y+;S9dN4cp2Do<;Hh(mwg|Q z0pSK_ez10iG;?JDepj$|e&=cUkO5SX0S%x}Cbw>TM^i9}X(aFfYqvA{;0F(I0hk5= z1n~fh)n+4wfL(V(lGkqy2pHt_VhZJcFC>8c)mKZkdAlTHMpb2p#TpY6NqzQzSJzgg zg z0emEym4}cJadI$gu5QAsYX@7_a_}~_!_;G)z4i`WL zR6qjohBX{ed;s788Q=#w_y-=xcejH7gc-(*%qC&JXm3pze@{p)j|G3JHH^aZG)zcl zx3z0qR7u!0QP6lY%|%9Q#c-vEbwBoTvgRC6n0jq^5pPIW<;X2^h%dAjP7FhZ4|V~o z$clnk01Ds^h;RYkh5-w(03@((e)jwXWQ4Ajb@F?D2zkplRuGu#prm?0eQt^l)5raO!!|8C4bp8J&T9E_(Tk1cl&?{`v?LIQIZcpObh{#7~lY5 z<`x@CeYP=tT$X$esSpy$Wx)rSgBOY*Z~-L1lL_$wAn*azVQKHyV<}mRG*yEi7l96t z51cl8r>GDsun#6TZao>4NNF#@*o<|xnnuBtN12R|mu#wcj{X5)XqGyI=ScX=ood0JOowA7&ZTOr#qNA?IKRH^R7B&F)cm*wY zkSOo~H|aerN0@f!2N$3Y;|7?_r;7?ek^!m@^BGHX&(TWJ6MYaFnHz`i3-qd3ovT^ zZ~y^G1#)SjYT*aOR!$q|$kouUTIKU~&L3r8Tk-mo}wJ)T~Lgp#ks* ze}Do1;F1I2X;pdvy0`*(hiU&XkpCCp2MQ5=aoULk35b7?4;pBnc9)+EPzHpl7J!fe zf1nQGMuZNKaskk%v&gOr;I9r~cm-gu>VN?kz@%s4ch0vE0Eub)D3Byjp|wH{^c5Eo3(vq^wX-dUfUs1Y;o>?PJ>z&VW0759WQwXfi=@3IGwOsZG z1j!cpd2&Sn0)9ZYF${ivlU=oy@0q_^F>6=&f*>t^?Z-hQyf# zS)eC(r7!2VAMj-tP?M8tnE!RR0t1i%3*Z2Vu(1n(2t8O#Ycv4fHlD&+Mio<%80#(p z5P<-IT#cg;_~2~-#eOXdfBAp`7(i-hNvg^gc^$eqx2r}l+e$-8oWF%f*Q6`~2fW+H zvov~~xZ0Hq#;diGWa4PFtjBsn%7hp;x`4Q8HA1!jfP4ZGrQ3TT#{(1I=ACPCX*Q^f zAKre{SNfj? ztbrJ)0RC_V=9_#Borgcve;miZw>U@k-rLzTBynV~II@$yLP%EP?7eqJm%X#MU-l*> zWMywsWTo!sZ}^VS=ka>J5uFr46a<3RF_n#i=ncg%>X=-Za;^&|nWXGUV`wLE! zrf3C3PY(}KmuuK7H8aP?rC3UvaGVuwA37{Yild=Vy}{4c@8{j1H#|NtqEn)xQl7hU zLQ}5Xp>K*n_{+$1{vGO7!xX?BmTySPQVZZvH7jES7P(EE927upra5<*r0-jP42cCp zS&DCHD9_u}VV@}Vo*t=gq64h3Q2=KW=w*x#?b3g+?rh$9l2F;$gj1i$&X2EohJx^i z&u#*IXz~+8fO{Ac%`M{KNSb{KU}e8&TG2~Wzp3CMJZr93gLlxS1bAOH8umpckThjl z1Jlb#rP@$0^x}mvL)9awO2aMr8?p4r)%HTc3UvqDHl&`0~zvgqowyZDEsu zm);RZ)jk>!0Tk6>?NX#T7E~DPACmQ363_k7`3GJ^WcIWRgJcyXFM`3F4*`#(*5L9_ z+cLh#Z$iH<2B$37z91`(KXBUH0cbJsd*vh~{P`ZL-Q>Q&^;mYl1i+dEVp$^rVMqk% zKAv=7Z>C%`TVpI$oT@)6q`sen&Cn;ycY_Ur;0wTf z%@qT&h9NDxfTU)8c5v2C`Swvc@t?O2rqJAnTz@6B{v-k}RxdCVXp|5j3fq;*J<30(xDMOhgY{)QjwboWcZfm1kV@}HwsbpAb^xrVKbli9K`hy2oDtk330biF<&g$T6)aGx|^dh^B zHHx2T85vWU_*yzzuNx=V`Ow;%&zVNlR+) z?E_pQlSnIL>}8{&dBMbpR|Is)A#Dq{W^)k);7rM|LqBJOQ_$175`gqm&nyWvE%k>3uhUPvMRjK6uG8$K7U(|pkU6zfVM?>@X;OcZ^ zgqHgRRz){gz*;0cvpF>}a^vtCzuI@|vnWtHu5zObC?|yk>c+g30e;g5npM~_XAMo^ z)A)=BY}0229=@NKDr+5HD0KR0uy*$VNdOZ(Pt`-DE|hQ##G(r&AdQJ=B*C{NDOCO( zPnfm+4&Y=>QQvP((}5X{f`X;EZdjAh|46vnfOgelo=9$|%$odnq_)ol+0C}WY+Bkv zI~Y{25yRSEd($>w(vDHsKK&PRkJC54taY6ejvMZ=JO-lY^=lntp#Gl;uWkA|rK)6% zWeBag4LZLLH++BgbuBM<NJ;k1d>aP{*ub-&~e_ z&P*D+^ocs~q*=HiN?y-x9az1;=1;aS)3Xq;_ z*DwcM8O3qu>;WXK*Zx^QhUimnPnL*fhAe@Q~ufRxWS zq&fUsLpL=~-r|#gw^}X?om?Gz+i173>`uLVWj8B9{R@f69UA9Z%y3%a$o zAnOXH(jDL*z{CQ)EE!z@L|HMYw-5;BmZh#G0K1{AGS0fIc8>s&^TA26EIk0f%*B~+ zEr(L*UuJ6ab8L!#3j5I~L;GatcN?}qb>@&5Zm13&?Q15%;o84C1U?;?ZxmTXy1JN8l`DXqY@K-{jGCAPS z#nF6Q&?n}9f4r9~UEiz98~XZR9NEnav9L((ULPM<{`dYlZg=4zykk`u1;P(I{s_~R zCK9NyWmx0mWAv*~0Lm72%X?-=0O5WSxB!Mwyt7JvQhG_7J)Lq7zZ8TzC@Z1VBjPQl zLZ2hqX{=^=INzeRMA>2jD9f>oq8e&kxr{Sq1EGf9d)&n6@#)9-WyToZ5+T%944ymq z^!*S(8dWKl#fRmqWxNtDsifZHuNIzXctLu1K5PCbbnjIpuYAR?P~MjRv_@;=41Fzo zMMTf|>bi++5V-!q#!v~g`npm)8a&-_j@G_@^I853@BCm=TQyuzaxLcJ>-L?DhR?He z=x6C|-%CP;4E96DhTD(2DL!{t^e}vWcyz@1`ioAivSF9uF=xhqb183+jU7w7j!kZ$ z#gb;Oj|tc6f=)t4nUnzc?sE8;CpPTWh@ieA--(PS4*=fmR`4~cbPf-}J5B75ktFBk z3d5tCkxI@ND0|Q2@b~fqW%QeeEXaLR|N&yur5Kxc^c@d4Ei&uiWB8pT)5xmjSE@?s0-F}uzRK5%6L*;m(P!h;9= z93`ZURXE`N=rHER#}2$VPcECO-nqE`sQNL}IH7Zwk#F|>KHfWd;WlUb^1jZ~ZQ9j0 zZEID8)CPJoB2Uc7C5%Mh~n6n8#KF^2};S+h3{a$xG<6wLIB21Pkg$VseNq)Dq>W z9gD->OGU9~P*=2;DCnatI5sG(>nx+Ip45*-<{TO56ipX|EQd3ju}VL|N2Vd(Z{Ff# zN>$&)#f*tFL>cJTlKP|-3wr!Y-S#XlD+ZPEVaie#1coSOOiG><^kMfxcG~p*CWdCo z%q=3}2Kgi*qpl-<2~~#Sih99QuKY^r)z-99PuIC+!}>m~d9^b2-<2cdSZ~)C;;0_5 zq||!m60(;oc~x^tPJ0CIJ;SRG!{mGfir8rW@hvPvM=zc{*(szzZiLkyb zPy0rOB4oU%_6rS_#mJ`ngV4a!ig#m(P$V|G7{HuD_?~8u8dTJXf@o;~-K5ca0ZWIB z{&_@^{h-VPU8!h4fa_}?WreW{k5{(lvm}DdyyDVuu5&Df_TvX+71i%%C>IIshI8rL zD-6938Kkx|dX2;Raw`sqy*;z~n10n7MfY(B6wp0sCYz3$EWnvJC#gN5@b=8Vg(3~k z^~wbOH7rO(X!drt6H1ZiF?76R9oDbH36rpJT-iz(>KlCkX`0UDn zqS&~GTYUh}o0sFD&;ZzK>rkVyD_y3qy8(kl})*^WNcJSJIQz&F=E6ncUO8A*-D2yW?=K4Bb zjrVxQe!RV{+B|Zq9y!+6|8VT$Ea9*?dqEWk(Doy6^c?qdU?}hU9SsJ3?+n|O<0(9rL202K-Ewzwj;)H$$!2FJ?TA_BvkCp7E){Lm;s(NL- zu;8YmntKVc`c9c6)y6wCD3?I67G`JC+a~VbyI2*YA8SqP~UPvc)~rs*kSm zBp*>+AERUw?fo0c;afSoGKwz$brZ_6_!Nf`OR@wR>GHuBBcpovHX%{N-2k#3An-4H zn{mynW*J3ge53)5u*{dD0>Mbmqzsp`SXn=PN13e3rn&so>M!F6tQ{#WkGX z1F(XQ`TO*-N0IPVq6~Y`R~w-<@fQz593A$Do(hR*z%)v9gStf>tnq*^53(D0OT-?T z5N(gIh>wLhAH}^9*!0+OWz$bcSSfR@rF&wSm;JZdE?(BnlcaUvGxfcU>6KO<{j=pi zdYA3|cGWgx52Fn3pV^(h9SAzw9CL7cj(48i*nbNm<9^c(Cf*_`OCp01qM&d&_i%Yj zVDjTzfc5;t{qnsk(YPuOjbCbYhd5^?_HUcRnmd|g!L=VXoExeHZ}QyG$(9mzkBILp4|0@i z2DKN(z&=|G_PU+Et|&Mr`+>9SiycW&452Ld8Wyp$(g?B3W?zpl`mQ#80cLNt7$aKvXvThN~@= zhtVssu}7ArKNsX4iI-HJyVhPkg>T^B1-inpBOx_yAx)t%;MNd)Y>0ffNSQ8XwvwC= z({Mp|f9*@YI^t7)FDco#X~wawFweFKt?FAIo)=i3fqaHE#s% zr=TSI5eCWH!bH9v%=2n^9+4^Kr`DzkI1^vky_@W$R(FhiS&HRijK@!wx-xR!cJO3Y ze<`UbFMX}zI^PhXqJst@BPA6{ZU@bn7-{1GZSO#5gQUj>YEVQhC>7pPkLi4 zZQ@oGpm&vTL#&`i21-eexN7A6ieQ8iio30haN%3@l^^2mo|=|cUPW1oLhB$|jLLs+ z28e7KnY|vGK+ArqQ4Ob5F*K-~?YFZ8d|>(_`7uesT0(`HUaDVQqrFeXEm+;_&7j4L z5mFQNtRt=<)6vQ*YPS#3)lj}>rK@h}bW;lz@c%InyajWBo+663F?5>s35DZs1vCMBjexh_pd!Ydg|E~oh> zKj#bc0VQ8vyCl0{0LrR6CSMx0X3q9Ekay574a z%|qw0hB)Sfl$z?S+90ji52th`>>!PCB7DCmjjf*qdfNF>VUeLSMOh+%W26=VCT~yk zvnAgpVz8`k4Kz75CXo>p^~J$CL1EkA`gI2C2|MV~Aprg-(IObp9t!y?aFZ#=$fvvm+f z#o{9&WbkH)JCJ@(B=3&-3ytSz?1QWrDz%L}d@;nl7M(n2{M@hqth3{8jQ}2><5&3w zUYzhXY@!-wP^47P);IMncOEm4IM6UZIc~HfU|O}zqw&bB{%h&yUp9n%BS7l!z8a;d zQzzqCJ>Cs|iIes6`$hk8M$?(~k}(qeR}nP{NkT}!8KbpJiDZMDg)iM}n!Eqh)aJ{4 z%io@vS+QrGU-~u%TG{5yx(LX_89K)npRF$NixAZJUzZ?Pn}Y<^a|-aOi|8@!z6-pik4CGQy7JCU?$N$#VDsd`Zi>W0I*@ljXRejpGE2(zVx z`=Ge6tzc9)KK0}>+@emSnjTJ^H01Hz@OGtJQJ`W1*CER_ftN1%`R@c-=l5V^qv}WX zwOvzVDQO=Es=yf=q@qg?gaqvj%#`9)TADT)p~n0pEHZ2ltAOCIV@bg%P(5E|17B(^ z>%*@MOnj)|BIC9^ehkr6KmQ1M)s!(31XYSEfVl(G6jAKbhQKfjP$tMjiQp&OCK`@p zYk6!RPwMgoM1sX2(72D8Hp z@moWx_#w(32&5ZFz%WLeF^mWm_OA~CW5B^-q`?<;Ygn~b7!neE1%y~&x=>-Uau~** zwSNz#DSaI&%aUC29B{Vvd^K~ws~=hn+V}Cv`JezY!eG|sTp44S7r(ibH* zGEZK#40OR$-XsiPnJZS13bX=@K`51Eg?D>20FLg5)fYx6+vBcaZZ}V*&z^70I8g3U|ZWD2Q4d(G((i z7$9GpZ5!_Y#gCx^?(QF$Vk)N}`P|ffCOyscV7)x(H|ZJJC*nsKP?(MC4LHjm?nOZ zd^+@YysiMu*9tt^J>ww8eC|WoPiY#y(^(=EaBZdyM;#W(kyF2s#U?aC5&Uk)7J)u7>IwkOoAlwERba@}RW7qs^ zqU6`)Kd)7RDNs}xA_@v+^;Sh;yrsRNju>x844y6q?{wX9UMN~b6XW;$MWR5q%9-o# zh1=!DOZ-cBk~=V=4qZAbI7B0zS@9~~U3exzyO9D`#v&YZk0QI~ZR?YI^I%9)b_AU1 zp?dT#GY)u^_Kv*_|5+WePj{5!YVPe{dy5~h2YWO1SUlAq{SHL`%F~)4Ct4W>oy~PS zaa6jz_cqEK{LLSIyl@1^YOAs6dfuG)dOoYOt!YqScf2F!? zQ2ovmJqTKi4iXh89pFXtC=H}3;(B7w&3CE!BvU>HAiki|OALkkO~ECGM)%)#5J*h~ z*v-7&OAO4({FhVem-8v`@T1@1=YJ>fua#Q;jxYadj9>g3x;T@bW~pjeBt^^tj&l z@|QA$x6AkL5PT`d7qkNFZQy93=KTec%{qJzdAD0>vF*Mr5`3fx7US?4qLNDeXwhr$ zOR-Uq;qf~OmoK#riITZjk|)dUhX*TyQkX}9%omD+H7{XDco2G#ms7+F58uE^)mnCu zGr!dq*_1-on81W^M`P@V?hvs5IT=zZYHhs>U+U+|>B;fK+P_P=BxmB?A3uRWrXp*E z)SYX1AwT2SduZ?MmUwR{QI5=nTj7y|cO|cwXRrh6T?>8ny@;%kXUnY6t6T zUzRx0-q&}_s9P=HqaBN{p2iqK2dERk*Koj0=)`C)0SBB8;Y%{L;K2H1RC<6aBp|6r z7u>W705rCz!E(5O62nxLM}D&?8q3pe5dO1rVt~*Vn_zuv^I=VIuRUMI2w%=pY$={DNS;#Qp{wFB z`cSta8r1yU0ObFtI%xd@f30=notaSkY6b&q`#!=E8;sA6e9u6*`0#@tWeQKe50!zX zV0mPluV7EWF^vIU3Z7qqtBOArM(3H}(bIke{11 zx}ab908aW8U?-7w?_s%Lt(&YZT+R9gOel?kBZ!kwF!e_!lmOho>l%e;XNwtT;S88c zMnzGEso^1kna_y8+h7nul(}9%6`l0y5S8P}8Gsihc;5qLbTxh2fA@v-l;lx&N_~H` zzjA4$_V3U_Nvn3<6W z>gKQ~Mm+~C_5b;E!aIdJ4x3s6A0D@U{_js@*Pnk6PaDr_I*!{Iy0T(r|N``+Y8B%{e>S7lTHrDF5l?~{0RBml@H^?3M%|GbH zw;?ii%pX^cy{<{fspPO3EW-_TZ3`QKFc^?=?NE1IC`onx z9%UAp6MGt@h!U+*>q6@p_t9yi$*v+Z#Q@(K`0j&PJBla?WVD6d5o8Y!tmqTePPrLR zRY7vKBvWZqq8e-nH8}_z#fjCvn;Nn%vMCgrZ$Vgf3E{wWNe*SPcpy@dObwY-8HI(@ zb))6Vm6|A+Us+MTqE!_7TEOd8mr1?7x!1ce6w6$LzVvv?;E|qf%qHZ?Nu{vjhF` z_Sy0~2>(Z&GrDlmpv?1Wxy4F_ZZy3?b$euhofo0DO(O?%4ISqyT9Od(u4)o4iXxgBxZZV-oNM4pr-10A0s%9Mt8nPo#FvH zBTcQ92RSP79RdQYvqfxOajzVHkL~~Wv-p#ByoS>zI@|M3QutKJ7goq8U*8RovW}|| z&;Wr`!nneHDlPe|pCOmlFFIdd*SO6&W8g(kGZBOz=|H0W*#2luANJUXVGD|+F-iJ1 z4`#u-*i7*9l!`?;zR_J2L}ZYJpo0rga)_CrWHAVXtI#>VKj|FyWZ4bc*{WsKris{X z`;y`kZL~cdU7vJ8OM$7fm!wUg^>(AdW8$w#AvRUA+pO=t)}?R?km5t^?YJI{Q~6Vy zvt5HoIR!DMF|1%1P*4EKiNAwH)=7MfX&avzf2Bb9Y%s4;{M$h8>$=K!6Dy)6D|z<< zzX|MNtPrIl2!Qa)V^!gMChYw8cHy615Tvn1_@e7&(Y=z)FM8Nd|6Yq0PX7s}J^0Gb zf;&W8;ieOa&RxQyl>5PRMN42=l#V|ch>F^BJ!s0`=`NFo%{G>#VkIJoSqZ~LF(9sR z5s07pFi_=q$HGAQ$Y<9a?wxVUdwkK_*`k$aqOcjdJ=G%95}Z#kJo%hp7E0e3-I5o@ zo#Y#iV8syJO+6wTf4<`^Mr>)uboRthie)!QTX7!hIJ6HDHL{b|s z2W*$9w@<$w@t`TY$;&WGLlSb`ES;TT6kIgMnw}iND5phCiP5C>%^nZw!nA4IuM^_Y zQGl?-10}k=e|5!9=rYb%a>zD2hwkOR(a98#$t~ir+YD_On~@!=`|NG-U^fi+NBr@Z z7M~U`D3NaPe$aLX95A1Th&w^lzVE5xw+DtXt39!KSHDg2`XlaG5@OF>{LgJB2BMs% zm)h-!0QJBGx-H&ff}bhKBcU~*Zpc+A>gX|+HH<|7v)nU>pEYwGjT>=b3wKb4HH$L8 zGiPwBhRSc}wcJCs-(QVyGQ6K{xVhYaOqQM62%R@(So`_KDVcs`?Vbl0l#nb7QX4Lx zb&r6QmP)!?SJzB)E%e)M4}=JuZX55Qw(8i~PiX>D=xO}}HR~tnG{tmkt zq^J7sm6uc{{aNF0%-JIrO$8R8uICdYY?EwLKv(XJA0WT*c+WizwYPqGOu{rcFgY<_ zddqlYLT{2r1Fe1jCtKr+D=$4SPZ1RcL1v_@(^;P{LM#iWicA%?lnYp)ATWj-p#)ie z?(Pwtb}yH^1X1sCTMyW2(2v~1_k;BF%DCKMV=4h@Fp_NuW%we8dc%J4;so85< z+A6ezHWru_cy3XPXWfwoFMiVgq8`Onuod5j* zlD6$pvVM*PitiOz$9%gr%?Wt(F0t&HSC47}6JxQJRCTNEp1G+5CIekvM5o+iQl}6Y# zKV4EJP|^)cg;RFR2VmgFhKkEt%BR+GRymZ0Q2b?P`*LXW-=vK?F3;sK0*h2?ox!>& z&OJ-c@~J|LjmU{W>LS=4>dHjl7?kK65sS9@o&6zD)R%!cm?; zi4{SC`snx?A@kHr5IBWaxvAe+kQp{##wGQ3m*PIxJECq=NfQlO68qOs{GzyrCp@n3 zJwTfKDmghA%uTmGY{~xK3T(@GoLwuhV=cz*!ikJf-0@AGeXCbu=lLF`Yagsr<=jn6rxF z2q>5{twEZ`TdfLOD9h%%1S_SbWDu0#ltf3DI@cFS9_ii9U0prCq+9Spv$s2Y!lJHW zycH%*X($w^Z?U_#hv}5+HAl+q-qmD72 zY49W^yeTsc1RmxENc-H-%x zORx=M#4K^09YHp*`aATEY0iU;Qtu7pll?3-X|Wexe%u57$8YVLqF`u)F=xR_Jm=E3 z0_X>He$%>xhD5P}w~`of*{&__K-K8TSnBaoRFM)QCxf*%0P>eDPHBT-)db!xF-A2`S z^1iE}g@xmw&e%Bf$tl-vj;LcU9p2gRm@u@!HrVt7b@n*Ra>Xf9q6m}G9tB}> zD_U_~ta`F~txxmg%J)`+mnLXU43NtDauz4arRidatxqYw?$JKdo7nnmo!m8tKRV|# zN=s7!b9>6#*In+~R@&OTn=fRO4R#FD84SAFrNR@qzuy;RIuNw;GRq}*CuJQZb)XnQ ztT;zEa!t3Z<+43D9msv^Aj7@8nwX*x9@Oc!QSo&*V*ToEttl0g<&~-{Q^8@1JI(&% zYBJj1T6XKJ$9JlXXxQ2b^+;guh(n$w`s!4{))gumfi}*myt9uHi9YDXu06tAs%OY6 zh_;6nU(0YA5bJutGcxjp_P-rK5ovnm>U4p7~xa(t)66OtaBx*1K3= z|JxE4#hc_Lp2vi-TVQ>d<5M1Nr)6xy#SYU#mR93GQoefG(p9>yP-@TgP4jABf@$c1 zA3p~^vEr~`V%p5KG*9cd!w0>)|8(%~v#F-J3Fd=qf8jBv-2bNB{{oMpmSb1Eh>WEi z@2uZzIS_8EMId3#Uf+G=x=vpqCw$KF!f}?}Vl=`?gSMN{@>iLbX)$c&aL%ZqN=*p{ zU-2LP>?(m_y9-kBjOFH#@!>Qyo9zR;v0r&L=cx%*bSZF=OH0o0L^v8~zLam8C^cQA zQZo-xzvpUsU7Fgn=CQmmJwDp}cJ7;n9iD%qkpWiGeMt)MHTn~mfl$88^U_qSx%>BZ zqT?bcWA5ufkC?wV$_g=ETOLX#pksKTSOz$M{KZUMl2*pGuQC!f?e}-hSy@n8`BH27cQl(O4{EYY;^4c zERF;0JskeBi^(TXY6+)g{P4NhaCkxOqOf~s?ld+e{E7{KA7$xx@C(yGKxu{~63Af&)z%TZJ+HG@<%ezpGFh{y1VtzJ@#VNu_=;D& zDWFYIImDS`@~#*IPoB{0ccT(+nRa@l+s)+JhCx@;j-|n81s8#a2JYwh#!aqOOH?nM z5I>K_TkJN|mXz177$I)Po1zbIT1{*u%^(u5&4_GI@-YmK=Hf?{Gr#3oO6sC~EX-U6 zh!46;q%wkbk~YsU92gLhYB{&2wbXpjpWITt*C{OvOneXQ^!xF|j|HVl#Z<%f))vV5 z?Hq&;jg_k9Rev}N(m9dXkUe(rFdz`+m=xi@zlD3Ct}iarnsT9EBmN;sNny(fL{2V7N@5^L~4-z zveJ0Kp6a9cTqxL70PNt$(jY~DAY~(7Lsv(Y<-?}JBc#DaeHu)co@f8|{gHX~sZB3k zOF36-j(Tv1rGfE2HHm?1v2IIGKlPJ3Upu_!GQ)PdhHnS^Z&aL?GNkoiu&&A!cYfP7 zrMDoucq}X4j&7mWI+8SQ{n0Gqzbg7MRj?tV(Ers(25OmU<0oub9X#f~V)jr2`Uu%% z;kMx+|0)`zcadx(IUn{Mk@D2_BTSwaZ)l#k^80+?tFJ3G@+TR()Voyoxb$W*<)rPV zg>q+(4QK7r*nm)Uz;iO7n`C1?ma}UbUpi_A(PAndCH%@)j9=$vbN#=C3ub0DcLt_* zpM-CfSLk+vE=xadj%{swC5MhV%Ix=Y^9!iA7M zY9YF1DW5=<7G-1Q=%T`{QjYB$*{!Xc;T+jFg)WoA5Bqv=966?T9=q+h%#ROQ`QiWZ zBYe0Mh?6_|p**cw^e#u%Qj>T>^hot`HI>*OZRoucf zEe<|4B(q2=-vIlZFNPoOXhd7{=Ez@2YRNLc-^A~v88n$pa>{AeO>jlqiyDqIEW})O zvK4#G6`#}CEv&+&p1uYH;tp?lQRil4mIDYFNE$lDbGn8Bh?x!;;A9^ZLvJXhx3Dj} zV-gry*tevy-v>d&kRpS&6g$mKVcv+e=wEgkX;Q9pvv-`t=OFoDJSv$qN4-MT@Dio= zG$;MiCm&SO54?CLGNjZ09hJ{ijHSYKBJL3yx|lRLOqV=mxL{LmbbYL7%P-Y!(BboQ z2lurj^X2nTL7>gwrzD`6YGK?y6GDJeEd_i&L1i7MnzeP#=H0>-_8trg?O9-)%A!`unL?o1&%X&QRW^X0}MT zMOE=^OYi$5`b410VLs@9XBw7ge9#xa@_>esNv zr&!3ne2d(O;m7j;Fod*J;()|eFlL;@E8>f`5+1F7OL~qq@BU;X z_>Ol*L{aLOb$EGU2t{b2`uN%Zoc1&9B8=@b7zXS3V-_jYSYKMXBz{G#{L_)$XI%WD za9w}MupE3^gREton46UQ+5Pn;E3}K^^b_orHTAE-*LDdCv8>+{Jz@N#Nk4wWy*!N5 zZh9JOwxLIfXVE8rUWSixgseP^=KNL5k%G4{&yjT5<*WMLp(@!u_Fl==_r~O*cft)L zZh;N#p+?5OTxQN_8=gm#fr2SZ2$Px=<&s$T-4WsMw`V`a8bomHUjht@!OYk@cGW}# zwI_P1u%0Mi){4~zk|eo_X}bZdj`%`bFeNcI0%4)ZdKQ8f!4g?Y-m|5zjT6Ht&?RXJ z5q!-|xg<9=88d^%j(D1;7jA|@A69yjRJVT3-^3ic9!68&83g$m0$u%fOzv&D+mXdc z$G&FTr>;r)-TaJdS6-${t?6nQNXfo9`&Fo$84#Q0!tn(?hfyr{5ah9Uep3<9%@FrAbt5@ms0Oh0HK58 zZFU~e!($e)S#p@B|9-L9&Gb@l_&*_ECe&#(YO^Tsv0g}W2o~QP&aM~X*}@dvUFrkZ zBgI$L;9}nQ@a2zRT`U+QEGy7@>qo#)E@r8e>&y+5R{{K+1Yr1c`*bY`C#4gf`(4=G?dZm*Xibj^ zRt71vkSx$?G;Sow_d6jW$am91q&*}C2&i#CXKB%*!uF#IDXIIkI?dU71AebO79Ox2 zxR>RQ6ZfNF1i4;m=@e>T&Ol&^!KyE_LfUnAL5dd=dSmM#0IP$Q2!h1LY3Cl|#;sR43cW#~fK` z5_N(zC(X+%)xGlhy68OT>w4bR{D5KsCBcfJ_DpJZp{A5IitpM5*A^hv@h%*MI+Z6> ziSN>nG>3UD@{w#)E?62PiGP{~{x{8=Wd{JXSA7sh6b!fxMMSBgtN}e48o*h8v4n!#&ar3n{qv@t7Nn#bXuT0NeOgfypty8wnJa^nzh@vA5*&o^^pU5QH zICz<6rd(Bpy%O_as{9qf8fgk2ene@AW~^@)6{=xLZ-9sJ(X>ibma;lsxV$5~X?5Vm z_gSq{3+UWTj`5CuFf&b&_P&rQ29^2hR5baOBMp0^`(zv+-`$(L&*HkU;GOY1U(3cj977b0gTc2fzcu4^nzVdWNT475L~8D3V5 zdb0Z+$+=uk9eEli>r&aV^6K@c`>NB_L$R41NtLT}%SUC5TYVExj+qPk0G{nv1R_Q+ zH>u86*^^(t=3YOpib983((l=aZAGc144fAD_}7(Klx7B~az4B0(E3qmlZf82dg9Wh zgYP`(%;b$zO!;J+$$2Na`2Mr2rQ2+`bxA+MQny|>X;_ZyDUQ24(HnWU9_~nCQto=Z zN;Efs2;wF%Og(|E*#q8PfI1!;AmivAhAkXI4A)Gbvlr-^vcx1!XTI#LSEd^l*zasT8U z+?H0<@5jW8;nGr?uIE24zdqAGKvj3vCakU{RJKnZROs+fN6+=2*rXl2@jjm8PAdP$ zKcBJ#+)vV!eF+IKS_n|#LZO%vT+d}{HJ7^qdP|)O2#&>T5Om9CEph}q30aYG86e-x zo&_N(f&0`XYGBJ+U=7iz1?4Nwj!WCowdjJWd?kIa@MFX(kymeD^ugn?G`BE5!q*p7$JpL; zuhF*!aG6Ejd)de+B2JUx+Ng`d7V8(Xxv@l*FZZ-`q}$890IQ8b(q~1EaTUU3uzOqQ z1My-T-pz-cNi~#B6T%?fVMYT=H zvS1~819Eggg0Y6p#6v~wUQlBtn>kj>SkFee(SQOIzzT^f$oyyZGfTJ*C7i0OO{h)M zndHJvx|OptcvW9!67%Dg%a7=UKE;13TZPj5u7~vHHBB$Dz|fPo>sR? z^3ax~!V%h1`6?|rS;Doest?x19Kxj@@Pn~`N1wOG86NWrY2iN}jKUi4wu55~V@}wV z!V0Zp_GwMzW|-if8$eKI$5)7+R5(8p4z8LMFiQ z>%a}lQyvxd-^VR#QPp0Nbx93wnHdwqgbOq>>NYOyKQ%+c5HstO@O*7mM5+{dG^1UF z>AT|Q_U#<&frnr19_5drDo7qFin`C*MGI~0lAiFvRHZpLTrZ%coc!ZTc(Rjfwu_F1 zA_ehciV2bhlw7sXu?;jsFS@WbU4eQ4%@od^%*v4rzBD9Eg2XM<6LxI~Hbs7si zPmWf}NJi<^)W-)}2RE|)4;;f^`iq&R2aY5}Z=K;H3R7CM0hFruqfu5TQ%< z*a`e?QjJ5HK7xq!Y_To=gzW721*EIEasWgamK#rD=_8iQq9=0QCXq!#L~BCQ0I5i~ z2E!(36d+yW1x$~gn2*&kvZ_Drlb|RB7NJhHcuUB{x&lSrN#}*$#&hS_gmt|py6mzr zP9Ct%-72w8kFJT>B+3~4;LS(FoigHqRz)UflGuElj!cHR)_H^cKZ(?I}$v4qRD1+uiyYS7P~ghlSRs58daU>AJY@B%sIiwfSLWAsr=4}_uw3u{1^T7- zeN~zJgpJ-bZW=G1JkaE*{6Z*?mObT@o|;qm_1NfqB=4OXZ_Y0{*@I|*atAxr%LwAn za}b{tn-v|;g2~@o#94S{N)<6svJql!^Fi#r!;18?dHG?b*8V+jRWR2kETg$*3 zof^4$1q2pt9j*E&1|d%SUw4|8S&NmC_qtMo_p28~yv?y<2}C9#4^;OQ%1B5|93$Bc zxgKZa#_2~#5Vx<{#~j8B%Dn*P{$p`)AzKtB*o;lru zoJ-jF*|7^~rqJv8%ZGW-l*LOFv{w*v8ipkFdILV`xJ5}9oKeuo{{TZkyuX%!57??| z0)a_cSk1$M=XUIx^;pj??V8nsv7&G3n1g-!F97#T!Vv>9;0kg{+eR31$Hpa@5yRa7 zfHVNsXkN(yGQh%kDILFN0D{y_9$+;Jph-*fA9`tLmI(whFePeu0W1uPKkwdl5rAj4 zinj^;y2(B=ZX@rIR_uAo(0AlkhgEbB$|x4IoDcWnsb1YQJ;J#460Uijm*fhk^eF8k z2a1Z?Wp+7gcA5{$e)7*IwQ~ewK8m(AVnP8XD)ypp(ik!AVx=RTQg?1AMMk81;HVT; zu_cuARbwiDQCU_;88=(XrX&;)xpt>b51T{}*bsDEWqGOg3q(P5Ri#>4nd&ZcIW9Q3 zmw$Py^}>Uxf-f8wPEcT^d2w60m7NW2@1EH%J^(d849EU}tsWo)8&61eN{fj96y8wm zXm|+&+N#5zc!BOpJDXw(XVU}M^p8d@ovI}!)QPfA4mwapSyBs%5@b16NVS^NjZe#4 z*l^D#_Q}2&vr%X~YZq1s8>!K&uMV+Y{xi)y(fdjcj`_GZe~X+T&N%>Nj5x79vo2j$ zpicl{KOmBxMfyGBw@;KsRap#Oss*?wx9+XZ^6R;Iyb!=PlRtNZ2{}-0EJGGM; ztwXOYOM{ohLY#$BZu#q}D_HF&IG2&}NVuB3x%$2VBbd1)103VVaO?q!fUF0{tj~HE z)%q?5OC5ZSD;T3vkhsNW?jhKU3fy5e!fgR!Nt^%MzaPMwA0kUB&%Sy8(4EopL0|VH zwIdB}NcWJpZE#Xi#{+vma-$&iZ+@@EQD|HMb(|_P_iQ(I#VjM^DLNJ;U11SyxT8Qc zYMnkJdy^b|KcXVwqsE{1cZcrF7DxSV^tkLpArjyy;q-D~2hg6QL>eb|+9HiA4e4IT zjoJc5PV!^<&@4^4LACN#yYPyOx@*TWLuJI+1k9y;XM#?G(8JC{XBqA4_88+^mvfCX za00&#R?;awNW9uY5w|GC`@IUoOzh$S_KIf!p#TcNTm}mRq_hAQ00Z=THR@sWXlx0` zZ7Ej35#-90@Ir$UmNw~5H|J=#q04D2f^~?<-%`kiit}1Ba$D+^P)AZOWnl#Tq@`~n z7Pp=ax`em1uYGX{aU?e#Fm}~JaOFjc{Z<;Mwmh*q=40rD5)^l`I3|u~LykLkVl>$n@cW6l4(R^nd`LbDJ; zEB&n7R%?ou>>yYz6dmp6(y6B!sjM2(AKiaZ*l$G~vHv$7O!WI#-GIOgoYgv=UHy)& zq!|$|VBGu)fE=X7aBvzgU;$vbDOP}iHV!XTx5U@<;P`pPkHkXqMm=CMoW|3g;EW=> zJf#I1`J`T)8i)8o+sg3pwrHZu(f4>gI^j5wrX1v=I_k<}98|HzqolVX3W__T93ry! zQl)oJUnf&*siFDATrP_4k#bSn_e0=ZqfY8YQtD&uuT>v@C1mB{L%HKS->7kwVSl>M zbz1Hvom2+BOTq%Z3lysBR)S}_0mRU;m%NuZa2Oc~FX>)g|Ul0b)hJ6)OT53?MK7hXDT?I(!H* zqQr?44+a=;(IUf#2oWmCh*96ig%}quR2V>CN{<5z4(NvxBgT&^QL4naLP5WeGBYmJ zNs#9NiVYRQoEZSn2S5dL+T@v%W=?%8WcC}dLLW}7B2y0ddKIh80SY+0{Q425#eyB# zs?`WLF2;;;EzYfL@GjkrdF|posMjytyMgHv##^}WMxu-xJAMo~vSf~bEB|fybtpuP zn=gL-mpSxT&to%_2I(2~=&`C*_k7(NcK=VqeGf05j61jP-MowT{tZ01@ZrRZ8_(_G zAA;Psk2{a-M!7`G2P?v5-26H23jZi*rwrf zpa5q?DOM98B76|ihz6F#qgXxC$X``eUCE?KnJpRCXcis>(_3x5rPo|;BBoeffhDGq zLvqGhSe}5%1*c$b)|FqN#!*M8UH_teW+$Q?X_hE!rJW`kX{N1~8l|fpiD_%Jp{6NF zf_@6Bj4p;sYN@84YTOvop~~t+$+gp1q1M&e>SGZ6!@{Wwi0}X+1w6EZ0~tuR0w51$ zFluBC7+_zm&f2NbO$oxZkXk2InG;Mv8AX?pO%3#sK{pL_kUr$rM9@X#S}0*!C636V zQyr;@l0{Sqe%W?Ss{%B&SW@|3io8vg%l$R zRY?Oyj9V zEEF-D(a?tJ+EBSt*TNWT3q|4s9aX^ft_|r-UJg5$&5(GyQq5*y5G+cB0+cS{(FBQC zipuUTW|VA61vN9g;z%H-699HBVY^yNOLRgc!;LE^C-Vta_=86dBuQmj@l08?a+fo) z%OGmHU;AuRzRGlree;8#*!B`O4vw%mh9p$}E_q4xWQ2qy91UhF^_s3BFjMj)5hU5S z!by^?imGg-D?!z^R~jceumX-G>t>N!F>mP%P}5H61k(TYxDIBeCci^Z&# zigLuclkBKowVi_sTQ;W_*m!6}itF5nBGeINCgesKSyzjKIh;d!(2Bw{l0as1FMs7| zAVTtyjdJ9zd(n<^Sn5ZPY$7%6h^RsWQGikap|B*O1dmy%-9@UYGVaCXDq-5*_J}7l zLi(^SnEPfaVT4NenU78#L}VgQ+PaVGMVMx*WX~{pC`hH|AF7ejbQI;90uFGLUn7wM z7bw5!uyk#{9I8?OkBXZL;s%`;j3p&4^Q3ix$~)Y7Wl?uCfch{Mse6Nqzc3Uoe|eN! zBMMi&nCKDlY3xARED;XzwICMN%U=E=2wUz-pt-7UMD(f&#uT=?Qk@b;qN2&cUn1EgLx zDld*I&7~>54K4f-|O+4?u-MxZ~aJwiHn#$!tk7BMXg-r501W>2Tt#T=~$teat=E&&{&Q-e%=2t2;%#X=#etP@eFArzT zy(CTpv`S|6dUYiWxu$0CWjW0%EQS!u*NQ$Q))tcMn>u4FgnDQqFuo;WO2m*|>4IQ- zsrfiRqvHCYk}zh*P_%YEN(-*e7XG-gyT$8yu_!-OK$RT4+WZ;+O#t` z4P})r)l&ufIJ%OVG_(iB<#Vh1RAyc`M7Ya3G%s>c9^gkUTbb^NoHyULoFrb-QYCKo zrK2da^FLoKq@8FkBZeE^!#HrU=WsH0%3GA8uLPe00P)KLg$xl9a zAr8NDC5*E$L>1m2;VEb)D`(B|gp`ras<`Il1c^w>coytYta!eJo~2LOfSMaBolSKj+bzyXM$D2y3GUEPSy5KJDP?*RZEqtGsJK<>CWqxg z?s{W0-5b(w!Fo5%UDt>k4A73f1Fkq(MX#y&jVioguFgEFjZ+dP-%YnATxoL3zJUEM zvz~6OdCizvdGar6o|SMuEGJ6vL$skqvN>t(ihC!E6FwTrCeia^8Ix3MD}iKquc?av ztOJ!OEhVBhfsIL8>uWsTv1MpMtwO{UNbuqHOyYx3->;9(`}r&}$mY7VU0UtZh&_Lh z5<5Esmlr3g-D_EHe^2+~WyIgzI`gmpv)YFJ{OjMPwA{b{|8E&D@DJhufCXrP76)hc zH%vR$Uf|RLuVHEXc#6#sslPVGEV989~rYzWmQ7|Pt_+O z=Qu_(o;Mhrfbym zJsP1FQD}X{rcsu(TaGkjEjSs=wtBoZWYAU_(`HCtcWvXhew31JQnqbp$0)VIT&C1W zVbfsor+`mINp*;ak5NJPqD~=ofPaV=aRZ2hn13%|Z)Qb^iRe<|@DGNUhviloY;MT=)tN4(>DKH*6>w=E(uC9c*WLt=?|v|7aTPvH7Q^xep}OEz9fqUWMv72k6K1) ztcZu>_=x}i2psi>024P?i1daB`E3b_kZXq#e;0T)2$7T%kt`>X6-kjNl7Hy3FB`d6 zEASW=U;qjLiHb;)2JvQ&2#~*37d`YL8Zu!2Wg!6<6_m4Z{dFVx!ijQKiwoCq9)&}t zxKMi~f;iI=a^{8N749W7bL`YM|NaaKX)Wu$yssS!tglkpdGR^o$ zw)RlZ*K4S_7S;Dw)^{cZcWfJq7y7|{en?ng_804TY{DfeW~i01F@B)L7oHMjd?9X} zBBQVPcAzqu)47H?s-wGcna*;f&rxq0WTUgf9C*PG{9ph`nxsp*q)2+CO$wz?Dy9ES zdYe^xn@)PAO^T&YdYcyDr7vIsFCeBb;09#s250I4ZZHO>$);nVnyR^)EWnzr37fJ> zn|E3PxyhTo`J2G`7=H69(AhwiF<{WtUl&4=1{N1Npl1#b6(?6j8}varG`SLBVOrh2-SNzVq-9rBo)F?T;e@T zQmekljT}jAQfLwTGF#tR86$;0kZF!b`J?zYqJDWvC+aBHCapS!H918Yb#r#@$C%t& zG>IpuJ1VZ@`Vby~o4<*!>FS&6%BSq=o9~LQ@G7tIO0Vl03G<4t^{TJ=%CGiIyBP_85eW=in|C^!XaJkADW`I}nh~&?9uNVgIhtvDre#{D zV+y7(z${jZvMGzDOuD5j+p;d3q+6<`F&m{<+OH76EF3|uTXq>WghcDv5u}D8M?o%p zR%cCtAlQkMe|CYN*l3PtapK}aA|h%jXmvo=cFyL0vB-MCI7YZ8B2~hajMW;JxO?)r zGVV2?*3q9T@&U&f8}{^kIdO!}SYmw?jm>Bp!AgCH23MK$mREQ_SZk~?qO8jXWL@^G z(5gSzN|=%|m@ImRHEOjqMH!~#t(7{5hZnAQ#}_*bx)f1wiEyMxnh5_f%MYhpx}+Pk zs%yHXo4T;ex~5CBvzxj!Te`UGq`RxTy(^`>`@6oovNIdJ#S6U0TeC9TvNU@Mk}##p zo4nDBq#deim8&1xn2IG+GU6Z(?a;l+*#HO-zQJh^!08C=;0U_Oo4I+Xc1p1od$FvU zu@um;Y)YCT>jpBLqyi{Zp?h0!Az=UHBNB2UCisDDL3t9QfgW{}jXGB$*cTRJINF%B zED|m{K_er{sdQ;%nmeDZ!5~I~6&jW$)~S#nK|LZY!kf#LD2EYCp*+i_VYfp)X~`r? z*ga(7t9m6@Em?&Om_*tKqC8`i%WAzx)t6%!ewZ|v;wQPHWJUj;5mhpk8t$h+Wpl*- ztHoO!5fI=36!67htg#wPzh%6dW<16ld&XeA#$Qav99zb2tj1()0dSngcU;DJtjBrG z$9wF@VhqTCEXZiw$A!Gd6o3x@;0K1x$bW3d8he`5iycZyW@(%&Pr}9P11NLhXmE8e z;0Z+lqKf?~E}WPy38NO0CQafAK0Q%+qb#+P^+H*Exf`rbUC3KB+I{DTG9&{MzT73j z(<6C=Sw#}ednC-oRlVx+6gnp+s!Mz#P8^i2Gm??q&~1HFz?f?|LK31$*cZDh_a@Li^>=>;}Y$*Z+&pFjKtr%_V%v10;s0-cCY5$#v`s@Ll zDWXJS0tD?~2R+hgy@ji!YB=g$3i8QB1R_-XojN&JBSU$o473|)XeJl6O>2srT%(!{ zKh71|MD{0C6iJkkEdq)f3W^Xcy=N2SYrliH!~7^bE45vb6M!%eMSu?CiDA$Kp)5U( z6Z(94TN&CII@|cvd{~ZHh?MfyQc;Z04pbSAun+0HKPx)9C_1fZ8+NE6+}`GPST{Bk zG~8;9-Q%jy6X4eAl`I2Y&?-dNb^VC4_!=t0V(J1rvJ-nt(mAP0s^DaFTmNYtCpUZRKnFD7 z!VIdzKL^w^ql7iA!`$;d#0nR4>2%e{aaQQtl|+aAJzbS?+)7M-g1OZ;KCR28Tw9ZV zSoYiluBqmH-q}s$YW>e($Qa;FU7$L}hnF_jz1Djt5EG1v-58d}pIs6ZYiZ!L*Q(V!wEsy18-NZbKu7*L zL#%OHT5%N&3XQJid=O&dI5TJ<{ULvWh%9czf(zwi4dYxDh9qi9Ozh4!o}$Smxlc^i zIhCC>dg?>YLPgH%txgZHVW_k3NK zZru`Z_rkHzux@`BA3+)aU3ZAZ5bth+7i|T0w6AP|L9CIRQ%t{paELffSQKDtv0!Ju ztgCnP&2GdnFE*5st3ELx*8XA;{t`P_`X)?sH<4;rD>^fBj4F0o7a|rk=HMi;?%We& z+*20asS(Vs)K9G@sSJg~zk>Vj_ZmOebk8cs{|^Pg>E+Y7P3*YrO#N!#tq8e5F#jeZ zD)@JO_ut<%_~8Zst+^5Z`9NeV8m$K@>N={7}V!NnPZCJX(FHl z5Gxb}(ARH8fB^^tEKImip~Hp^9V(OvapFUU6SONn+{iKG#D5|GEj-A9BEl63^$FCc zAfd{X2n{OG_fnzB0W1Z+>}PVN%K-yK4kbF&V!{;y`+Wce=mSoIFbDn|&~E@unh80y z3>q}R)`|uL^qV-)q1l5M)9MS5cH%*}WCNJBc+u|Nk9Y$Vf?NIsik-ni%rw-dO(Y9>OZqMFUF59yhRVrA) zHf~zdX-&>mKX!#!j9)d*uN(2A{frXzi;th*BWLFjt12|mk}8ip0@;Epw3W&`usrt4 z>#DiZa$3nL5k9~wsh3h(usobN*l+;0fJCfwmOD-2xbd0|LW^Apn0He#X zI>KyBY%s=rysxs!Cd=$d%{~K-vLclPkiaKDvJXGfgyQi^!EVHIOD?W5yv%0l5;RJxeo}tQF-m3#qzDa_Bum;rog#4#tG48`5VJaaEUiJNm;xlJnxc}a zs;v~jX~2|ly$Y-WQ1q@X1r)u@ExYJSb=kCFbcj(+g{;rCz5-)zOKE*1EXZt)tjw|{ z!98*_0?R%1Tq>jd?@k`IWcOWoYm-7a&wR6Ag6YiDDD(PJ za4t!cTF4;qMpGzZlR_Qq&#@9Bi>6^WeybmhJGHdGDrp?qQcC^n&o5J7LhzYX7aS_H7E*6T9~F4n)*Nljz#3qDiK>c@G^Z8HL+=g6u<{B6%8T_S%?3?lhIj8 znSGekXz$8YFN39{_P1`+-mBH2xRv(XaK&BnRLsov3SEiQ#c$ezlVdy1zwZpZZ^8@b za&X+tG#pREjN^Cl)2;!{W?v9w&Dn6Ywlximg87;eUmijhK>7ddYuD?JNyr1S zi3!h9APZf1Kn9@|7OG;Girs%gLp!nY2UAi~8JYf<6B4b!X-Tsl5x+tc5pjTsE)mfH zh?tU{?W`V*Ngiitl_5Z6NNFy!ic4CuMXNY5j0jSiu*7!|vA_pepi9+S9CE(brAjX9 ziyvOb)EfEqtd4tR&fw_hqi+$9T;p<5A(KSF&-ibEQ<7aoHi(-BK2nmCq@;2pNgFrd ztCFXg;3l)A!3%=PU7nm5=7xf*7D@yvZAr^O#3DMZMXG$MYfy^{RlcjyrG}l#Aq{(& zmmAvfQhpHuJVa2;W5VMA$t-~gmZ?l+ilCX!G-flQ=}c*2^O^t4Z00;VU;+(Pu|$Wu zo>QEXfR(JoSMo6lux_QRnrKBIjj3Ar#P_v}oX;-cF;>^`LPxL((^Jb?&M2|NM}&3| zNPo0ZWZ;<<$skF8;!-5K7OAcPiiVW30p&(L`mRfUR5u}&%>zYR6vv&EeFhB}mO83Z zf&Hjj6?u=cG`1e|s7@il104(*CcM#EZDCe3A@hD3yY%repgRm(%F?x?g=|2oMv&?T zs9II4PPM96VeOoqEWAc z3Qx1#LLR%w=uy0sUHo+9n`^K*fYg1CNPkCPa4X>WD(rRh6M^l9B*cM5s%Z z3IBtF53_bBG(yV|uB<}5$#%tQBytHEHFg)RUF~XowQ*qc89o-;6Mbn5NgoaT=erc< zv5tWZ6eW9=mh9?;3reKjHjB5s0aGI@)}se0eQ8XeG_{W$v6?b%;uCYYY@Zerid|)% z)4lQ`v{V^xcgir%xl+crOqFr@IyTP=N-G_5qFmk4>w37O9`z`}0jwzj5l8?Y%0{L! zi@D4LNTUSsC_%M-$jYCDwp>x?ChBORK)rj&;v&tV-noZab*uEn*r8UFl(gCR`k&&qqu);LjURu zni}I9=lB6P4bzlz+-w?;V7)}$@%O3NU`A!*PlLxEi@cmY`~<4T@O|GIj`PTnaYqq@ zB5Yz8+n5rNKnJSOT@!$F1RD5iR>g|durfWYW5w#V6?Eq7UC%*>UJ%W4h7zTrghWAU zfIGr`s6@3h09lk38Nm*{U-7Ohx(z%7lvh!P(_20w<7$j{X_`n5yaL9^l zj8FsOfZ|#r2|s+|kY*$eQSPdfpZxNh=e%t)DO(bMnx!f)d1y)A`P~HiH7$${hLW2!P1TJ^r>$^1IuT=2GFtmrT=T7_#+6f zc&%Ubq8oj>)93*At!skzscK>{L5lBKI6X9*D($%P*4aUJevl2|14H`&d*>@5boVM$ zVCe}x`#B@7nFy`fC%mFLyxTK>v!em)v%)j5Z5g!3fDCUj5-7ron&_1Si4^OJI74c* zbBQ$4K)o^X3lS_q6ZDtq_&Ajcy-aJM)3db5!Jrd_E!qk#S31BN%MS9e9=Ym@p0ggH z+Xrh(w&2??6No;kgF3HjKBjxV=qtXYo31LTf_Y#zV;i;xxC(?a5QoW;he^Ac`#KBB znFV>V0*SA=&<@D(hqD{Q60r(%8X6>8I364TcPKkM95y|iqI%ee#P?8^i0LW3AdCCj zuR+l#z#=RmVZgy_z{NT|#0$I#gSeMy8pd0Z?9x zQ>J~W2XP1r{|b-ba+qeZw|n}(;97}VGpg_-h~;rLFw8cazzIQGx2*Vz%cut{sUkJw zfU+C50T3EM2)bkfzU)Ifa7qQH!>V<-JI#>`ddn}pIGyh4yT4~I(|FIpL(N2JcvYW zIsWpyLOei1E3AcFn}&4A2aLoBteacmm3ON|qj{^GL9rYDeQnML?&Y^0d@3>d9pJ( z`#bfCMzt`?^6@)EAuPdL$iECs#v%+8H~_-@CqhaRUs1StQkCp6km5m#1F10nc&daD zP4+3t&=k;;)J&0TMbRrY1BIjl#k^xt7VdBlxrj=Jc^&ZJ7(x-AuWLD2(y!a<2dhyb z-Fg(Huo|oBLiq@uuN$o8XfFrBtqm!WlmMA6Vh`ga!^8B4@`}NQz)tOi$9SASr!da* zYqKg_xE|~p@#H(8vM0b|k$Q8;Kx0qCB8&|%fe(O4ih>MAiy8>Q2`@S!5)r#EqKJiw zOefL|4BWiRguw=lQ#lok&zq!6;{VJNdr&xSsX6TpJ;f0~+{UCR7Tf|ZZ6wj-a*GW$ zMBsWBokElmCDE+GF}oPiwWv*RJW8c_h~+33o{}EqHYE z&@lPa%M{5JLnZj&6Mm(ULJ1y1+`IJ<3*S16YP^^?iX*WAoq>(Y4IRKoF`aSBt%~T> zB#W7vvA{HZ5b7xwr{M}Cng1ft%!p873%0-zreLzkXc`2mQ72lsz55=$%geJm*onYP zA4$Y%0Zf5A)|dp6dDsAHxvr!u+M-Ry5l97LNQI?qTDL4lVoXb^o!YVLs;29>%nK=d zbyox0S~;y#Ns`wdT}cMzTI;}92W<#idZpS_Tlf&2q?AzMF_hlq(4S(96vZ(YF$*cZ z6z#w}Mb$Bo9XW=W373&41Cg=@8N=l?5Oso#S!4+6*@?At9?Ak0`oc&BFr&SC+ZMsA zg7UlJl3B-S(kfNGcVP^F=m2R^KG#)Cw|qY3TZbmZNkM4Kr^Q`kTt0nl+R~gp;61{k z)voH|zN0%j2@tQ1Tc_96Gh62sO?d6v)KE~eeO|~x+mVve4b|DZbs=Rb34e;xz3h%w z;zrP^+kw!f`9KSVB@~q;K(hO;bfqON}%s0iPXWA}(ShK4K(J;%+$& zv%slTfr-|Uk2oUK;yRB((HtxT+-2e0qimzQ`v|4vlPCQtSJOZ*%nqe^w_4dXnb^Pq zQ8=Fvh&tq%ksU{=kj|&L&n20!gb)_KbtC?Ak?}Q3MC?07C7=H4HzlsY#mHbGF_E}a zBI~KLNmintpo%ubmHxEfsL52GkO_hq;1ntd<^f+DCatVZJ(AR69>%nj+#piimqngs zTCQbVzGYlizzTY~^Mu%dCDFe<#0|AMd~%Bctf_|p4`uF@+vG;TG7s2`3yv8G#GtHq zBAzr{vbgfCtALQMC;*W?JDYI{Z~u;!`6DMthP);M3Pes8(ZvghjU&Obr#|5?!OG=- zL9qJi5V%SdBy!4dgCau%yP{&exYFNJ=`o*z5Sc<{9QK=4Zes5^r*Z?8M zXO8Y@kN#+ohM&lxVkGm9M!n}{VVx4iTT}gFd=o78olDl))$&nFlE9eh8J=!5&Qj%x zQuWy3`68Dc9=HpNRE^xZNKBN7LlJl$MC;gEwUDyIMLV>`BJ#Ojb0SIpm3C4moz_Ne z98ugfpRM^C)2(Glj89DAPIyX8#f4e0u$9o53KB}EiJ1!DGGjs|YMs6dWq~Q?g$|0A zXu_7HG4NpuA}JHl=)41u_5T1+rGjjajBLuDY|EBx$qs4F-fYfpF@sUA-U2F}d9ecw z=0-8f9#bJpZKKyaBTMbqW#&em;ZTOQSgkuW-!W(|#9u?(nP3^)0Jdu&Iy1I&3fm)* zq)D>LNQtas!le`5WIRpkI$nDuI%NCFViMPCBeq-&He3Wk@b=Ok#96d{$VW!BIZRVF z<8A!CI@=PX4#BMlK{6Q9nfEB!1e8nB`r(Imsls;X|6UESg`il>XyBgc0PmX%o{r#> z2->sFgKXj}ZlMunlttY;QqEgs`5fH*4mC0zKG7WT?ZWlw)H4IOHr}yzJ0h>PW3U9^ z09Kvlycs->>#ESdU;l}x-(|w5WkMq;#w2vzu$sECN`YadD(99yD2(n=8f@7xIJLeH za=t(YbGYL{#|os1s8Kjk;feTd>ERR}(lCiCLP_f=+XO!;C%;UKHk{MS=n}ALIrri@ zuX8%Tb34y-Jl}IY?{mGI=a~g`uNibf7jz~1igrr$_u>E{YIH__bVrZ$cDgc3pL9!K zbWDHrOt17!zjRIibWR6#PlvKVFLgpU^+KQ7cW%g(%dHKi#(*V{>7 z7&h`Uwrs-gpZ_Gf;7eZZ^TiH8+9NDYc5lA6WWvvk4tYj(dOvlq@$=XD2Pi*ptI&rn zdR)(_QTm0^9Ho_)X&FdS2)Kat5Yd$@>-0sxy)_r?{?4E@zxYpU^TK)a4d?*tu0oKv z$B-X+kuQ0YKl$pe!jva>mS1_7S9zFk`IvusnXh@8zxkQZd7IaHoEKiBjc(#ITA?3$ zqAz-*KYFB3dZl0brKdjYetM{nda0j!s;~N}S9zOkT~G{Ojxda>snn5B=obcLf^|uK)|mx*2!k z>EZEm-WeWmKEJze9h)#&@EX41V|t#H-il|JiO2tV;P01LULe<6Y~<6bud-U@NB-qc zeyV+b=zspF6GrNve(SGV?00_cZ+`AaejxvT@DG3SAAj;MfAc?o^bf}9Uw`(O{`Pw7Lk^$!0G(w9DuC#>B0vBI1Op5xNN^yBg9ZZ(Trp6fz=#AB4#cQ0 zKtqHO|ABP4LSMs#16G(sD6yYO1tdM5L^)t)fdL?I;>@Y@-@%I<5wb+N0?-GdfDkH} zIg_MI4lH{@jVg62)v8q$@YJbc!`7`vyn@~ERqR%-Ud?trt6{C#wrtea4a!%iK0HtpKBZsJE{G`U30QuaCcPKK=Uk@8i#}e?R~J{{P#D z-aZG+SHgMaA()_o3rgo)ZwFmA&_Wdb6o5b$O1Ke0POWrMLQ+L^;XoQf1W`W(JrrVu z6lLfUOcp|g+#qPOE>TiIiP859zVL89c* zUlO%M=>Ud#RFYCh7La65O|fLylu<%?)Jz;Sl@v)Io(R>KR#jZp#XVIjE>v0N^lY^4 z+y!Q#DG_RtQ-=X%kYY|jIxqjqPvy7*j+R|okh1k~WvxykuU4(k*A88@(MKblw9>S> z(azFDKOMD=Qd3d zyz#~kU8up~544{58Ig;fx=Rl904MS~vSJ}L`0?>tS zXayjd%+p~GgD68F8Zn8%D&qB~Qnrj?Bt;qfTDd3}p>qkOY!0cI?JB3e>}5({lUo+H z@HI2UXT-*DtBiM$OHgZN1$qB3fyOl8$3;4~t# za*45=rRY}qv>SyTq+=L?kcmpPy>eCTLhribQ$+Nx?xAl>@B31wVuquqv}S$(LWpFT z^soHUZeR^?N5KrH7o?0tG6Ps#Q3f~w)vbV=TYJdZAPB*Ucn^>h^4LZ&C>v|tB}O@yeEmq~$#++AE1#|1_gY93u5Fs?i^Q zG^D!nsM0tl5qh4>BN^+6Wz?uHcBQnFFGXR-Di*UBkqAe(OG(~hgt98}4Q-rM*r5i{ z062P6Za`9@m52f*D9y1)3&e|r0?ErQ6;hDi8|248R#9LnvSj-_RKDnlGQHJKCP1O1 zN^*oL+kI$%StRL$9!i&@m6D=#6)9`@>er7Ri->#uD`5+JwZT^JAPe2vmRuH^p7J40nXe!kqO4+mQmLMfM0I@o$&?7Vsn9BgoA>~r#ja#dl0@kP<(y+t8Q8gt zktmob?Pc4lno@iHY+84@tW{=W*@PiwN;)d-IfW7_%aZnX|CfGcJUe$)$hOvm~8$eNZjpR0BT6e8=u#!Io>gkW69Cx<+aDo6S9zl+(WmaE3+1oVu*5S zxrc0zA~5wN+Z0RL?%^!D5$dUEJNvJ}3?^|LAsjvwoYaf-C3S*vs%!xv&K^-UV{3be zZefJv%J~++MjP&+n0l!fYu1_u1@uWOl#(zn*+f{W|6X=wN#q$5Int75){m73>Azlj z(t_2fNxelP8Et9GexZ=LHb$cLQDuMQ($+#`G@`+%#!DkC@Ca$kTpuO)f2@Uyaf{R> zqeQ8nMC#zSV3g-$$LP*=?l7}+6;`_psxp$~h?R;|R$tQ&lb+PJyc&`zsX@fXropI@ zHEr&5quXb$>~6Z9m+5w2FQie~toW)dS`Od^NWIOZ~w2S29)i88Pd!!xDG#_Auts^gIp=h-Gd zIU4z?u*eh`QyQbDHuZ#XPusqCId0 zvLl_PGkYn*(zcOLAl(XE&cX#S$}E|FMMVhT6sWKTI=q`_Ml%Ry;v_Z`MJ9WnN}WH> z%NmWG^Q0;HSWV)%l(U$GXZ3QaS4=w#eaO-)NP*)GrupyR9sJ^V&3G>ld7|$|`Qs0o zcd7CrePmJ2a_lrzyq&HpU$!HZVbQkIDC4~Ch2zLhKs8Q+)r9V$olnffND#^jkdp`? zSO-lJ$oW=_R2gs~lCpdl!3-a-4c;87P*C_tS-l9AL{W@I7v)@+r5%LM-OBpGU<}IO zXb7G0px>j-V0wkuRD21&iAcB97e&|{=-uCB|8bFpz}}^t6ku$bm|O%l1x!#d1@4gy z#PQnwC%d^u@~sT5~N1%;n6Xt=L~!SF!O>3vK{N z;xG~;<*gf#ou42Uqpv7q)|?#2ZBOb+NCjcfK-f#2rQXfl%ewqTrq~27 znNFn)n2~JL28E)B5D)_W(@@+|QHVf0|1=O|*;(<`-554wEPBw8AQTqPoXZ&0fq7DA z>05o>pwHaULZpBZjNvjeWJ9K#A3Edt=}NIQ<9Ai$A5sO{@Deda#I`^k+k^y%SW5O7 zTc8mf;y_od#mgm?7R(IHC+Zq&X{6tvmXGuUQDEE$NY2?rke$g>11+QmiiR1UA(~MV z8wmx99T;Q41o}kMS}DX$&C-TBWK~*ayII~<)>0xBq;`$pMKWG%)y)wrN4#N1%1;wwc&Ej&P>z;KD?3@dWc*W-oW5wQ5;1)dd`(;m_B?J-q8v^7FSa` z7>!txCsh+1nhOh^4;fYuj+wwe29$tSa%N}$3R(GiCLsb(S863#4jl!~pNSOP`{0hy zG)Yh5n@`-7j6H;m&5vF%MhI?VqBvYRTH$?#-5#j~VBS$+PMv#IPGkk8aKIvnd64Op zpa@!1%Rt{*afoi1=4+7KPGsh0hNgCYXLxQDvWTaLbslNzVRkl=WYJBiamlGYgw|XU zLxD?v>08bSm`W<(?e$$zZByjvC9e(8VJrm)w4+gEqy$;mVOl0}D5m@IMT8Dm2~sE= znk7r%kRh=fX>@0KdgzBz6hwmPVO^evGNgI}+2)i_*=!FB;SEl}lhoOb7_}7cpyW)- zCf_85S}BFY;R6ERN+MoFm0S`~B?WQ+ihyyRnortY_{~^zE|Q2rC!!SCE$ZTnKnisI zB6>2XvmHctQe{J8=ZKOiD;;CJ0Ue5ZDVOqwni30}w%=5+8e3N5n8=-KZpr(h;AK1t z?Zpp@REcxdWZ4nSu1Uo;vC)+LUTn=@Rq0^;vx0H*WDi+k`% zRZ$f{J>+GO)*Y{{1Yt82ffbf@Q8sCN`+D=U})#3E)BVjY6+C8 zRoarQrfRwlM^>sUD7C6VqN}{Gy=#d^MKY(rl(h5&n1X9n22Z@UYsn%9Xqs&L znOnK;Aj!^ZRCG;-RE)c6N=lr~NXkt~M$xX`$loLv;nV|=291-k54a=>H31$=NMMtJ zYra0-P&(&g_JtJki!N2-toc&MP>!JrB>?3ezGS8d>`$Z#qMoJy?AeBDACf29vQ{#> zY!iK7A2HLj5G1;+SADgimW9@m5~v6gs$Im<#*B}`)Q?j5L)|Utaw3PsLa31-EsZdo zN<7zd`GsNP2FoT3=9Pd1aBgVSi3EtPm@db>vTb?_%c_>{xRGuWg)Az)ZFxN-9XgL6 zU8jus1-!JBF1d)REd-SgPQeyxkkX?valoLOL;W6vp13Sj z_uvj=EL7!ki2|l${{*ejs_tNZNyTKuVocM(B&(g4VbnJNuIn6{j-pRwMqfQ0*16&s zMRtJr-l14*fCO*=DuR0ODPPwc4JsR>43 zxKRQ41K=u0$1&~eOwyEe(EN0*PCTzzRjPzyDMaW9T)jx%$#0iViwKN>&KW@pm_SUT zn+Z4Q3L8xe?-;w1ac-#V6;F#8=U5E8TgLDk6ZS*y+U)*}(2_C+t_{Ed6EJJc(*Y|9 zfl87-G(eL&>2NS^r8;g)1&KixCnO*?i+dbS6Lxs0g73oP;*x|#A z4#(&$GJ`=UPGRkpGGps*=MRZ;HRpvEXEJEigEez=H+%CpgL5cLYbuj-uc&YuyK!i? za;>bg5>qm=ioZWQ8B*rKj3Jx z?XTy|7yJoT7Ntb`bm9Q`j}Ut*YSJANUXlV7Kt4ch+U`UXLt6xI>yKn2M6eA9P*+xLCrcYf>l ze)D&KM?wXxtvx~Zfcpj-bM<4du~dj`H{YikmC#w~#V7vg9~baHD1?rjR!f6=eUd?xdHq`jqkVt1OSukc#`|LlvDYXTX~gZ`IT#VmUH=*TRDn*Kt~gJ zyEY>#e{^M!IXt6zkzMpIa#Bjq2D1OX%Rev*kp!`DnBl`UZiquO(qJ~PP{5MEz@WbX zBn-Nt6MCX6x}uZ8qC5HvxWS-DdZa^orAN94So#ZK`lV}nr(^o3H@XLq0U3yTse8Jn zyTPZYx(B#Hs;|1MV|oCLfdNzkXumiiW8+)-6 zJF+kPuphg#EBmu2JF`oBvrjv;Q#-PI06&EJ^8Orxqir)vw)%p5%8qhphBKrgbOZ1M!uwSKAOJs{I14NP{L9mZ zS3CT}Lp;c5_PNXICAy|kY&iRMIKdFio_9P#Xqw0y7Qi1orWyReyCTdxeQ`AW(@Q;b z>ig8wtXC1vY{=F>5M{wkEXwrxxv{`MjC^4sJ<1b20wlf5TYcN>gn>J}+efp_qpDMR zs!pr`#;fv90OkqcWzepcYeP6s!JE+okLVaY+Hc;{51HIMzM0Rw<4080LpD+Pyp7C6 zP85Jn;xrRWc(E3>^zu2)BhA=X0O>Plz$?DInf&TAujIe{xR)_yQ$Fl}YwhP@#IJ^I zZM|~*10Vo`X>9(_{5#YV3^zH=6aO{kQ2;-jzS$=}rl~#CH@)poz3k)97474`_2)F* z6Bh691Lsq?7}`cV3eQahKWXfPBdmv-;}F!|MVaj#*O%Z~fgcMP00gY|^aOx70E7WI z{|FW|co1Pig$o%rbodY=!2s<9CJYcDVn&S{Id=5;5oAb_BT1Gtc@kyHk`vmgblDK& zOPMo8UbIOQXHK0vd49C{vu9AD9zPWQu&+P?gg=3fjLGonLZC;LK8$K`K!L0U8sKa2 zk7>*R3RdU~FyQRP0YllgJ;=g9g{yAq7M#d$U|qdn5g71baqeEhg9#V^Hhfs92Ly;2 zH!jK4ab(Gp?S>6Z8M8wXMn5F24B72Plv|S~bqDq)L9_ulnB|Z*b8Lq!_(8ZV+hl+S z6>+D$NHOn0(Y=WmH-0?xn8nDM3kKaBdUTDJQ%lcm^aO|xmccr|EnH#Nty@tKmzWl< zg0a?RpXK+0y-(orIbQ7hcji#34Ml}iWU)nqE@}WT7h}8;#~g!FOh@FX zlX1Zyg&dMdA$vUXydaSTj60M*IN*cL_BrZ`V2$LuH+{N0w-ntk42i z%azw&ef<^KVEsgFQCR`y2#_cL`M?oKsVt9Dh(5SfTB-EmfK!S$Cw*syJr06F^w4Lfo5+-&ZHsZpL!$H|anuY~X}uA> z=mRQ9fH*fm7P!EX)$c-FxYSi&QuyIN0?>GxI$xgo=AAFQK(IpckF{ z?tMKoQCA%j8qR_Z+NXDfBJsn56L?O(|GozU=x0T6g)$KJG-Z!yL1+iK0ss!!p2zwp zkpetm0cG$V)b<5Gf@RNv8F^d;Px3qqcJ3_G<5~tyq(Ki_Zi69g9_KPx!s+pZgBSdo z_gL7%eo+PxxPw+>VhETzd5nS)1I=&RbhPNPC2w+4(oYUxyDLB|ECu^fLG*D23# zsw*djlhl;lMv(1vCrMl+T7eW`0Q>c%fI|u5An|7t4gdlG7f=fRZn#8=*y92V@Iw^! z*FdWX5|mT3RVceCMpK>=m8K-)RVqD*y;i=mjIcD}8dtU}GKoY1T+3yNb~%7t&Jmcw z3>HBWu}S{T4UtF;4DA}0#b8n7koiL)|B!TSkb4U~_4IM@*gut!gE zHsIwd04^4QKm`WSj~7LAYh?hyh)E291?1zhyUDB)KY#)%l$gZoD8N``+Q$OGc*Gb$ z!9oV0O&m+|JOx<5O0h&Ekh*lFYV+-qak51wH>9~yrW|+k=~U`U*}7S-r&9l|3__!7 zuF7A#)~2X>)v0!^t!J+4SacUXE zKMWv{#43QXE1pRJ{O|xo*cyQVpu%bH7rQ|o5C9~2z_2ec;s>Ch0ReD_HZKsB0^Iu6 z5BP(LSsI>NL)!rh&~-h6tkMTyi6T>^qJz$L*?&D5-3dWnx*vk&DHrw1(v5fJs9f)* z+LNgGc50Nj>&92I_su5CCBQjY=C%%CQ{>%cdC6bc%O z(DL1WXq(JL?RLq zZpWn`q-m5CuQZW3XMw16fI(dm)qhgJ$Kldye&B@#6)>bUy;{2k>^Q|e{1^v7fPs~o z=nA{r)TQL)7$>`sjt26RgZw(`@Z+M|Gn-(wdevMAoa*!wC$1K{M!G6%JbrAN}T+fBfms=m@VL0gecI{?V_0 zN2EXfg3cZ);Qt(f9SX3~{x9eZ(ElPK0fX)UgU$gZ@B#I&(Hsy0^bhDF0p?)t0wr(* z9Zdsa&e1}!1c9yseoh5}?&nHy1z+$4e~zj`PK~Tes{rb%@+_;M>YvPFH9V|DYEQya z5Yi+q0w%z(ZqBbLZRc3P=YEa=eXjuqU<#*jm(c5SWJV*5&M7`3DIhHB0ssL%p!6CK z@*EEh&rl5+Ee+Xl4JWS+;qVRPkPhe24&4wBX{-X^fOHHs3V;eQV4Sq%BK&{^NZ^P<4HF+AfW9Kg z+Kvjagn@JmCVomL%x3WTq!Y$~1tNe2aFOSDE*E<-(s=O~cTv)YQ5c)>7>n^3c`g~5 zu^5q28Kcn|r;!+`Q5dgL8mmzovC$f{ksG(s8^h5WeeveX(Hzh594(+6)e#-r(H-CM z9o^sr;PD)7PO#>&9Ocm*C*U5HP#^iRANkR*<`J+65((vTAO*_}CV(OPk*{800u1s5 z7V-obG9q&hBP;SEJ%AxO@*+P{BR!xZIg-*MvLydcpd>{SBUO?lTT&!blG0)_=UOu7 zQ1T^NGA2cm=46ryIdUhDkO_0{Cw)@qiZTL@@F;a|0+N#En6fFC@+hNHDyNbvtI{f^ z(kQdCDRpiPjIt>wZRfZ$0t9NTQjW|>&L;lJ&^#>B7>xyskmisuA9HRDe31nPF~X#f z3ind2)QaTz3!nlMtOk>yWP%Gr!zS2JF&EP-8nZMY>;oXc4<_UVe9G*NRjThlZ%vnyYK=Efib#=thY5;twLHa*}44$(JT z4a9b$Ui@GINHKpTF@Ab1lAgjJ5NOB1svaf&z!O0+01n^*FyH}bs-?~%?JmHW@-FWD zjA#z$b7qm~L@Eqx|E3nDLjiJ$su)hKKF%-It3CCh%&2PW=**X>>OIG6ywnRn@smE$ z%i#F)s@#*808~H0>!5tFtppT8=`2DYR6^-Y&Kwj&FH}PfbVc2v0NkNQ+rb^Wq5*c)N3&u_Q?Ew{z$SQu z6irhvP2V*2taMGu^i9hYO?7lhYmZ9FbWiP+PVJORchn#HbVt3SPx;hO0o5O-kV)HN zP!)Ah1+`2M|8-Jzlu!4RO8b;jDHT!wGzim_O6_z?6V+4wva8XOd zA7Is0>6BJcl?q2yRqM10U-eE)WG%mntf;W|-m(*zvC_(sByrIR2Qdn_bWY23q0;PS zJmhVzB8F~DCwvYKoG{Y5bz8eN(y-J@LF7hD10TLHAJVlwKW^&U)m=Z1UE}p#*L7ao zRbK6tUh#Ea?-gG2wO{wuU)wWa1@>MCHelg(06661zN=;8tX&hX9~yuH4Dk>xK<5_G zJZ)kHYR41N$#-lDUAQxnK)|X*Q5{(7B8o^8y{X4AfC{LkDZ&B}R0 z3gfs|y`t7?@3t(mmT%HaaSyJ{5_jYnsw{SijPR@nAx`2BZsaQ0Z+(zqd$8jw7pe}I zzEtjTMfaZ`Ez)Q%36)SNmGudKF8rua99oqx(M-F%Xd|*JDvABBHkf-X|k4tYRmC1x`=wLP{-A z{{RF2;E#X^vj*t{ij4}&;0iLp0RjlcE}{UW6YQ9x9{?efP+&M>_9vDN0Mz#>P^#^K ztfu@fdsXIVpKe50h-fGHC63mD=ay^5wwE+`dDII&k+#j|maR%ey)byr*lL6k&ddg` zYw*_Ku0=u9ESE+Yg|Q`tx|CY3gWm{hBsx0^K8x@&dg@es(k5i zO=N`60&uOT!>US#eULST_@D5T&f09w;0({E3XIo`jFI?>W2teW7|vXSEdI%=W)R`9 z*ao$$id}=NT%({6u5^2Fen_{gwyLXC4y^j22R&35?-B~b4^5M;bO1?gvch*Z{~}CR z)OcHzMN32VN|an@ObZR8Tly;!yf-2gAOZ})V#ilKb|X0Rh-HYA1TsK(Rpobxrf)K- z6Gw~$8~|ojc7KTo6$L1OQ{yKFz?HASr0|y&U6D>;5l0@lSKdx^C|H`Y!h)-LYY1!6;!|UMi7QVFhTDF;+B{zf-4x5>_yv&PjNx0#p zmNoKLiKnMQ)An+!IBePZmlVpq2%2FNH=gHqoZ)z!)hcj(b#O@zhNrGI5-y-Ky1b-# zdNLZ!`15bAmg4XkzRD|`54wu0$3!HKgq&E2((}=Ozw3nEOQ=5Zi_^J}_wf{NK!a0joc(Td(Kc}jV)tRydbhnLLg^jqQ zV(D?Lb20+B9ptUZGvBYiZF!2vo z%4SbW?{v*VX7=kWL_=B@!RgK(w))$Ih-Zg}l>T~PI4QtbM#&2MR;+o0;dyHN(#5|> zYFj+xyk=`1y2iCOZoNi_BQ9)vDYN~yFYmVD+!?YJ?wrL-Z&$m7ADhUzCgGUYpcD>m zzxLopd&>7Vo4>1S8&w#uyk^i`iiDZ#|Du5-!gyVj(#3?SnHog8 z$6HWMs(lAJ70G`SNN(CUWCLKrsUQPXD~8thEGkif38DcI!DV&!GX}tK5XROssoE;+ zeThtn791*6T$xBbr+nkY(NoxUB%ywzGFLO!T z#rA7XSlSKlYvBC1-%PY++uQq7ZMnF$t=PBItUs}wu^n5CuUOiVIEqOay%FxrP}tk2 z+t3>{gk?B`B6p1=|uzB7J8JaLkJQiwfJ z|Mnx&&3+@6eVfU4$8EQRmj}JPo!{wGp>iC?OZciTduunCvcuMj`*w5Vvu>rm$f3M1 zmAq^pTFLpgo(EUW(b?N8I)p3^(X&=+&pT~3sNuMny9FJF9}b6E18`Sc$Rj%S&r8qO z_Me!XaYs69;r8Q3+P@H2&)Iq7J;?E^_@D>(yX0)AxlTw@E(>EhBi7y{zNRExG%H%v zlD7iWo4R^up62-ns&T%^zWOB!pnh=-19lyeK!#{mzz;HM*lx`MUTgtMF#tYb$OfSx zIG_k{Y{V|WA7}>y+>R6NA0QTF2taVa6#y3s{9DMdp~Hv&2qNG&1|q`%7Bgzx|H!eU z$B!UGa>OXIq{)*O394MlvZc$HFk{M`NwX$PlsHk6%*nH-&!0ep3UxT3sL`WHlPX=x z^r$`#NfjKG+7ziisvM|RHL9;@R|QX{X6?#V>wu>N`VGiBfU4TFVyT9;%AqR^eq$%UHQHA$Ww?kxOZKWjvR}!N z2Lvb$+Hu>&`YKD69lPjTwQnm|mb}7m0LoeM#{R6^_FA>OuS%|*b%k2gzbmV5D><)j zm_tF*jJ<+;?AWD?Pap7eWp;-uK@KQTG<$*f@YlbW?+?Cx`!@6I&!6%m|3L*1u)q^R z;_&0fc^?gs!2^gqz#xN&0D|C10kmVlfhGm;0Tmda5F0`60O9}u4lv}-0}DJrl7A_M=oA0~26&*+P7@}nBu+>!>128PMJeT!Ej?-F zl~`t}<#}6m>19=BS#?xhU_!-IUtgKY8B}I!C6;Y!rUg})Wc@`LUu3p5r(#B778#sZ zt<_kam`Sx+TYplez;A1rMP_wqRrja@Xikb}nvF?^CRnOM#VJ+G846VZtfti#vBpJ5|K)xs-NaHqpAu9o zr;&jbEpWv}*APJr70_fv8_9IuObdPc4|z+@cOQG~*7Kgb>ns2gXmnHi zW^84NHTP6;)pe)$3f;l&6>TUdK4nCwP6t?b-7QMl3M#2a|2cU5l-5u`_1gPzx#``D z-MIXLydJ#v)%WhY@G0(k>7Wmg?|>Cv=rX_(6?u|A2VGEvYK;5w2L=yZ1jquue^h`2 z3!pGEL=aOUakE3}p3(ESXRrPCS-$QP_uz+rk@(rkZ2p&RswuRfYa(j&*-P09Yx{_r z%I9cCjfNSf?B{w|QiFO7)BI#8sEtWZi1HJyzS0%4)WuAu$(mipQ!Uj%a94R+RidV2 zs#^7mETYO&ZuAtW+02hN-KkcfGWRH^8R}2cnI8vv#lcz?D_h>nVN4n~9*jr_htnyD z5GhBvMIk0Ky1AZtAaWDYy#zt?0+_m{Sb+M-M?Uw_|DhIL(vR7htO2$o-;($h#*zf! z035mi3a-FL6bz3F5_%X4h=;w4_+bGKm|jPE)FG2iq$1Z7qaX)K$S2vPkcdpABF}Zc zMmA-ANSg}Gghm#vyoD^Au}fYe85&1nB`#6HS>BdLt@rswQ9Co1u^!kLya1*(5+vGT z*m6m?(I#()QqC%sHkec5Z#1<*%~)P_D;Q2DhO0@8S{OqWz9myCVx!C!rG%UyT9a76 zIn-8)!#E+v=$fk0Rkq%?w-wx|J?N3i1N>l4<6VFX``F_H>5}IkK#+h6784SY1p1IV{_~?C4XH>+ zN>Z0O@}z63Q%aMProLKtOJxLN)|69y!cAC;!`ILk}t&UMvm6sAeaDv*z zQz--0D-v$XgS*j{rGEJ*NqNeGmAn+Nq++U1flz*c`xUSJR6(!m$(1VP%{}!~!RP2| zZwMTxZmtrYMa4*5K$I&uiAA{}vT9KeLrOZox6UQDbDtRsT}WP+r2@EXJy*0(WbL`u zdFdlx`bmkv%2?9Gwu2%#;lKf|fPw}gq6|gQ0s3>n~nb`pSyftbu59t&h= zcgz3V-um{pz~u-^hwD<9?d5%r+zeN#rkAsj#(hph$}A&UGXbteQ)E-_o~XvvQ=V-z z!i45p;KCKZ2nH|s!&)yTs2N1zPglb1S}hG@ox`beGHVkI;@VqTnh=LLRb5*)hl3o# z=~6dfk?Z8Z!>S)Pj&VWMtAs7M8UrgUhDTi@3 z5D9&rj7a_pE`>HKQmPj(5!C9s`WHK%Qhv?H84nGKH-1L$3T9&B|)zwXX z=C9Lv)vKWO*5dRdI-lXQH6d5HP8Bgaek4SrGRrWn1Fvu!l9R z&cq^?dOExk73Dcud)~FqT*U0j2IhfH^H?Ie^)aUZh+|pXT6-S7wXS!qq#y?wrpNtK zD$nK_p#ckNKx6M@p7zMeIJaoel_^tK=_bNJxsu7Aw$@&_ZJ?p|Pht5?Cy!Z7>QI+8 z#IE6O%!cZgh^96YRsd#_QyO?%b2y4Fbb{gA;A>(lJe^t%qVX1y%sIz70G9A|!a484 zRk}S**)UBwt=JELS|pkF$vvNh#S^1Ai{I?;qH@@;iOcxLyGD6`c+GN><+}gnFpoJ( ze0@G5PkE+9W{OR#5>upLg->=APpTDSir?PwV6sGL_AJqghp`pvk6OtyEYQ z6mwf)^&3vLs>_~grU=`;l1Iyxu6UO44Fmov!|CQ)eDx~j>OwfnUGplL%NDC5=fh~} zeL8u+rh?N__Ly-Cc7yG-#YK#94xa=*N?dEYu-I{WR{lMe6XJC?f9jWn{5ss>7Yiz& zxRTSWvjiC!AazX;dl+E#Oq99oXHWaugOBEp4EFnsPTk(q#goW>CE0@iR!kB*t7MDi zp|BD_w-v23uX&&(W5__QJa98^GY)CY%gI>_~@xQMUQ4lLe4g5kF;Vgzd(C2P3@ zdmSZ%pcsmxIEoWPgTyt2N1;tUr&7TNZP2z{&ek*CW){h|Q$M2@W)VTjr%JDAHjt4t zjS?2`b$v;(9Cc=1)dV<^fo}qqXLE;Odlxy&k&JnUZ~Qh_eU&-z)_&B18-1ZygtHuX zkzj*XI0*+l$wW8=xHqNY9P;KH(qxTv_Hd4Lfct??_E;s@<9O>rAtt9pvG)*6;SLSp z4wa`cmq@ph(6drS(EtNLkOrAz;m0ux$$<`u zd78&X0YF5Pxf1;#4wYzG0e}Yl00AKhVp~E+p2&#-;CiFEEi0Lts=1oC){^Igi(`^C zjbwCL#TI}87`3!XKEoNtM@xahi+loY^pzOjh+KqMS=HEu$_YbPlN_=k9IRoEQ~#Mw z)>4H*6likjSM6wT?buguiEoXgV3Wy=)Oldn>1V28mJu<4lrew}_HTvrjui$ip;v%G zmXGDvPwi5IBX9V$QbViAmI3*M5 zksvv8BbRrd`JofhY9BhHD=`riaFVQ+%7935XeIV@(2>4F7Re8FmyH zrihe9cA&nyRY0s)O{R!lrv(@`?u(oUlqWI(Hj^A)C{dtGpT*(=>jx5uI>_7}7}{ z?HM)zhMVDJo_Ny_wSjMAnN8$3XbU!#2FHwqlW)Z#fKq5p@I_$;CT}pLH`^*1-PDbD zH=Y1jVT5*JbcuUx%2sZwVQ}hs<1wd(Wgm7rfquxRPL!8s;SN~p*C?>KuW4o5~3hGqLIK4C;zHjtSOSLy0R?$ zFRpqfS_VM2YIAYYb2p=hLPsV6RAh@GKgOs-q{3IZ(iL85EPFOAduJ?685!ZYg+zN* zZpV(-nY3YgXx8Lbz^a{owrI;3Er7?gM0;n5Mqy$Zhqytu67~;M`e)U+m2k(j-U)as zfwMPgWbvAxNP%H{8mHNUuc6bIs>Po&frx}_r%v>Tha`|)XS7FwKTDCQ3F&|e+n|8! zs00E5q&KmXN)wX0nf!nTmO76PTN57psUuqxCV09bX{spNvab8Ou$v?=TNO2{6v1{} z>yt{nS6;GNAFvPm;0OtPxPRC=0?H8hQ%zJ#L59eIA<0jMDo^;( z0Qm48_|OjbL5ZXnu^d|y5_+kbI-!(9p%O|t;W3wOYPuw<5fj0JEN7yv8@oRI!$ABp zv#X=C+9kXvlQKk@S`nlYmT+8|98HTEY^!&Dwxn81mQi(oVmPI<0kv4=wXFhZaaTlw zXLebLmSc#XSV<6Q`gU9kH+9%{o+6HMSH&2SolzwxP=VT{mt#a&NNM*=gDRE3qzkCO z>%afow|MF~_OTE4vB)v8zz!_Q2^>-W*nkpT6N7nQN#S4^9H<|spnCceAN(%u057P+ z9wA%+lG>?_n!+(m5Of9s{Ll%T`z$aap&WUaHmnk(TM&~*S83s;(o4*7=yxw+s;V23 zC3~vn62#E|9L>^Ph(pX2DpjMl$}`24x32T7&iS?D#2etuLD1WbwK0xWWh~ukp6A?! z(>oo_OO9X|yvTSgN(r{lRA}5`P3sHJ&AS`Hd!F_ztyyd%NE*;KPW2j*;On=H3?Gj0$o(1Cbd_g}cG_t{`@^J6 zTR@#+D;o4EM9qbnfrZ85q;r*LNZPDd+1bf>H`i2$P#RW#wP#@nRkMw}#Ic3{@P7Z5 zP1rKqMU;kxH@h)~(RkZtb7ExpNXU+sM3CGq@ZrdgY{14X6O&xH4ZIKRbXXTx-4k3c zP6bZ$I6g9n=5S)SP(heca-ip~0bQa$bU;q|i0d8R5_d0C-xfgM2!7xOu;2{d;0w+V1_0p_9^n)|;Sk>72i_7F?%*8W;S}!SARgi(KH?<* zUg9Qx;tXy9&g?mVU6Lx=*Dda{fvpt9b}A=BjR070ZnHJu#x)wfXz=wM=ZqZU^_ISI zrN*JY%pt9MHmunzIK|tj`v$%7i5SyKjs~U|Q~7@4TF{13SNPV?@$4K|yNqIt<<_XK z=Qo*c_Ycwt9_i-?R@+UhArU>!{A^td8orZUDM2>n{M{36AP5!2-s9?8uJnXutv#knGO?{_M~m z?b1H&)L!k@e(j?NkTJgPiv;7`o+KV!%_oDVVtKWx;x!f|jp7uR=^nG{l;g^E983@tAT}RxP|RWhg@oC&nchP zQh##7y>hyi`oPEQ7igmKl*VM|v+E`^rROujn;XpmEtH9Y! zxDN=)+fB%bw-95=L&)^Woowkzfnpn+5(h+Xb)iBWx5OAT(97Vpx;@`gZY(IO2OkDHQt;(oZI=-AT722%-LA>JD8(+g3 z%LFM9ZO`_W+gV;W-&;)s$hD4BH}JfEX1i+mA>`wTIZTV$&g)fr zV{lOZ=Gn4f;kY(B-rCX#PUmXA#rhp&es-}dUi{y9b^U9v1nu|EpWDq;Jh`?FT+eF_yI6gZJ#0DS}la4^V0A4QfVKMs8VsNkZ(l?NN1EGbfA zN{=3SCR|w3Bg&KgUXqMx5u?J1E{k4V;n8Nls3Zeuyy(*=$(s-tRy~^Xq}6|08%E5T zGU8Z@XwiNQ5I1f?x^?Z|#hX{}-nwPw{skOZ?m(3eIS?#paRA4Ji3=+D7df)vxd+V} zl$_ZwWyu9RfA*|dGG)>MJ`^~edfz@r1wEGzAX;_-*nvgY#w}O&&4#^w*R@Gk;N9VS z3k>+2Tlw<8vMCp+2fdxU0`2fw2TdC;;1zc zoCzY{%o>Tqs7A6;rltmR(IORv1X7?HIlO2`wt#Gk#+eW*X$6s5v&<#iQmD-m7o%~$!9;{KQb~!^u&x_Tvh*;! zRMM2L%xde+Ec{A~EW6V}bB?)vP}}FUR*x%)I@Ipl?=|Lb(~vXKgd@(OUU>~J)JTs@ zE>~l9`&2N`qKnSDc0T(cf=*})484BhDlx8WOWXDy+HFZ&&k+qBt=2rb$UQb)`!IAc zE_UCAH(q(?rMLfHd+)_JUjr}HH@|c*vZ}jp*DCI=mFkj-rlQ6wvB-!P8>l2k3o{C0 zrGg}Aq{7^KDdCA0%PJ|9qDt&Zj~i0#sx(vlC@YA6stM$nLmHE(C!r)6BR5avimEgj zUFju`)&j~ahhkm`Vx2)gC}cSg4T(dbG`>n_rKCzUX*p|-DK-w?LQ3mCyP_(hJ^`CF zZM8+*j6S;#G+ z)kRh_UxzC=*n)xU6?0#WL(W#p*PTpR=uR)7lM@0jK;QtrJ=eY0XZN<;@vapwu67`Z zBzAL^i;w?1ZD%LG+u>`6eRSpf#g%#IpNBqr>8Gc@`U?5&H&+k0AAYVPEoF$xG8g@l zqD5sw2x2KAUU4ag%$%tq9$$1=eKg~rvPey}>~j1TZM+IgW>h8@k?2k3YgtidRKS*? zY=J@vNlJAr8PD_$YuQ7E&Ah6r_o_(v0!|vn$T&#dDn7TpNYO7h;J{jms%rU?L%{0?4WW z*UA5m?7Bx4^N@#-#*<6(*3+Iu#x6b}BF%TUl{@+%uXx?Nqp|SiF6(vDlb-}-C`CC+ zQl^W0XtRqF1b%4E?XK-a7Ggjjt6Qp zaSLx!rZBZ6a7kbL$j<;JLld@=mU9|Ojgl9~9~LxiZDY<|KE@NeB=KOfd<=>ndPcN8 zZBTv_OlMe!QRN(tRj0wr;#y}F$MHulfCv#8)0jz4W(JPDX=COt_nXgsE?9c0BTN64 z)3*8a@mcl2&L2MmM)2e&J%q$oT*T3d-o5Uqw1X=0%oNqBPLg;eJte?O6dt&AbwgjR zT3EdrR-NwjT`eVLTGhH%wzk!+X*JdZC zC@YAc%8E$XDeZ|-I$D%pQ4~R;yy;RgJm8&6mY2V%&M@M8`ZE1HF*K!0jvTrmF}l9bFt}My^H@{HL|gD zysBwke?!ptiYTXLwa@7q07yU%^>|Pv=_M)C4i;44F9WrmqbMmm2+OCC7Y-hXtTh$@ z(+5ImeXI1a7h)2Z*u*D}kSX6wSN(2xn#ep7OlmY7#u&yWp@ND~qB~3hNg0yDNT#R z*H`D)zoF|ebC@L^Fb3|zMKzbe9Mu2_IM|U%Ess%!hg8Z8w8GpPBql4mVp8Y&ZwCS~ zinYCMZg<<;`HJg%tKDK-zf>VCo6=={)msnt2)bWMl+?`kJ(IktA#`?SGUcbD4qEmw z6G0}2e1w+5)=Y)P4WL&Hi$jMsNk|1;ikc9@GaKFPf_}Oq+$RsRcBAO91=236lGa*MW zjp;CEdeg9J)vFL^xV?bU9s%Q7d-bz9RA0t2%~@%#os(*^&k_H>uXeH5^Bc$iTDvv{ z7;sQ~-RlQOJ21QJ-5~`dRjOPoTYV>~-vgr*vXb>9;?etoYZq+UeJ5MXnQOJ(KHsg6 z_}iDqeC9R3c}YWl#e)VS*=KzyVQmB@WmZ~~xmnGJJ&a~ver8WF#SzxuErw4T2}2JRYvRnags+vAjkk{QJiyVug<~$MEIWscHFhqhoxE|& zS9X?7tAPI~Gt)Dku@cS*!h6W3PcVydK7UE+JTF6}y($TlFvJiRnFq5GN$V#ldb$hw zj8$QcD`GYBsv=f7fG)y|*H{at5VTUO6<84t_zDiqnX5+y?6BP^H?*s@Hwr!elBp1^ zi?*9Q!N7;b0jlZ<91U|Sxga(O2#m)Y9UKIVt@0i3ASC!eBt-g)k9aoRh`be~mD1t7 zC1n4?CUinj3ODUSr4o#_ms-16(<_H?r90sz5-Gmf0=OK4lMBKtc)GX}Y7sj79?$u> z6Irdwin6;ZAt-qXE)@ERJGtji6}d{McJx8*({q_6ha{%N^3J|vN`$_#YnL~yl}S>p$mctnp3=#R5X$u={&yK z3hm;q0@SE5vZ8$`K+341sQWZJ@{CL5GlfY){lGxrc$5v?!0qwC58RdQx~VGcuQ*D( zyjVuiu_Gjmi#|df)&Y!dA-1N1uw+9QNcx>fy1_)6s;R<_+%ZSMI6QKCogXPeC;I<9 zZCtu1)W?0~$9`-O75h5Sd#Muy7Fol>Hlm8XVVNeYiF0xjWV)sknWt&`iJ>{BU3!T} zldi!aRiIgYAl=nx37!5;iZs`{OQ5uWp?K<}`fzl;AWyX%(p zm@wOsE6=00lyW zfH(QLb9*C@NUVlK5fBo%?-7cc0l5y!rPwk!8VL(4VIPER3+wElj4;kYBoo&P$&s5l zYnq=y#J4emGtN?&IH3|DA)hf}2;qXc3WCpvkhvBLMX3bPy(mRfe8szv!{>~yg))+t zNesVCDWt2dvTUgXG#pe>OWbHc#HpfTd5tTnp%jS|t-H%(1R}k>#?hM#qeH@H480L- z(cJV66BNCeGR3XrM#j`e$n=Z?`;5wD$MDe1bKDjj^cL5t7Tgh}2Q&ZDav{gYtGlZj zs3j$hBeV}vqtZKi%`DZ@EqyC>kxhXtNP`$zm_{s-l}Jc{>PSR!J@gTZfNGj(V!x*R$sOXM0A{k(;c1 zn@&-s&Ktm^w8ihzv}P;~!_lG*m6Y&;v!QgkkXQ@$y0#1C3lBqE<2iiIKwp>u}@kQID{4luHg$1E1u0Zm{N9toR4axwoNNdh(<+&j@+M{_}{ zr0AZf1eIz<%}L8HBm6urMc9N@*ylk_D*aOUqC&H?&23e?0BxMPW5S&v2D6Vq0F+&MD7%Q2o&df)DffTfbDcnTSfYe4; z%efZ4O&X0+UI98J@)1-#MY2`Xg0KhG$;MneouCS&q5}UsK{C?BJD%>ymia(ZcJxfC za?*3WD$`wEB-I{qlU>98JS#0&5nb5a)!p5-kc67WD+SY;64Qc2%)L?Ei(Sr=?IrNj zF_W5`oXbFKjr}jIF z>&?lXcnb$|KH*Ktt{_x>dMBsw2%a$-CYc&TeP((Uu9d3vZAz%#lVO+zrB= z(Q+fmnktU)vC3M#k>(Y;K|PpuhyVm2S9S$31)%@TxM;y8_O){PmS-zn(tzC^gx6ma zO>KE9;;~mKo!!&)QrNKFuA5!U52pSv)#wgi$MOT;hG>iEU$xg{O-9*d>Cp zk%J%}0TTgow>~C1M|7-FEl|RmiZZ1v-O^8a>MfWQEG|hOO5CiCRpc=#lruCl>RpsI zspAb?LoF;nMcxyg#UVy9t=DQdl9*&IAz%Y$Rd=8ZSoWd@K;TK)ks*mFT+S4GBO;-D zlL}5oPwWi|<(JQ>P)CSc!AK3pK^$0tKwvBYyoxP}gW55I9~Dj`7H(l_C0-i79NL{c z5VT=A8caAwG_s{bF2t1h;lkql9t2)@!eMi!;o+TFi=AUyni&bAC0ueS z?K>y(>)+E0Gp^XBl)S#5&=I2OFT~ir?h{MZJ1jp7vZTq8n3UhM_?e+uAlsWUJ?*_f z(cfHxKb=-4v1p-?ncB0c)2pbtGKmqWRpTjiWvP_qt}ciMpbJ{2%1H?y1%)nz0l@up zU?b~FVD8m^BIa8}OA7^N%9vZJ>!AU&2RK`=dAt@6A)lNmsjbsy0kz-(d|`0@OE#)a zsN_H@9FCSw=R7qGh%uf`~jJ5w@@_#QK;r$srYb6UKti)q;}o;ki1bnULc` z`FuFBpx=~}8v69kN{k>fK_7n$+9sjztw35TX$qw5AEX7C6B0E#Q*5Z*YQB(VFN);> zFxP^p2Y)D62?uKm|LVJVRkul-R>X@V>fx>A+JiZwt2C0j9!n`q+qnL&3T2#R1`JmL z;lhR#x^pM7YN4qmJmP*<{Ps+Q6rDxa89Fxbg{-IC`=|>3@V{js;O};5qK6O z{o$W27oZVIYhvlJ9etM0)&D_&Z8i{riy`$)gx;{#1vWFc$2h;;D@}6W4ersn^E@Xp zidHqr+aAa+UgPFgX?*M&-d;JjSf@}QMu_lzs?|&GZs0o{X3Ypw? z@CTppSQ+-PHg-sf@VmfK-Bz2qAzLCkYyL~nStSg)DU7(ekkT;T7AL?9HA^a@YZa{x zSJ9&L`k~hVY|5Cw$^KfH8_?jK>Hr4_6hw{r$?Vjlqdie zhy?ZVGMd&u^~6=XDF4@GS0i4Fgd(7T=xCiMo)2?$RxMU&->L1>4qeZ^Vi%t_iw0-l ztzCj8`O4ArmUsEds2<;ylPP`C+Vs}5n^q(bOoQ}hZFO$>84em*qKtSS73$dtVi|V> z+Wom>KK87lK`cj1&=QIz88H+mX_NU5!%kM3sd-a7rkbM36M3TEIB}T43dEUZ86laH zbj#TDBrc%G&zK;gJ}J2+DQ;np@V(#gzVC1jhZKLv@v$cBklmEVV9;mJ3$9gAOi@#D zuc)-8w6&GATRas7?BKs>=BZnaf_luU{S!WU+M7_3roiS;8S-dFC669cC*R9^N8D<} z&2cTF-k8n}y8n9dEE09r2Q2qFKMtRj`4~+BxRz=BN9S<3V<`_{oormZ!E5v6ag>5) z^LkCv_pt548=ZGI`L6?Z4|)0UZ+YLI7pk=PX}d4N-1m+q^k2EcL>F`%%Lz6$>TCib z^Tjjfv!z9w$cO+41snnlAXre}iU9>V6kIqUVLuM~9tsSgPr=0j{U9<-Xi?zAgbP<7 zoESi2!xb1kQk>XNBSV80VG?{vu%^a?DhaZL+3;pa1wSP!VCm6hN{1(t?)-^R=gNZ$ zZvI?oQDxPe1NEH@;PvZQg<{8&Eo=5H+O%jJC}gWvtUI`J6~-M<4=p~hdiR2TOBSS9 zj0!;tX8%}Lpu&g~#|l+=5#<1h|0eSdOZ9Tr%mF5Irp%XffzbOxhwl8ZGV0ME_BqRQzRXp~=iWQ0!$`U@{?Qk0uknUeEKBNiQA8rBQrdvN? zC8!r}1%Bn=f)NI=pjQP(SfFhQQuyG1H+kq)K_7x>8HpkOXJT-}skkDGExPz3j4{eM zBaJoMcq5KEnzosaJ^J_~kSg*Rq>CMT_*`XWH7ApjV==j;lVmjqo>-Vs$z+sNUa1_G z1pgJK&p<*UL=p%6V8p?f1F@G7nL<&8kw9i5q^6fP4N#JsRF>J$nJ>xd(U%Do)g78o z;Y8=4UUKza0BK^z(+V#E`jAIx+SzAhUb1x2qKXo!c@Cvw2=VEcZD%qEnRB>5lmsM(+tF$5SH5^2Q?v~S_g#Y?U zZhZl>s%BzqCWe4|`n<#^n2QcdDtKwW)T)zes>`0XX|^>$aUugZpn!S}RN;h?dPQM| z92yAq)x0I>V22xih~mhn@dY-C$cr7=pBbow?MmQZ}%|1SG0NL#0TiC%CuR=4x1U+pfI~j=M zI2Sci@kTl?s@V59)|bruZCZqD;g&?CG?9_4c8EC>?-0|F`CO$slZ%~TW|E#rgye$1 zF$@niHxcV~L?|n9B1#0-!=PxVEXA53Ujo#Q05$C^Ub`At3M91&O|6W4DWPq`_^^#^ z?QH%_k#N%HMvRRuNnxuSjSh&%J@T=Se*gRUU)y{un>`s6rAGx z2E)Z+X-pTX)}UPVGcU1cQ80s(pm2q$Zn30HU(%H5c-D|h-U(NfDoMn^)G+vwN@wl5 z{Lhy_&>Q zhAT{=_wF*Rar%Rt;!BIO%%Uu^tka6dqRU(cfRDXF=X(K*%+1I(u6sI0EZqu>w-^E@ zlx0gO{WGLid~+28b(5i&jMp7W)0)yCFkkyP00ck4PjDiQfevkyO$^iz+bB$=hT@7V z=>(h;a@0u}i(zs`#K?{X?on8zq5p7}*%G15%AaN;8ho$nI}B$(ya1XaV~d3-*(lR&HU~3NG zugF+1gZA{j1p*=DyrP?lwJ0>!1&`TaO4CUq7H~i5YRGa56WLN}nA`b? z5_LFCp$u`jEiMyE1{j|gtIj;a1IbclCn)=UgiYcq4_WeZ(}FS0X=WuOlr@B!H11`L zVYRYd!E&QImaRj8{W6%tEaowj`JirSo5O&7N?JJ@RxOMeEb>&OUfqmd!YK&h=yZ7t?=*yezlL!8i2TaQs0wyy)_^$*hM ztAPcUj=uDDfYrRk+G-)qLH=jTt~^X?>FSiaaHBjEwrJWalqq9jSkr_Du8=4ucxPTx zD1#I#A+k25=~C(_1V8`+?>kdtIw$d$Njxe~dCHzm+014R%5^2<*3Z=Ra)G&5A?>*7 zSItP)CtCz+CDbs6^zy|HwiS*&dShXaq}MVtz3EPWI@AH&A-UF7+%6m(WhH4PM$Y-M zgVXxg;Y{bmO8*w7%DI)Bo~O#6x{jdj@yTgD%iowN2h&(F&qvskD*EhBOR1wsZfSec zsdO%N%sdKWZ0kEdyCjyugMQHcjGlCqKZ5 zM}EFV+EL##TO`+xR)c1Udi)+|5wa1vy%kAO&R&00ek|lqgLBFd1q1Tp4}Q z7gdHBMgI+yDTunQ4LxyK)YZt+ji3phAPVNy)bUL{g~gg>42iK{*S+9j)mbDV&Sd2o z*EL)NzR;d4Kw+pDMJZ`oN<>P= zc~Zg%&7Kw7C1Fx6DNLXZkQd4fW>kw5vQk3v37a^Kpv6k>gvz;q6X^vT`J~=hwA1VD z%IfigFT?bu-el(u1unADtD1fOO6g!R~p!>G%rVVajn-SX{= zXfz*m@QZ5f110`TD#AuP)I$MyP|ppJzo=pi1 zZvT_}`3H4y2P?sZ$l(unC_s920?JVv&_J5MSe3UiQe()P#AS}6gbV!$3egzEQw0W@ zKmY~U+rb%0&TSxr@LXDDpcjFW)Yt~yxDkq&NDAsm3GSmm{^MWe4am^d*Bpn5Vcne- z&JJ>&BK6?<$yo#1AlXf%+4&B4bOi^*QV_+MqnQ#)M1_&_aQqz=_ZkS)WQePb|KN2&#yO`J#1= zjh>B|B5hrrX~r?~;$#RDO*&#kM8rxoKqq*dRP>M$F;Cxp4&Ch;=3UTKdDfu;5NBc5 zV#owhe9L^S00cz9;^hhMxfDBc5oweV*I=O4NR4uCSO*&29(7qi0;JtYCv{e*+_WH? zX{Ob=P-9__oI%N*E#yItCwPu0Ln}0jj4t=tcRh>*XDu5iKg|*qH#)~ozx7>WZJ4E z8ctA!ES{L?Sg#RAW#Y zBUa*CA{4st8=Z_{EuxFW(WTKzl)K%Rf5h7R|HaAStqHMs;U1z zCw792iG*N2LZqH$3D|X~*ewouzFC`=r<>twFs>(?a!C}m-BE~6|0NHBvV`-vL`S&C z>3|jhw&rF%O1I!-N&J{n{F$Lh1#u}Q8X7C5JQ21YUggl1xOU$bM%d;Yk5!bLkyV9J zWX~PKMmnWaUC16s)kOnfSF;)0uI-w#!G*l06TNDsuiR@}$lg^7TbfD-nzj$N-H=(r z=EKfc{Y->+cne5aqM9yGZkz=8(FQjm1Vq*4x{=0Rl45GKM!faqDrQSddDw0!)Ko;w z2YHCB9A5zWhV_)`)2vj7p^#)wCUUT;dM;%86=SZ#X`BXF!~r1R=@vsA6%haK2}iVt z{k^T(&v3O5i#D$ICXttq_T2Pu< zSdbGnB8Ixxke**wm?1T7X{1mORWK!Ox%*EXd$g!@4!kj<_ z+Cx+!TB0Q^IYv)7RT3&k?F?EgbwWBCM8FVLHiF8cz08j?;@b|089|3~DlHz#Y2OA( z1s5?A-|ZWzDjhjoVCg_Cho4zAgvB9*wrBHq#aV|&ho^J{$a*) z<=CE4L=#DcW@V&FJWuYlP8=RuQtHlg(WD3=A6Uv=MkZxKNssIL*11yNYEf7(88TG( zPBt;Jp}Z?uC|d*c>-F|wT2yjW_6o8XqOxdZ`JHGc@0zkq9h(0>qFSWodjW?0kR^N} zhCn@p{BE8AaSj_XW-dL=Tq>W)j_iQtWpu1YYoKg>)Yr6y#{Wi4Z(vv>8&FnIj`QG( zNf;v$E3oS=Fwh3AMDnU-dY!F0F$M3fQw+dSK}PeOE2s35N@&Lmmrh#VW>D21-B}fO zjO{@Xs^ML5Bt!sEnvQ3cN9|;e+RjAq91?Ue&D>tuhAhZVO!M2!Rdgn?-5}#bL-dvv zF?Is3)gkTGnb_Ec=g{8jK^`u8Lh<5`9T#UY;c~Q_KBS-R(lR=cz}*_7{mkgdTAK(B zRRu~hWy(t$AIfAz4QI|KeMfCk58=sTp@N)z9_FGM6BGaDNr;MCZt#qzqwdnbtjx;En(uI$ zV|?(UpzseKDz(83qb(z=^k#El3Zyl)UTX|bkt1twRCIJv7=7xe{s)m{*+f%xi)5@2 zBanC>opt(0(lv2Yx}afg9S(*x)>X)ovPPcmyclj_j`5sSa5*0F@Uc+< zvQ!LL^KhbL2Cr3mrC1vWvzVTWR*YMm55Zt%A+j|*{Te8rH7PIbBQ_Xd=Fhk25E3z% z4)GLFm!)K&23+n&ZM-E=5GJ0`M&`NikdQ21IyQ7f5M;B4?b&ZC9=XN1Y$oDtEPBu^ zXIOzhS0Go|Wl-4%LZ&ZT^EIzfN4K^~-{60@sYZWusG5d8-Q7}nux6<;%ruG;;SZ1D ztYL60p8)rOM^;AG^F!pbYsMzn4%y|9V{ZR)M|a>J{do?W&oy5NNJ7)&9KD~7*im?& zw~)}&iOtBY(|QX2Efcfgss?S7^q@*}^K0*SBgwa)r)LDKUx~?T(LSyeryrUeC^d#o z&Bg~Wty(ZWAzBV7FZFg#Jcg~X$<5SKn+#L5Vop^^?L1?gtu!G|s0l7lvgK@1$N;pA5Q!LoX=ax zUc?W1Y6ldyZCj-_tTQ^}nOkZ3G6?K?(ErlLsQgb3V+5BASiLZiFP9?pO;rA*VuSgp zU#i8uzT$)yuvJ9NNb#Fa^_jhi$ z9&B4@7qfXhksk(qxsO?_?EC+G%f#{?(R>cZ{uw|h2%v9801o|DBosJcU;u^#`f(Uo z@L_;|7cu?|u#uz1iv>Uy*jFH9N0TS_IokK3z=wSU25`^^Q=bBvFky1QITL43pFe$q zblA^eQGF8^PE_cy=n8=npEiWJLSMg%H4z?Vc=Tk)j9)(%06IYIShHuxc7^HiAx^e$ z-L93}wxV3PY}>-EYnNeHzkdOr1p7~?L4q)?8XTYiC*uJtU^0drk>O3q10gPkNmX;i zni6L+edrkRPHQq`fsI@_R%E(7 zaM!^aPxkfroO*TZ*Rc|3*S?*4bLrf(PX|Aqe0lTd(f^a*sGRxw<#nT@6+JiZ-ul4j z;-$M^e}DeJ_p{Hx0e$k1zP<3m@2>m~94)b!L{oq&hj5b(GRSa~FtgwW(CQ!w599Q7! ztS~045;C%_3o}b7F%UHr^04;&LJKB$U}^x&0U8MNtTAya)61UF^a-D_)GRAaID7I3 zCot6eT+6&@7tBIhs)i6xRSN^~QaTxv;9Nn?tvC;!qWb+jj;DpEki6-L!6RAGJHsz|0J z1TNSBCIpSff&jBlPMD_E)mpU{%!@B=v!zQv1Q~2_L08EQ3@p7e(<;XfK~!-?o7l2y z(j27}PQ%aiQ*lC?aBC3)3fRqz$(<;~$*RbdgtW!O=yeEE;b1&ULMYi~vD5$y0QYy5}Dx=MB!LOqD;Jy2v$S<0${i#nv zff|%ev)XjWPC^eYlTV!5xwFvE*%Wal+Iuod|apZ-SVT=tHIOhVNX#QWYtrk`i4>)|G_w>EPaZ$2}P@Sx|5VhC^P8;Z6%v8u(XfAi0Rv-#kMTRc+`Uvx&Fp9$?|Aafu`z%nJ+ zP{t`NI?ZG_(3+~v#xk9$TWH8OHW494Cr*)`XHwH46V~K2htnFj>IS&V@J()@fsgMJ zlR>9cuxjuD9NBcT78(U0BsL4#d;cETo|!F?iPrf`U%u1C>q(J{Rh(JMk`o`ZFfeD} zVw%tvh_o;kh>ScFR~NasGcPjmTLT0TxHL3E8kJ^553|@}_SGzkE$m(zDiVxd^dJcV zYbd61m>-|C7RFemUxU+3mL4LpNCu~HiNm9m3fUw=_K-sR>X*Y>lp(0VFGH=7m{vY1 z7Zm|gU8n@kgGfz%k_krl-GXY_^$KdI2@N)Xl5K|qz1 z4Y5-o`lHVOu&FDx94&ze3=jcJnm{&|@s&vg8CdSPq6={dQc6lui9VzvZ8{{BRC3H= zR^+Zla-aecxMclEl_mGNZ9e1l(8L}I85R;rs7QLuWZWiIcsYqQ2i-^!vltlLiR@$d zwBlJ!7FOHU46QC}>s#U49c%^AfMg`#eNGxKZJ9BzZp;=+3#i5d1`V%;Jz!uN^fl() z#4uFC&t)Eyp~V#DG_>(rUglR5!ys>PmT?UREBm&YT&99rOH$dkCLF~CN3s-aXK5>Q z*~Dlzh7cW5Qch@_yZ_~gM_a33sWKG3+dk$m8Nr>Jj#;~6&dwdBqb0RW=MP{?w{<;{ z-7IS-CST5OyFV$*o=~^CvB*i7ri;_AtQk^WArE=F@e*u&Qa+#PZYE`W+hp~DPLtFn zDH0QrQ3g^h1r2Ou`khX3M#8;BwIn4giJE)WdlsR1O`}K=&3}7j6w=c6zgsHpfGrF? zk3I(^y9!E#y)@D_>eZy-a^qg{N;JZ{^nqYmQVZ?ESP9Y*G+1*@Xl7DLq);Zn1o;;U zN5d1&6ii!;Bw-yFDO%7fBqEZH-rIP?$qivuF_6_Tj@bfMBy#D1VO<$4D> zX-E|%rnuEH3qI78%gt5SbeQXjuzS*uJ}EPH+|3iPiTx93ch}jGv!weIk>s@M;346uG=YX@n01f@}-G9uMMPebIyfj zglR}y9{+mKvQ!?&nB40_S0zXapTy0C2q}>-Vhbi^ioZmndX9IjU)7up$cnKA(r&pC zRqroT{;&{pKETI+EfYpt+0QG3?hb@lVuPmejBbC5N^auJM*b-g?c~PRzuZ zZ;M7sr4mi#MSaea{#O|%0h)a@?J2s*T zq44~;&c*6%?gs0ODlp_6u#H#@?~n%V1gp@7W@*O7qhDcB6 zX*9UZtjNQEeh+h)Et-Ij_gV*yl zcXlV;CWS@HDV%m9-6$oTjN~tfg;Y-ISrqPlki{6c4@B(k-ohf^6f7_RY9C$#Hh3|S zg6b#8?fD!9B$A~^IOS8e58gthSt#nF9EvLv$JL609ODOBkOLD3;}*>)WcG{-`)tLq zkmSCw&<1DAG2Y^XN-Jtc59zoD2XTa9I!(YLX7ik4YATW| zeGtt|=Gf>&m~5}uE(g0@PnL?!+Jvo4fT?qoYxYuS6U(GpBGXLXWcjGg9v?;ifCnui zg;7ou`$$Opih{uK2Q-L6qB_muCPK50;u%#BpNfN3T+<{P?4NY!@N`Aotc7DHio%+r zHfcyH1Sfsy4OzmED}@ZVY;zzZsDG*jr0j97D3H*^&;j$T?gooI15zLd@_m;73}3Ql zG-50CRx5)>3$;$LOkQ)z){?2RCToZcZLn%<_U0)>3o&2|D~}|ykOJv;5Gze+Gq?ps zh|GtYW(jRFQG)Vi%xuj1;zH8{C|kxua}w($6z;$%#=x-T$nZN2WGd4RAPMbTzSAjH zRD~c;9EIaS)X);8$_-~D%8rDmm~L(q$utfEZ3G690%@`ssniCkrs{{MQVA{^Y6Tq; zM`WmzPz|ey#7a42O8-zJcdn*73MXeqxyr)#vTeK6MA?4A6f?6>$fQlqYcvgob|ljk zO-I{cZxr2BPh!!Ulxf>Y^HGARpblzxvI!Ukj#gY#Q=CLPaVz|i%HdxB^I4kWF?3Tr zKqB89Vg0CaB}U={IAfq@!Z;80P;bRrz@!~O0)HlkSdK#BK*d-<_JKMu4Kyhq@&n;co3@(jNm0JqPmT9O*{VrIhXnD`aX%4n{ildeX=8G|Dq-5W~*0-lmaY!$SS`tvNW=-!=%Ge9Snlk2TH59SUGD zpHo>X)qZ@%^zg7R=I1OAqA3RAVkSa!IS*J#Y*DyVq#C8$W;E^)jVcZ8!U#Ikni=`brZ9DxEFfMKRAYeJ~XTBv&H1zteNa3HHT3{P)#?0sBT zF%Bj^&jyCnhP8UuM^@~x24rBz4CM-TJFqB2H#B^&2w~0tS35*h#>S8+?F?P=QD4VU zX=D=69QKUrw_y=WTdXxpEv7{P*w*Uelwh-d`i={^Dv*SZliG%2YGOyG7mqSS2M@_e zBe>BZV_@8;4SUu!#&VJ>agz$kM-Yx4-wG8~(bz7h6Gsszm`PAEhfRmA_ONT(CbPW2 z1Wjm9G5r)wYRMm zOgB{|JQXnV?Vm>V7Z0YJ)};E@1enC606ry0mXRD83U3WbVvYq?2MOW`cv#L6;j%dO zpz%43_#PV&0)=Q|l}0Jc^NkE7Fpt+BD8RYSQ+9X%vsE5ROwZNlR4B>d1?NDCk(Sq1 z39N@64$W_v}I(U)^ds&M3 zHkiX>WsGuPP1NI7^a}rRjMUDgoY`o`sLupTU+q@|N901P7Y2Q>eH4$$ObE6LCU8z; zNe44Oc{a*6q#{+&vcB+Y0<*H*B|@P@o3ZsmtW0{iY=(wQ2itQfhN>;cR4|5F&ETXH zQ`k?5Pj&_+P|ZY6Z_k%NQBItzcH{(@csPgQw1p4FEr$3_TR4y#rF_^Hq0FMaCM#7< zx={^-Ss3az&$2dqEOR4Q$SOh#ce6Kr(^XXeA|(#s03gPK(sufOr=+a~b%bgEZs<07 zb|cg#$tTMH6d0>PpZtQ!}EAQ{rzw6;wqvRe=#z zp^w7{j#`WZj{(d&fkR2EIG&(0AttIx6^^QY6<6hvtG~i5!upzEbWMzPe`g!Bi;~Cr zVqxaxM^*smjbXy7gHSSooWzm<_(7_C zM`ykL!)Mu5Fk!T9-QWOH8p(pXs^W zRM$J(m{sw*)sN?V4jQTIE6J(o$eh}|^5w!a(Ov2M|dw!E604g7trM_;W zcna5~<;(ew3&j5~DZH%0;xV}R^HL#drr{?l z+c(gJ=R|CXVyGe#CZN@`j0=}tAX5&Yi?n!Wh_8FBYqn