From 9ef8ee31a94f53ce48e852beb753febda1947c7d Mon Sep 17 00:00:00 2001 From: Chris Kennelly CA Date: Sun, 18 Oct 2026 19:17:14 -0700 Subject: [PATCH] In debug builds, directly guard against reentrancy. Enable the thread-local allocator reentrancy guard in debug builds. PiperOrigin-RevId: 1002025069 --- tcmalloc/internal/logging.cc | 16 +- tcmalloc/internal/logging.h | 17 + tcmalloc/tcmalloc.cc | 17 +- tcmalloc/testing/BUILD | 13 + tcmalloc/testing/CMakeLists.txt | 16 + tcmalloc/testing/fast_path.release.golden | 2 +- tcmalloc/testing/reentrancy_test.cc | 365 ++++++++++++++++++++++ tcmalloc/testing/testutil.h | 6 +- 8 files changed, 448 insertions(+), 4 deletions(-) create mode 100644 tcmalloc/testing/reentrancy_test.cc diff --git a/tcmalloc/internal/logging.cc b/tcmalloc/internal/logging.cc index 870487481..6c07e6cee 100644 --- a/tcmalloc/internal/logging.cc +++ b/tcmalloc/internal/logging.cc @@ -17,6 +17,8 @@ #include #include + +#include "absl/base/thread_annotations.h" #ifdef _WINDOWS #include #endif @@ -55,7 +57,7 @@ namespace tcmalloc_internal { // may not be able to heap-allocate while crashing. ABSL_CONST_INIT static absl::base_internal::SpinLock crash_lock( absl::base_internal::SCHEDULE_KERNEL_ONLY); -static bool crashed = false; +static bool crashed ABSL_GUARDED_BY(crash_lock) = false; static const size_t kStatsBufferSize = 16 << 10; #ifndef __APPLE__ @@ -167,6 +169,14 @@ void RecordCrash(absl::string_view detector, absl::string_view error) { if (!crashed) { crashed = true; first_crash = true; +#if !defined(NDEBUG) + // Reset reentrancy count to allow calls to malloc/free under any signal + // handlers that we may trigger from abort(). + // + // We only reset this, rather than permit an unlimited amount of + // reentrancy while crashing, in order to maintain modest stack depths. + tcmalloc_reentrancy_count = 0; +#endif } } @@ -287,6 +297,10 @@ PbtxtRegion PbtxtRegion::CreateSubRegion(absl::string_view key) { return sub; } +#if !defined(NDEBUG) +ABSL_CONST_INIT thread_local int tcmalloc_reentrancy_count = 0; +#endif + } // namespace tcmalloc_internal } // namespace tcmalloc GOOGLE_MALLOC_SECTION_END diff --git a/tcmalloc/internal/logging.h b/tcmalloc/internal/logging.h index ce61d8951..9a1cd47e8 100644 --- a/tcmalloc/internal/logging.h +++ b/tcmalloc/internal/logging.h @@ -358,7 +358,24 @@ class PbtxtRegion { PbtxtRegionType type_; }; +#if !defined(NDEBUG) +ABSL_CONST_INIT extern thread_local int tcmalloc_reentrancy_count; +class [[maybe_unused]] ReentrancyGuard { + public: + ReentrancyGuard() { + TC_CHECK_EQ(tcmalloc_reentrancy_count, 0); + ++tcmalloc_reentrancy_count; + } + ~ReentrancyGuard() { --tcmalloc_reentrancy_count; } +}; +#else +class [[maybe_unused]] ReentrancyGuard { + public: + constexpr ReentrancyGuard() = default; + ~ReentrancyGuard() = default; +}; +#endif } // namespace tcmalloc_internal } // namespace tcmalloc diff --git a/tcmalloc/tcmalloc.cc b/tcmalloc/tcmalloc.cc index f8d77ff5d..a46da909c 100644 --- a/tcmalloc/tcmalloc.cc +++ b/tcmalloc/tcmalloc.cc @@ -860,6 +860,8 @@ ABSL_ATTRIBUTE_NOINLINE static void do_unsized_free_irregular( template inline ABSL_ATTRIBUTE_ALWAYS_INLINE void do_free(void* ptr, Policy policy) { + ReentrancyGuard guard; + // TODO(b/404341539): Improve the bound. TC_ASSERT(CorrectAlignment(ptr, static_cast(1))); @@ -972,6 +974,8 @@ template inline ABSL_ATTRIBUTE_ALWAYS_INLINE void do_free_with_size(void* ptr, size_t size, Policy policy) { + ReentrancyGuard guard; + TC_ASSERT(CorrectAlignment(ptr, policy.align())); // This is an optimized path that may be taken if the binary is compiled @@ -1364,6 +1368,8 @@ ABSL_ATTRIBUTE_NOINLINE static typename Policy::pointer_type slow_alloc_large( template static inline Pointer ABSL_ATTRIBUTE_ALWAYS_INLINE fast_alloc(size_t size, Policy policy) { + ReentrancyGuard guard; + // If size is larger than kMaxSize, it's not fast-path anymore. In // such case, GetSizeClass will return false, and we'll delegate to the slow // path. If malloc is not yet initialized, we may end up with size_class == 0 @@ -1373,7 +1379,10 @@ static inline Pointer ABSL_ATTRIBUTE_ALWAYS_INLINE fast_alloc(size_t size, tc_globals.sizemap().GetSizeClass(policy, size); if (ABSL_PREDICT_FALSE(!is_small)) { SLOW_PATH_BARRIER(); - TCMALLOC_MUSTTAIL return slow_alloc_large(size, policy); +#if defined(NDEBUG) && !defined(TCMALLOC_INTERNAL_WITH_ASSERTIONS) + TCMALLOC_MUSTTAIL +#endif + return slow_alloc_large(size, policy); } // TryRecordAllocationFast() returns true if no extra logic is required, e.g.: @@ -1716,6 +1725,12 @@ static inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* do_realloc(void* old_ptr, do_free(old_ptr, MallocPolicy()); return new_ptr; } else { + // We guard here since callers to + // `TCMallocInternalRealloc`/`TCMallocInternalReallocArray` and the above + // conditional will check for reentrancy inside of `fast_alloc` and + // `do_free`. + tcmalloc::tcmalloc_internal::ReentrancyGuard guard; + // We still need to call hooks to report the updated size: tcmalloc::MallocHook::InvokeDeleteHook( {const_cast(old_ptr), std::nullopt, old_size, diff --git a/tcmalloc/testing/BUILD b/tcmalloc/testing/BUILD index 76790bcd3..ab80b3b38 100644 --- a/tcmalloc/testing/BUILD +++ b/tcmalloc/testing/BUILD @@ -1392,3 +1392,16 @@ create_tcmalloc_testsuite( "@com_google_googletest//:gtest_main", ], ) + +create_tcmalloc_testsuite( + name = "reentrancy_test", + srcs = ["reentrancy_test.cc"], + deps = [ + "//tcmalloc:malloc_extension", + "//tcmalloc:malloc_hook", + "//tcmalloc/internal:config", + "//tcmalloc/internal:logging", + "@com_google_absl//absl/base:core_headers", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/tcmalloc/testing/CMakeLists.txt b/tcmalloc/testing/CMakeLists.txt index c00600e44..b638dd70d 100644 --- a/tcmalloc/testing/CMakeLists.txt +++ b/tcmalloc/testing/CMakeLists.txt @@ -1311,3 +1311,19 @@ tcmalloc_cc_test_variants( "tcmalloc::testing_testutil" ) +tcmalloc_cc_test_variants( + NAME + tcmalloc_testing_reentrancy_test + SRCS + "reentrancy_test.cc" + DEPS + "GTest::gtest_main" + "GTest::gmock_main" + "GTest::gmock" + "absl::core_headers" + "tcmalloc::internal_config" + "tcmalloc::internal_logging" + "tcmalloc::malloc_extension" + "tcmalloc::malloc_hook" +) + diff --git a/tcmalloc/testing/fast_path.release.golden b/tcmalloc/testing/fast_path.release.golden index 7d4d0ef79..8efe42c63 100644 --- a/tcmalloc/testing/fast_path.release.golden +++ b/tcmalloc/testing/fast_path.release.golden @@ -12,7 +12,7 @@ new(align)->size - - 5 28 - 6 - 7 - - - 2 new(align,cold)->size - 1 6 33 - 6 - 8 - - - 2 - 1 new(align,nothrow) - - 5 28 - 6 - 6 - - - 2 - 1 new(cold)->size - 1 4 31 - 4 - 7 - - - 2 - 1 -new(cold,token_1)->size - 1 4 31 - 4 - 7 - - - 2 - 1 +new(cold,token_1)->size - - 4 27 - 5 - 7 - - - 2 - 1 new(nothrow) - - 3 27 - 4 - 5 - - - 2 - 1 new(token_1) - - 3 27 - 4 - 5 - - - 2 - 1 new(token_1)->size - - 3 27 - 4 - 6 - - - 2 - 1 diff --git a/tcmalloc/testing/reentrancy_test.cc b/tcmalloc/testing/reentrancy_test.cc new file mode 100644 index 000000000..8ac24ae52 --- /dev/null +++ b/tcmalloc/testing/reentrancy_test.cc @@ -0,0 +1,365 @@ +// Copyright 2025 The TCMalloc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "absl/base/attributes.h" +#include "tcmalloc/internal/config.h" +#include "tcmalloc/internal/logging.h" +#include "tcmalloc/malloc_extension.h" +#include "tcmalloc/malloc_hook.h" + +namespace { + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void wrap_delete(void* ptr, size_t size) { + operator delete(ptr); +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void wrap_delete_array(void* ptr, + size_t size) { + operator delete[](ptr); +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void wrap_sized_delete(void* ptr, + size_t size) { +#ifdef __cpp_sized_deallocation + operator delete(ptr, size); +#else + operator delete(ptr); +#endif +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void wrap_sized_delete_array(void* ptr, + size_t size) { +#ifdef __cpp_sized_deallocation + operator delete[](ptr, size); +#else + operator delete[](ptr); +#endif +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* wrap_new_cold(size_t size) { + return operator new(size, tcmalloc::hot_cold_t(0)); +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* wrap_new_array_aligned_cold_nothrow( + size_t size) { + return operator new[](size, std::align_val_t(64), std::nothrow, + tcmalloc::hot_cold_t(0)); +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* wrap_new_aligned(size_t size) { + return operator new(size, std::align_val_t(64)); +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void wrap_delete_aligned(void* ptr, + size_t size) { +#ifdef __cpp_sized_deallocation + operator delete(ptr, size, std::align_val_t(64)); +#else + operator delete(ptr, std::align_val_t(64)); +#endif +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void wrap_delete_aligned_array( + void* ptr, size_t size) { +#ifdef __cpp_sized_deallocation + operator delete[](ptr, size, std::align_val_t(64)); +#else + operator delete[](ptr, std::align_val_t(64)); +#endif +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* wrap_new_nothrow(size_t size) { + return operator new(size, std::nothrow); +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void wrap_delete_nothrow(void* ptr, + size_t size) { + operator delete(ptr, std::nothrow); +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* wrap_calloc(size_t size) { + return calloc(size, 1); +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void wrap_free(void* ptr, size_t size) { + free(ptr); +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void wrap_free_aligned_sized(void* ptr, + size_t size) { + free_aligned_sized(ptr, 64, size); +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void wrap_sdallocx(void* ptr, size_t size) { + sdallocx(ptr, size, 0); +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* wrap_realloc(size_t size) { + return realloc(malloc(1), size); +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* wrap_reallocarray(size_t size) { + return reallocarray(malloc(1), size, 1); +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* wrap_posix_memalign(size_t size) { + void* res; + if (posix_memalign(&res, 64, size)) { + abort(); + } + return res; +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* wrap_aligned_alloc(size_t size) { + return aligned_alloc(64, size); +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* wrap_memalign(size_t size) { + return memalign(64, size); +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* wrap_size_returning_operator_new( + size_t size) { + return __size_returning_new(size).p; +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* +wrap_size_returning_operator_new_nothrow(size_t size) { + return tcmalloc_size_returning_operator_new_nothrow(size).p; +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* +wrap_size_returning_operator_new_hot_cold(size_t size) { + return __size_returning_new_hot_cold(size, tcmalloc::hot_cold_t(0)).p; +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* +wrap_size_returning_operator_new_hot_cold_nothrow(size_t size) { + return tcmalloc_size_returning_operator_new_hot_cold_nothrow( + size, tcmalloc::hot_cold_t(0)) + .p; +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* +wrap_size_returning_operator_new_aligned(size_t size) { + return __size_returning_new_aligned(size, std::align_val_t(64)).p; +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* +wrap_size_returning_operator_new_aligned_nothrow(size_t size) { + return tcmalloc_size_returning_operator_new_aligned_nothrow( + size, std::align_val_t(64)) + .p; +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* +wrap_size_returning_operator_new_aligned_hot_cold(size_t size) { + return __size_returning_new_aligned_hot_cold(size, std::align_val_t(64), + tcmalloc::hot_cold_t(0)) + .p; +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* +wrap_size_returning_operator_new_aligned_hot_cold_nothrow(size_t size) { + return tcmalloc_size_returning_operator_new_aligned_hot_cold_nothrow( + size, std::align_val_t(64), tcmalloc::hot_cold_t(0)) + .p; +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* wrap_std_allocator_allocate( + size_t size) { + return std::allocator().allocate(size); +} + +#if defined(_LIBCPP_VERSION) && _LIBCPP_VERSION >= 150000 +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* +wrap_std_allocator_allocate_at_least_internal(size_t size) { + std::allocator a; + return std::__allocate_at_least(a, size).ptr; // NOLINT +} +#endif + +#if defined(__cpp_lib_allocate_at_least) && \ + __cpp_lib_allocate_at_least >= 202302L +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* wrap_std_allocator_allocate_at_least( + size_t size) { + return std::allocate_at_least(std::allocator(), size).ptr; +} +#endif + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void wrap_std_allocator_deallocate( + void* ptr, size_t size) { + std::allocator().deallocate(static_cast(ptr), size); +} + +extern "C" ABSL_ATTRIBUTE_NOINLINE void* __alloc_token_9_posix_memalign_test( + size_t size) { + void* res; + if (posix_memalign(&res, 64, size)) { + abort(); + } + return res; +} + +extern "C" ABSL_ATTRIBUTE_NOINLINE void* __alloc_token_0_calloc_test( + size_t size) { + return calloc(size, 1); +} + +extern "C" ABSL_ATTRIBUTE_NOINLINE void* __alloc_token_9_realloc_test( + size_t size) { + return realloc(malloc(1), size); +} + +extern "C" ABSL_ATTRIBUTE_NOINLINE void* __alloc_token_9__Znwm_test( + size_t size) { + return operator new(size); +} + +void ReentrantNewHook(const tcmalloc::MallocHook::NewInfo& info) { + ::operator delete(::operator new(1)); +} + +void ReentrantDeleteHook(const tcmalloc::MallocHook::DeleteInfo& info) { + ::operator delete(::operator new(1)); +} + +template +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void test() { + SCOPED_TRACE(__PRETTY_FUNCTION__); + +#if !defined(NDEBUG) || defined(TCMALLOC_INTERNAL_WITH_ASSERTIONS) + EXPECT_DEATH( + { + TC_CHECK(tcmalloc::MallocHook::AddNewHook(ReentrantNewHook)); + void* volatile ptr = alloc(16); + free_fn(ptr, 16); + }, + "tcmalloc_reentrancy_count"); + + EXPECT_DEATH( + { + TC_CHECK(tcmalloc::MallocHook::AddDeleteHook(ReentrantDeleteHook)); + void* volatile ptr = alloc(16); + free_fn(ptr, 16); + }, + "tcmalloc_reentrancy_count"); +#endif +} + +TEST(ReentrancyTest, Matrix) { + if (tcmalloc::tcmalloc_internal::kSanitizerPresent) { + GTEST_SKIP() << "Skipping under sanitizers"; + } + +#if !defined(NDEBUG) || defined(TCMALLOC_INTERNAL_WITH_ASSERTIONS) + test(); + test(); + test(); + test(); + test(); + test(); + test(); + test(); + test(); + test(); + test(); + test(); + test(); + test(); + test(); + test(); + test(); +#if defined(__cpp_lib_allocate_at_least) && \ + __cpp_lib_allocate_at_least >= 202302L + test(); +#endif +#if defined(_LIBCPP_VERSION) && _LIBCPP_VERSION >= 150000 + test(); +#endif + test(); + test(); + test(); + test(); + test(); + test(); + test(); + test(); + test(); + test(); + test(); + // Allocation functions renamed by LLVM heap partitioning. + test<__alloc_token_9_posix_memalign_test, wrap_free>(); + test<__alloc_token_0_calloc_test, wrap_free>(); + test<__alloc_token_9_realloc_test, wrap_free>(); + test<__alloc_token_9__Znwm_test, wrap_delete>(); +#if defined(__x86_64__) + // pvalloc is not present in some versions of libc. + test(); +#endif +#endif +} + +TEST(ReentrancyTest, ReallocSizeChange) { + if (tcmalloc::tcmalloc_internal::kSanitizerPresent) { + GTEST_SKIP() << "Skipping under sanitizers"; + } + + void* ptr = malloc(16); + +#if !defined(NDEBUG) || defined(TCMALLOC_INTERNAL_WITH_ASSERTIONS) + EXPECT_DEATH( + { + TC_CHECK(tcmalloc::MallocHook::AddNewHook(ReentrantNewHook)); + TC_CHECK(tcmalloc::MallocHook::AddDeleteHook(ReentrantDeleteHook)); + TC_CHECK_EQ(realloc(ptr, 0), nullptr); + }, + "tcmalloc_reentrancy_count"); + + EXPECT_DEATH( + { + TC_CHECK(tcmalloc::MallocHook::AddNewHook(ReentrantNewHook)); + TC_CHECK(tcmalloc::MallocHook::AddDeleteHook(ReentrantDeleteHook)); + TC_CHECK_NE(realloc(ptr, 8), nullptr); + }, + "tcmalloc_reentrancy_count"); + + EXPECT_DEATH( + { + TC_CHECK(tcmalloc::MallocHook::AddNewHook(ReentrantNewHook)); + TC_CHECK(tcmalloc::MallocHook::AddDeleteHook(ReentrantDeleteHook)); + TC_CHECK_NE(realloc(ptr, 64), nullptr); + }, + "tcmalloc_reentrancy_count"); + +#endif + + free(ptr); +} + +} // namespace diff --git a/tcmalloc/testing/testutil.h b/tcmalloc/testing/testutil.h index 1fbb8755a..bbb6c417e 100644 --- a/tcmalloc/testing/testutil.h +++ b/tcmalloc/testing/testutil.h @@ -336,10 +336,14 @@ class LongJmpScope { private: static void FuzzLogWriter(const char* msg, int len) { - if (!absl::StrContains(absl::string_view(msg, len), "CHECK")) { + if (!absl::StrContains(absl::string_view(msg, len), "CHECK") || + absl::StrContains(absl::string_view(msg, len), "ReentrancyGuard")) { return; } +#if !defined(NDEBUG) + tcmalloc_internal::tcmalloc_reentrancy_count = 0; +#endif tcmalloc_internal::log_message_writer = previous_; longjmp(buf_, 1); }