diff --git a/ChangeLog.md b/ChangeLog.md index e8332ccd1ff37..19d13ac143729 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -32,7 +32,7 @@ See docs/process.md for more on how version tagging works. - The WasmFS OPFS backend now uses `readwrite-unsafe` access handles in pthreads builds, so `unlink` and `rename` fail with `EIO` on files open read-only. (#27937) -- mimalloc was updated to 3.6.0. (#28026, #28068) +- mimalloc was updated to 3.6.1. (#28026, #28068, #28121) - Restored `--whole-archive` behaviour when linking with `-sSIDE_MODULE=1` (fixing a regression from 6.0.0 where static archives were no longer included in full), while keeping native archive behaviour for `-shared`. (#28055) diff --git a/system/lib/mimalloc/README.emscripten b/system/lib/mimalloc/README.emscripten index bd9898cc32541..a452384b2e53b 100644 --- a/system/lib/mimalloc/README.emscripten +++ b/system/lib/mimalloc/README.emscripten @@ -1,5 +1,5 @@ -This contains mimalloc b18ab4eef383f655e760881b02eff8af02800879 (v3.6.0) with +This contains mimalloc 56dc5eb80145f9a3ba7ca23d47c6a248b9d8586c (v3.6.1) with Emscripten-specific changes. Origin: https://github.com/microsoft/mimalloc diff --git a/system/lib/mimalloc/include/mimalloc-override.h b/system/lib/mimalloc/include/mimalloc-override.h index c34093fe00dde..05722ca428c04 100644 --- a/system/lib/mimalloc/include/mimalloc-override.h +++ b/system/lib/mimalloc/include/mimalloc-override.h @@ -55,6 +55,10 @@ not accidentally mix pointers from different allocators). #define posix_memalign(p,a,n) mi_posix_memalign(p,a,n) #define _posix_memalign(p,a,n) mi_posix_memalign(p,a,n) +// C23 +#define free_sized(p,n) mi_free_sized(p,n) +#define free_aligned_sized(p,a,n) mi_free_aligned_sized(p,a,n) + // Microsoft aligned variants #define _aligned_malloc(n,a) mi_malloc_aligned(n,a) #define _aligned_realloc(p,n,a) mi_realloc_aligned(p,n,a) diff --git a/system/lib/mimalloc/include/mimalloc.h b/system/lib/mimalloc/include/mimalloc.h index df3b0ce41a6e6..fbd1c256adbf8 100644 --- a/system/lib/mimalloc/include/mimalloc.h +++ b/system/lib/mimalloc/include/mimalloc.h @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_H #define MIMALLOC_H -#define MI_MALLOC_VERSION 30600 // major + 2 digits minor + 2 digits patch +#define MI_MALLOC_VERSION 30601 // major + 2 digits minor + 2 digits patch // ------------------------------------------------------ // Compiler specific attributes @@ -144,8 +144,11 @@ mi_decl_export void mi_free_small(void* p) mi_attr_noexcept; // As `mi_free_small` but `p` should not be a NULL pointer. mi_decl_export void mi_free_small_nonnull(void* p) mi_attr_noexcept; -// For runtime's: return the current owner thread-id of the page containing an allocation. -mi_decl_nodiscard mi_decl_export size_t mi_thread_id_of(const void* p) mi_attr_noexcept; +mi_decl_export void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept; +mi_decl_export void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept; + +// Checked free: always checks the pointer is valid before free-ing (and otherwise ignores the pointer) +mi_decl_export bool mi_cfree(void* p) mi_attr_noexcept; // ------------------------------------------------------------------------------------- @@ -434,6 +437,9 @@ mi_decl_export void mi_theap_delete(mi_theap_t* theap) mi_attr_noexcept; // Abandon all theap pages, preserving live blocks and the theap for reuse. The owner must be the current thread or quiescent outside mimalloc (excluding concurrent heap/theap deletion). mi_decl_export void mi_theap_quiescent_abandon(mi_theap_t* theap) mi_attr_noexcept; +// For runtime's: return the current owner thread-id of the page containing an allocation. +mi_decl_nodiscard mi_decl_export size_t mi_thread_id_of(const void* p) mi_attr_noexcept; + // ------------------------------------------------------ // Deprecated @@ -497,7 +503,7 @@ typedef enum mi_option_e { mi_option_guarded_sample_rate, // guard an object in the min/max range about every N requested bytes (=0) mi_option_guarded_sample_seed, // can be set to allow for a (more) deterministic re-execution when a guard page is triggered (=0) mi_option_generic_collect, // collect theaps every N (=10000) generic allocation calls - mi_option_page_reclaim_on_free, // heap-creation default for reclaim on free: -1 disallow, 0 originating theap only, 1 allow across theaps + mi_option_page_reclaim_on_free, // heap-creation default for reclaim on free: -1 disallow, 0 originating theap only (default), 1 allow across theaps mi_option_page_full_retain, // retain N full (small) pages per size class (=2) mi_option_page_max_candidates, // max candidate pages to consider for allocation (=4) mi_option_max_vabits, // max user space virtual address bits to consider (=48) @@ -514,7 +520,7 @@ typedef enum mi_option_e { mi_option_profile_inuse_interval, // number of in-use bytes grown between automatic snapshots of the heap profiler (=0, disabled) mi_option_profile_time_interval, // number of seconds between automatic snapshots of the heap profiler (=0, disabled) mi_option_profile_sample_rate, // sample rate (in bytes) for heap profiler (=512 KiB) - mi_option_stats_merge_threshold, // on a page stats update, merge theap stats into the parent heap if N bytes were freed/allocated; 0 disables (=512 KiB) + mi_option_stats_merge_threshold, // on a page stats update, merge theap stats into the parent heap if N bytes were freed/allocated; 0 disables (=4 MiB) mi_option_arena_purge_immediate_size, // ranges above this size are purged immediately _mi_option_last, // legacy option names @@ -546,8 +552,6 @@ mi_decl_export void mi_option_set_default(mi_option_t option, long value); // note: we use `mi_cfree` as "checked free" and it checks if the pointer is in our theap before free-ing. // ------------------------------------------------------------------------------------------------------- -mi_decl_export bool mi_cfree(void* p) mi_attr_noexcept; -mi_decl_export void* mi__expand(void* p, size_t newsize) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export size_t mi_malloc_size(const void* p) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export size_t mi_malloc_good_size(size_t size) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept; @@ -556,21 +560,25 @@ mi_decl_export int mi_posix_memalign(void** p, size_t alignment, size_t size); / mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1); mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_valloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); -mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1); mi_decl_nodiscard mi_decl_export void* mi_reallocarray(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3); mi_decl_nodiscard mi_decl_export int mi_reallocarr(void* ptrp, size_t count, size_t size) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept; -mi_decl_export void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept; -mi_decl_export void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept; -mi_decl_export int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept; +// C11 +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1); + +// C23 +mi_decl_export void mi_free_aligned_sized( void* p, size_t alignment, size_t size ) mi_attr_noexcept; +mi_decl_export void mi_free_sized( void* p, size_t size ) mi_attr_noexcept; -// wide characters -mi_decl_export int mi_wdupenv_s(wchar_t** buf, size_t* size, const wchar_t* name) mi_attr_noexcept; -mi_decl_nodiscard mi_decl_export mi_decl_restrict wchar_t* mi_wcsdup(const wchar_t* s) mi_attr_noexcept mi_attr_malloc; -mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s) mi_attr_noexcept mi_attr_malloc; +// MSVC +mi_decl_export void* mi__expand(void* p, size_t newsize) mi_attr_noexcept; +mi_decl_export int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept; +mi_decl_export int mi_wdupenv_s(wchar_t** buf, size_t* size, const wchar_t* name) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export mi_decl_restrict wchar_t* mi_wcsdup(const wchar_t* s) mi_attr_noexcept mi_attr_malloc; +mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s) mi_attr_noexcept mi_attr_malloc; // -------------------------------------------------------- // C++ wrappers @@ -684,8 +692,8 @@ template struct _mi_stl_allocator_common { using propagate_on_container_copy_assignment = std::true_type; using propagate_on_container_move_assignment = std::true_type; using propagate_on_container_swap = std::true_type; - template void construct(U* p, Args&& ...args) { ::new(p) U(std::forward(args)...); } - template void destroy(U* p) mi_attr_noexcept { p->~U(); } + // no construct/destroy: std::allocator_traits provides them, and declaring them + // disables the memmove/memset/trivial-destroy fast paths of the STL containers #else void construct(pointer p, value_type const& val) { ::new(p) value_type(val); } void destroy(pointer p) { p->~value_type(); } diff --git a/system/lib/mimalloc/include/mimalloc/alloc/page.h b/system/lib/mimalloc/include/mimalloc/alloc/page.h index acbda3bc6d710..0c0dde4039b64 100644 --- a/system/lib/mimalloc/include/mimalloc/alloc/page.h +++ b/system/lib/mimalloc/include/mimalloc/alloc/page.h @@ -54,8 +54,12 @@ static inline size_t mi_page_block_size(const mi_page_t* page) { // Page start static inline uint8_t* mi_page_start(const mi_page_t* page) { - // multiplication must be done in `size_t` - return (uint8_t*)page + page->page_offset; + return page->page_start; +} + +// Derive a block pointer from an address in the page; On systems with pointer provenance, this keeps the authority of the page start. +static inline mi_block_t* mi_page_block_from_addr(const mi_page_t* page, mi_addr_t addr) { + return (mi_block_t*)mi_ptr_with_addr(mi_page_start(page), addr); } static inline size_t mi_page_size(const mi_page_t* page) { @@ -127,6 +131,10 @@ static inline bool mi_page_meta_is_separated(const mi_page_t* page) { mi_assert_internal(page != mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN)); return true; #endif + #elif MI_PAGE_META_OS_IS_SEPARATED && !MI_PAGE_META_SMALL_IS_ALIGNED + MI_UNUSED_RELEASE(page); + mi_assert_internal(page != mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN)); + return true; #elif MI_PAGE_META_IS_SEPARATED // usually separated but can still be in front for direct OS allocations (due to size or alignment) or due to MI_PAGE_META_SMALL_IS_ALIGNED return (page->memid.memkind == MI_MEM_ARENA && page != mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN)); @@ -137,6 +145,17 @@ static inline bool mi_page_meta_is_separated(const mi_page_t* page) { } static inline uint8_t* mi_page_slice_start(const mi_page_t* page) { + #if MI_CHERI + // derive from the arena or OS memory as both the page meta data and the page start are bounded + if (page->memid.memkind == MI_MEM_ARENA) { + const mi_memid_arena_info_t* const ainfo = &page->memid.mem.arena; + return (uint8_t*)ainfo->arena->start + ((size_t)ainfo->slice_index * MI_ARENA_SLICE_SIZE); + } + else { + void* const slice_start = (mi_page_meta_is_separated(page) ? mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN) : (void*)page); + return (uint8_t*)mi_ptr_rederive(page->memid.mem.os.base, slice_start); + } + #else if (mi_page_meta_is_separated(page)) { // page meta info is at a separate location (at `arena->pages`) return (uint8_t*)mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN); @@ -145,6 +164,17 @@ static inline uint8_t* mi_page_slice_start(const mi_page_t* page) { // page meta info is at the start of the page slices return (uint8_t*)page; } + #endif +} + +// Re-derive `p` in the page memory with the authority to change its protection +static inline void* mi_page_vmem_ptr(const mi_page_t* page, void* p) { + #if MI_CHERI + return mi_ptr_rederive(mi_page_slice_start(page), p); + #else + MI_UNUSED(page); + return p; + #endif } // This gives the offset relative to the start slice of a page. @@ -152,6 +182,12 @@ static inline size_t mi_page_slice_offset_of(const mi_page_t* page, size_t offse return (mi_page_start(page) - mi_page_slice_start(page)) + offset_relative_to_page_start; } +// The end of the blocks relative to the slice start rounded up to the OS page size. +// This is where the end guard page is if `page->memid.guard_end` is set. +static inline size_t mi_page_guard_end_offset(const mi_page_t* page) { + return mi_align_up(mi_page_slice_offset_of(page, mi_page_size(page)), _mi_os_page_size()); +} + // How much of the page is committed relative to the slice start? (or 0 if fully committed already) static inline size_t mi_page_slice_committed(const mi_page_t* page) { return ((size_t)page->slice_pcommitted * _mi_os_page_size()); @@ -253,12 +289,12 @@ static inline size_t mi_page_min_commit_size(void) { //----------------------------------------------------------- // Thread id of thread that owns this page (with flags in the bottom 2 bits) -static inline mi_threadid_t mi_page_xthread_id(const mi_page_t* page) { +static inline mi_thread_id_t mi_page_xthread_id(const mi_page_t* page) { return mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_id); } // Plain thread id of the thread that owns this page -static inline mi_threadid_t mi_page_thread_id(const mi_page_t* page) { +static inline mi_thread_id_t mi_page_thread_id(const mi_page_t* page) { return (mi_page_xthread_id(page) & ~MI_PAGE_FLAG_MASK); } @@ -300,37 +336,71 @@ static inline void mi_page_set_has_interior_pointers(mi_page_t* page, bool has_a mi_page_flags_set(page, has_aligned, MI_PAGE_HAS_INTERIOR_POINTERS); } +#if MI_QSBR +// For qsbr, `qsbr_xthread_id` keeps the thread id of the last owning thread (so it is still available when the page is abandoned) +// The lowest bit is set if the page is in the qsbr queue of its theap. It is only written by the owning thread. +#define MI_PAGE_QSBR_IN_QUEUE MI_ZU(0x01) + +static inline mi_thread_id_t mi_page_qsbr_xthread_id(const mi_page_t* page) { + return mi_atomic_load_relaxed(&((mi_page_t*)page)->qsbr_xthread_id); +} + +static inline mi_thread_id_t mi_page_qsbr_thread_id(const mi_page_t* page) { + return (mi_page_qsbr_xthread_id(page) & ~MI_PAGE_FLAG_MASK); +} + +static inline void mi_page_qsbr_set_thread_id(mi_page_t* page, mi_thread_id_t tid) { + mi_assert_internal((tid & MI_PAGE_FLAG_MASK) == 0); + const mi_thread_id_t xtid = mi_page_qsbr_xthread_id(page); + mi_atomic_store_relaxed(&page->qsbr_xthread_id, tid | (xtid & MI_PAGE_QSBR_IN_QUEUE)); +} + +static inline bool mi_page_qsbr_is_in_queue(const mi_page_t* page) { + return ((mi_page_qsbr_xthread_id(page) & MI_PAGE_QSBR_IN_QUEUE) != 0); +} + +static inline void mi_page_qsbr_set_in_queue(mi_page_t* page, bool in_queue) { + const mi_thread_id_t xtid = mi_page_qsbr_xthread_id(page); + mi_atomic_store_relaxed(&page->qsbr_xthread_id, (in_queue ? (xtid | MI_PAGE_QSBR_IN_QUEUE) : (xtid & ~MI_PAGE_QSBR_IN_QUEUE))); +} +#endif + static inline void mi_page_set_theap(mi_page_t* page, mi_theap_t* theap) { // mi_assert_internal(!mi_page_is_in_full(page)); // can happen when destroying pages on theap_destroy page->theap = theap; - const mi_threadid_t tid = (theap == NULL ? MI_THREADID_ABANDONED : theap->tld->thread_id); + const mi_thread_id_t tid = (theap == NULL ? MI_THREAD_ID_ABANDONED : theap->tld->thread_id); mi_assert_internal((tid & MI_PAGE_FLAG_MASK) == 0); // we need to use an atomic cas since a concurrent thread may still set the MI_PAGE_HAS_INTERIOR_POINTERS flag (see `alloc_aligned.c`). - mi_threadid_t xtid_old = mi_page_xthread_id(page); - mi_threadid_t xtid; + mi_thread_id_t xtid_old = mi_page_xthread_id(page); + mi_thread_id_t xtid; do { xtid = tid | (xtid_old & MI_PAGE_FLAG_MASK); } while (!mi_atomic_cas_weak_release(&page->xthread_id, &xtid_old, xtid)); + + #if MI_QSBR + // keep the last owning thread id around for `mi_thread_id_of` + if (tid > MI_THREAD_ID_DETACHED) { mi_page_qsbr_set_thread_id(page, tid); } + #endif } static inline bool mi_page_is_abandoned(const mi_page_t* page) { // note: the xtheap field of an abandoned theap is set to the subproc (for fast reclaim-on-free) - return (mi_page_thread_id(page) <= MI_THREADID_ABANDONED_MAPPED); + return (mi_page_thread_id(page) <= MI_THREAD_ID_ABANDONED_MAPPED); } static inline bool mi_page_is_abandoned_mapped(const mi_page_t* page) { - return (mi_page_thread_id(page) == MI_THREADID_ABANDONED_MAPPED); + return (mi_page_thread_id(page) == MI_THREAD_ID_ABANDONED_MAPPED); } static inline void mi_page_set_abandoned_mapped(mi_page_t* page) { mi_assert_internal(mi_page_is_abandoned(page)); - mi_atomic_or_relaxed(&page->xthread_id, (mi_threadid_t)MI_THREADID_ABANDONED_MAPPED); + mi_atomic_or_relaxed(&page->xthread_id, (mi_thread_id_t)MI_THREAD_ID_ABANDONED_MAPPED); } static inline void mi_page_clear_abandoned_mapped(mi_page_t* page) { mi_assert_internal(mi_page_is_abandoned_mapped(page)); - mi_atomic_and_relaxed(&page->xthread_id, (mi_threadid_t)MI_PAGE_FLAG_MASK); + mi_atomic_and_relaxed(&page->xthread_id, (mi_thread_id_t)MI_PAGE_FLAG_MASK); } @@ -408,7 +478,7 @@ static inline bool mi_page_claim_ownership(mi_page_t* page) { static inline void mi_page_qsbr_unlink(mi_page_t* page) { #if MI_QSBR - if (!page->qsbr_in_queue) { + if (!mi_page_qsbr_is_in_queue(page)) { mi_assert_internal(page->qsbr_next == NULL && page->qsbr_prev == NULL); return; } @@ -424,7 +494,7 @@ static inline void mi_page_qsbr_unlink(mi_page_t* page) { theap->qsbr_last = page->qsbr_prev; } page->qsbr_next = page->qsbr_prev = NULL; - page->qsbr_in_queue = false; + mi_page_qsbr_set_in_queue(page, false); #else MI_UNUSED(page); #endif @@ -436,13 +506,13 @@ static inline void mi_page_qsbr_enqueue(mi_page_t* page, mi_theap_t* theap) { mi_assert_internal(mi_page_all_free(page)); mi_assert_internal(mi_heap_qsbr_is_enabled(mi_page_heap(page))); mi_assert_internal(!mi_page_qsbr_is_clear(&page->qsbr)); - if (page->qsbr_in_queue) return; + if (mi_page_qsbr_is_in_queue(page)) return; mi_assert_internal(page->qsbr_next == NULL && page->qsbr_prev == NULL); page->qsbr_prev = theap->qsbr_last; if (theap->qsbr_last != NULL) { theap->qsbr_last->qsbr_next = page; } else { theap->qsbr_first = page; } theap->qsbr_last = page; - page->qsbr_in_queue = true; + mi_page_qsbr_set_in_queue(page, true); #else MI_UNUSED(page); MI_UNUSED(theap); @@ -460,7 +530,7 @@ static inline void _mi_page_qsbr_clear(mi_page_t* page) { static inline bool _mi_page_qsbr_is_clear(const mi_page_t* page) { #if MI_QSBR - return (!page->qsbr_in_queue && page->qsbr_next == NULL && page->qsbr_prev == NULL && + return (!mi_page_qsbr_is_in_queue(page) && page->qsbr_next == NULL && page->qsbr_prev == NULL && mi_page_qsbr_is_clear(&page->qsbr)); #else MI_UNUSED(page); @@ -513,29 +583,32 @@ static inline bool mi_is_in_same_page(const void* p, const void* q) { // return (mi_ptr_page(p) == mi_ptr_page(q)); } -static inline void* mi_ptr_decode(const void* null, const mi_encoded_t x, const uintptr_t* keys) { - const uintptr_t k1 = keys[0]; +// Encoding uses just the address and the decoded block is derived from the page (with `page` as the `null` value). +// (the rotations assume `sizeof(mi_addr_t)==sizeof(size_t)`) +static inline void* mi_ptr_decode(const mi_page_t* page, const mi_encoded_t x, const mi_addr_t* keys) { + const mi_addr_t k1 = keys[0]; #if MI_PAGE_KEY_COUNT==2 - const uintptr_t k2 = keys[1]; + const mi_addr_t k2 = keys[1]; #else - const uintptr_t k2 = mi_rotr(k1,13); + const mi_addr_t k2 = mi_rotr(k1,13); #endif - void* p = (void*)(mi_rotr(x - k1, k1) ^ k2); - return (p==null ? NULL : p); + const mi_addr_t addr = (mi_rotr((mi_addr_t)x - k1, k1) ^ k2); + if (addr == mi_addr(page)) return NULL; + return mi_page_block_from_addr(page, addr); } -static inline mi_encoded_t mi_ptr_encode(const void* null, const void* p, const uintptr_t* keys) { - const uintptr_t k1 = keys[0]; +static inline mi_encoded_t mi_ptr_encode(const void* null, const void* p, const mi_addr_t* keys) { + const mi_addr_t k1 = keys[0]; #if MI_PAGE_KEY_COUNT==2 - const uintptr_t k2 = keys[1]; + const mi_addr_t k2 = keys[1]; #else - const uintptr_t k2 = mi_rotr(k1,13); + const mi_addr_t k2 = mi_rotr(k1,13); #endif - const uintptr_t x = (uintptr_t)(p==NULL ? null : p); - return mi_rotl(x ^ k2, k1) + k1; + const mi_addr_t x = mi_addr(p==NULL ? null : p); + return (mi_encoded_t)(mi_rotl(x ^ k2, k1) + k1); } -static inline uint32_t mi_ptr_encode_canary(const void* null, const void* p, const uintptr_t* keys) { +static inline uint32_t mi_ptr_encode_canary(const void* null, const void* p, const mi_addr_t* keys) { const uint32_t x = (uint32_t)(mi_ptr_encode(null,p,keys)); // make the lowest byte 0 to prevent spurious read overflows which could be a security issue (issue #951) // also clear bit 9 which we set only when a block is freed. @@ -554,26 +627,27 @@ static inline bool mi_ptr_decode_canary_is_freed(uint32_t canary) { return (canary == mi_ptr_encode_canary_freed()); } -static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* block, const uintptr_t* keys ) { +static inline mi_block_t* mi_block_nextx( const mi_page_t* page, const mi_block_t* block, const mi_addr_t* keys ) { mi_track_mem_defined(block,sizeof(mi_block_t)); mi_block_t* next; #if MI_ENCODE_FREELIST - next = (mi_block_t*)mi_ptr_decode(null, block->next, keys); + next = (mi_block_t*)mi_ptr_decode(page, block->next, keys); #else - MI_UNUSED(keys); MI_UNUSED(null); - next = (mi_block_t*)block->next; + MI_UNUSED(keys); + const mi_addr_t addr = (mi_addr_t)block->next; + next = (addr==0 ? NULL : mi_page_block_from_addr(page, addr)); #endif mi_track_mem_noaccess(block,sizeof(mi_block_t)); return next; } -static inline void mi_block_set_nextx(const void* null, mi_block_t* block, const mi_block_t* next, const uintptr_t* keys) { +static inline void mi_block_set_nextx(const void* null, mi_block_t* block, const mi_block_t* next, const mi_addr_t* keys) { mi_track_mem_undefined(block,sizeof(mi_block_t)); #if MI_ENCODE_FREELIST block->next = mi_ptr_encode(null, next, keys); #else MI_UNUSED(keys); MI_UNUSED(null); - block->next = (mi_encoded_t)next; + block->next = (mi_encoded_t)mi_addr(next); // never store a capability in a (freed) block #endif mi_track_mem_noaccess(block,sizeof(mi_block_t)); } diff --git a/system/lib/mimalloc/include/mimalloc/atomic.h b/system/lib/mimalloc/include/mimalloc/atomic.h index 2cbe2486d3d6d..f3ed906988fa0 100644 --- a/system/lib/mimalloc/include/mimalloc/atomic.h +++ b/system/lib/mimalloc/include/mimalloc/atomic.h @@ -411,14 +411,14 @@ static inline mi_ssize_t mi_atomic_loads_relaxed(_Atomic(mi_ssize_t)* p) { // Guard // ---------------------------------------------------------------------- -typedef _Atomic(uintptr_t) mi_atomic_guard_t; +typedef _Atomic(size_t) mi_atomic_guard_t; // Allows only one thread to execute at a time (without blocking anyone) #define mi_atomic_guard(guard) \ - uintptr_t _mi_guard_expected = 0; \ + size_t _mi_guard_expected = 0; \ for(bool _mi_guard_once = true; \ - _mi_guard_once && mi_atomic_cas_strong_acq_rel(guard,&_mi_guard_expected,(uintptr_t)1); \ - (mi_atomic_store_release(guard,(uintptr_t)0), _mi_guard_once = false) ) + _mi_guard_once && mi_atomic_cas_strong_acq_rel(guard,&_mi_guard_expected,(size_t)1); \ + (mi_atomic_store_release(guard,(size_t)0), _mi_guard_once = false) ) // ---------------------------------------------------------------------- @@ -430,8 +430,12 @@ typedef _Atomic(uintptr_t) mi_atomic_guard_t; #pragma warning(disable:26110) // unlock with holding lock #endif -#define mi_lock(lock) for(bool _mi_go = (mi_lock_acquire(lock),true); _mi_go; (mi_lock_release(lock), _mi_go=false) ) -#define mi_lock_maybe(lock,acquire) for(bool _mi_go = (acquire ? (mi_lock_acquire(lock),true) : true); _mi_go; _mi_go = (acquire ? (mi_lock_release(lock),false) : false) ) +// use a unique loop variable per line to avoid shadowing warnings for nested locks +#define mi_concat_(a,b) a##b +#define mi_concat(a,b) mi_concat_(a,b) +#define mi_unique(name) mi_concat(name,__LINE__) +#define mi_lock(lock) for(bool mi_unique(_mi_go) = (mi_lock_acquire(lock),true); mi_unique(_mi_go); (mi_lock_release(lock), mi_unique(_mi_go)=false) ) +#define mi_lock_maybe(lock,acquire) for(bool mi_unique(_mi_go) = ((acquire) ? (mi_lock_acquire(lock),true) : true); mi_unique(_mi_go); mi_unique(_mi_go) = ((acquire) ? (mi_lock_release(lock),false) : false) ) #if defined(_WIN32) || defined(__CYGWIN__) @@ -531,17 +535,16 @@ void _mi_error_message(int err, const char* fmt, ...); void _mi_prim_thread_yield(void); typedef struct mi_lock_s { - _Atomic(uintptr_t) mutex; + _Atomic(size_t) mutex; } mi_lock_t; #define MI_LOCK_INITIALIZER { MI_ATOMIC_VAR_INIT(0) } static inline bool mi_lock_try_acquire(mi_lock_t* lock) { - uintptr_t expected = 0; - return mi_atomic_cas_strong_acq_rel(&lock->mutex, &expected, (uintptr_t)1); + size_t expected = 0; + return mi_atomic_cas_strong_acq_rel(&lock->mutex, &expected, (size_t)1); } static inline void mi_lock_acquire(mi_lock_t* lock) { - size_t ticks = 0; for (int i = 0; i < 10000; i++) { // for at most 10000 tries? if (mi_lock_try_acquire(lock)) return; _mi_prim_thread_yield(); @@ -549,7 +552,7 @@ static inline void mi_lock_acquire(mi_lock_t* lock) { _mi_error_message(EFAULT, "internal error: lock cannot be acquired (due to lack of native lock primitives)\n"); } static inline void mi_lock_release(mi_lock_t* lock) { - mi_atomic_store_release(&lock->mutex, (uintptr_t)0); + mi_atomic_store_release(&lock->mutex, (size_t)0); } static inline void mi_lock_init(mi_lock_t* lock) { mi_lock_release(lock); @@ -562,7 +565,7 @@ static inline void mi_lock_done(mi_lock_t* lock) { typedef struct mi_atomic_once_s { - _Atomic(uintptr_t) tid; + _Atomic(size_t) tid; mi_lock_t lock; } mi_atomic_once_t; diff --git a/system/lib/mimalloc/include/mimalloc/bits.h b/system/lib/mimalloc/include/mimalloc/bits.h index 293dc0be05080..68202ba3a6ea3 100644 --- a/system/lib/mimalloc/include/mimalloc/bits.h +++ b/system/lib/mimalloc/include/mimalloc/bits.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2019-2024 Microsoft Research, Daan Leijen +Copyright (c) 2019-2026 Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -36,10 +36,13 @@ terms of the MIT license. A copy of the license can be found in the file // but we also need: // i == (intptr_t)((void*)i) // or otherwise one might define an intptr_t type that is larger than a pointer... +// +// On CHERI, pointers are 128-bit capabilities but `INTPTR_MAX` is `INT64_MAX` +// (as only the address is an integer value), so we use `__SIZEOF_POINTER__` if available. // ------------------------------------------------------ -#if INTPTR_MAX > INT64_MAX -# define MI_INTPTR_SHIFT (4) // assume 128-bit (as on arm CHERI for example) +#if (INTPTR_MAX > INT64_MAX) || (defined(__SIZEOF_POINTER__) && __SIZEOF_POINTER__ == 16) +# define MI_INTPTR_SHIFT (4) // 128-bit (as on CHERI for example) #elif INTPTR_MAX == INT64_MAX # define MI_INTPTR_SHIFT (3) #elif INTPTR_MAX == INT32_MAX @@ -82,7 +85,6 @@ typedef int32_t mi_ssize_t; #define MI_MiB (MI_KiB*MI_KiB) #define MI_GiB (MI_MiB*MI_KiB) - /* -------------------------------------------------------------------------------- Architecture -------------------------------------------------------------------------------- */ @@ -104,6 +106,10 @@ typedef int32_t mi_ssize_t; #endif #endif +#if defined(__CHERI_PURE_CAPABILITY__) +#define MI_CHERI 1 +#endif + #if MI_ARCH_X64 && defined(__AVX2__) #include #elif MI_ARCH_ARM64 && MI_OPT_SIMD @@ -129,7 +135,7 @@ typedef int32_t mi_ssize_t; // maximum virtual address bits in a user-space pointer #if MI_DEFAULT_VIRTUAL_ADDRESS_BITS > 0 #define MI_MAX_VABITS MI_DEFAULT_VIRTUAL_ADDRESS_BITS -#elif MI_ARCH_X64 +#elif MI_ARCH_X64 && (MI_INTPTR_SIZE > 4) // not on x32 #define MI_MAX_VABITS (47) #elif MI_INTPTR_SIZE > 4 #define MI_MAX_VABITS (48) @@ -146,15 +152,46 @@ typedef int32_t mi_ssize_t; #define MI_MIN_VABITS (43) /* 8 TiB */ #endif -// use a flat page-map or a 2-level one -#ifndef MI_PAGE_MAP_FLAT -#if MI_MAX_VABITS <= 40 && !defined(__APPLE__) && !MI_SECURE && !MI_FREE_IS_CHECKED && MI_FREE_USE_PAGEMAP -#define MI_PAGE_MAP_FLAT 1 +// Always use the page map in `mi_free` on CHERI as we cannot derive the page meta-data from a (bounded) user pointer, +// and on WebAssembly as it cannot do large aligned allocations well (needed for aligned page meta-data). +#if (MI_CHERI || defined(__wasi__) || defined(__EMSCRIPTEN__)) && !defined(MI_FREE_USE_PAGEMAP) +#define MI_FREE_USE_PAGEMAP 1 +#endif + +// Probable OS page size (as statically known; the actual page size is determined at runtime). +// Can be set explicitly, e.g. `-DMI_OS_PROBABLE_PAGE_SIZE=65536` for arm64 Linux kernels with 64 KiB pages. +#if !defined(MI_OS_PROBABLE_PAGE_SIZE) +#if defined(__APPLE__) // check first as `PAGE_SIZE` may be the runtime `vm_page_size` here +#if MI_ARCH_ARM64 +#define MI_OS_PROBABLE_PAGE_SIZE (16*MI_KiB) // macOS, iOS, etc. on Apple silicon +#else +#define MI_OS_PROBABLE_PAGE_SIZE (4*MI_KiB) +#endif +#elif defined(__wasm__) || defined(__EMSCRIPTEN__) || defined(__wasi__) +#define MI_OS_PROBABLE_PAGE_SIZE (64*MI_KiB) // fixed WebAssembly page size +#elif defined(PAGE_SIZE) // only defined if constant (FreeBSD, musl on x86, bionic without flexible page sizes, ...) +#define MI_OS_PROBABLE_PAGE_SIZE PAGE_SIZE +#elif defined(__linux__) && (defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__)) +#define MI_OS_PROBABLE_PAGE_SIZE (64*MI_KiB) // default for ppc64(le) distributions +#elif defined(__linux__) && defined(__loongarch__) +#define MI_OS_PROBABLE_PAGE_SIZE (16*MI_KiB) // default for LoongArch kernels +#elif defined(__ANDROID__) && defined(__BIONIC_NO_PAGE_SIZE_MACRO) +#define MI_OS_PROBABLE_PAGE_SIZE (16*MI_KiB) // built with flexible page size support (NDK r27+), so it can run with 16 KiB pages #else -#define MI_PAGE_MAP_FLAT 0 +#define MI_OS_PROBABLE_PAGE_SIZE (4*MI_KiB) #endif #endif +// Does the OS have guard pages? +#if !defined(MI_OS_HAS_GUARD_PAGES) +#if defined(__wasm__) || defined(__EMSCRIPTEN__) || defined(__wasi__) +#define MI_OS_HAS_GUARD_PAGES 0 +#elif defined(__uClinux__) // no MMU +#define MI_OS_HAS_GUARD_PAGES 0 +#else +#define MI_OS_HAS_GUARD_PAGES 1 +#endif +#endif /* -------------------------------------------------------------------------------- Builtin's @@ -208,7 +245,11 @@ static inline size_t mi_popcount(size_t x) { if (_mi_cpu_has_popcnt) { return mi_msc_builtinz(__popcnt)(x); } else { return _mi_popcount_generic(x); } // see issue #1291 #elif MI_ARCH_X64 && defined(__BMI1__) + #if (MI_SIZE_BITS == 64) return (size_t)_mm_popcnt_u64(x); + #else + return (size_t)_mm_popcnt_u32(x); // x32 + #endif #else #define MI_HAS_FAST_POPCOUNT 0 return (x<=1 ? x : _mi_popcount_generic(x)); @@ -219,8 +260,6 @@ static inline size_t mi_popcount(size_t x) { #define MI_HAS_FAST_POPCOUNT 1 #endif - - size_t _mi_clz_generic(size_t x); size_t _mi_ctz_generic(size_t x); @@ -361,4 +400,130 @@ static inline uint32_t mi_rotl32(uint32_t x, uint32_t r) { } +// ------------------------------------------------------ +// Raw addresses: the integer value of a pointer (without provenance). +// Usually `mi_addr_t == uintptr_t == size_t`, but on CHERI `uintptr_t` is a 128-bit capability +// while the address (`ptraddr_t`) is 64-bit. Use `mi_addr(p)` to get the address of a pointer +// (and derive pointers from an existing pointer to keep provenance). +// ------------------------------------------------------ + +#if defined(__PTRADDR_TYPE__) +typedef __PTRADDR_TYPE__ mi_addr_t; +#else +typedef uintptr_t mi_addr_t; +#endif +#define MI_AU(x) ((mi_addr_t)(x)) + +// Get the address part of a pointer (without provenance) +static inline mi_addr_t mi_addr(const void* p) { + #if MI_CHERI + return (mi_addr_t)__builtin_cheri_address_get(p); + #else + return (mi_addr_t)(uintptr_t)(p); + #endif +} + +// Derive a pointer with address `addr` from `p` (keeping the provenance of `p`) +static inline void* mi_ptr_with_addr(const void* p, mi_addr_t addr) { + #if MI_CHERI + return (void*)__builtin_cheri_address_set(p, addr); + #else + return (void*)((uintptr_t)p + (addr - mi_addr(p))); + #endif +} + +// A pointer from an address without provenance +static inline void* mi_ptr_from_addr(mi_addr_t addr) { + return (void*)(uintptr_t)addr; +} + +// ------------------------------------------------------ +// Capability bounds and permissions (only on CHERI). +// Internally all blocks are derived from the page start which is bounded to the block area +// of the page, while pointers handed out to the user are bounded to their block and lose +// any non-data permissions. +// ------------------------------------------------------ + +#if MI_CHERI +#if defined(__has_include) +#if __has_include() +#include // CHERI_PERM_SW_VMEM on CheriBSD +#endif +#endif +#if defined(CHERI_PERM_SW_VMEM) +#define MI_CHERI_PERM_SW_VMEM CHERI_PERM_SW_VMEM // permission to (un)map memory +#elif defined(CHERI_PERM_CHERIABI_VMMAP) +#define MI_CHERI_PERM_SW_VMEM CHERI_PERM_CHERIABI_VMMAP // older CheriBSD +#else +#define MI_CHERI_PERM_SW_VMEM 0 +#endif +// permissions that data pointers (user blocks and internal meta data) never need +#define MI_CHERI_PERMS_NOT_DATA (__CHERI_CAP_PERMISSION_PERMIT_EXECUTE__ | __CHERI_CAP_PERMISSION_PERMIT_SEAL__ | \ + __CHERI_CAP_PERMISSION_PERMIT_UNSEAL__ | __CHERI_CAP_PERMISSION_ACCESS_SYSTEM_REGISTERS__ | \ + MI_CHERI_PERM_SW_VMEM) +#endif + +// The alignment needed for exactly representable bounds of `size` bytes (always a power of 2, and 1 if not on CHERI) +static inline size_t mi_bounds_align(size_t size) { + #if MI_CHERI + return ~(size_t)__builtin_cheri_representable_alignment_mask(size) + 1; + #else + MI_UNUSED(size); + return 1; + #endif +} + +// Round `size` up such that bounds of that size can be exactly representable (`size` if not on CHERI) +static inline size_t mi_bounds_size(size_t size) { + #if MI_CHERI + return __builtin_cheri_round_representable_length(size); + #else + return size; + #endif +} + +// Restrict `p` to `size` bytes (rounded outward if that is not exactly representable) +static inline void* mi_ptr_bounded(void* p, size_t size) { + #if MI_CHERI + return __builtin_cheri_bounds_set(p, size); + #else + MI_UNUSED(size); + return p; + #endif +} + +// Restrict a (non-NULL) pointer to `size` bytes and data permissions only +static inline void* mi_ptr_bounded_data(void* p, size_t size) { + #if MI_CHERI + return mi_ptr_bounded(__builtin_cheri_perms_and(p, ~(size_t)MI_CHERI_PERMS_NOT_DATA), size); + #else + MI_UNUSED(size); + return p; + #endif +} + +// Re-derive `p` from the capability `auth` (when `p` lacks the bounds or permissions, e.g. a user pointer) +static inline void* mi_ptr_rederive(void* auth, void* p) { + #if MI_CHERI + return mi_ptr_with_addr(auth, mi_addr(p)); + #else + MI_UNUSED(auth); + return p; + #endif +} + +// Limit `size` to the bytes accessible through `p` (`size` if not on a system with provenance) +static inline size_t mi_ptr_bounds_limit(const void* p, size_t size) { + #if MI_CHERI + const size_t len = (size_t)__builtin_cheri_length_get(p); + const size_t ofs = (size_t)__builtin_cheri_offset_get(p); + const size_t avail = (ofs <= len ? len - ofs : 0); + return (size <= avail ? size : avail); + #else + MI_UNUSED(p); + return size; + #endif +} + + #endif // MI_BITS_H diff --git a/system/lib/mimalloc/include/mimalloc/decl.h b/system/lib/mimalloc/include/mimalloc/decl.h index 657eb7b8cc688..35fb70ab1e74b 100644 --- a/system/lib/mimalloc/include/mimalloc/decl.h +++ b/system/lib/mimalloc/include/mimalloc/decl.h @@ -16,11 +16,17 @@ terms of the MIT license. A copy of the license can be found in the file // -------------------------------------------------------------------------- // Compiler defines // -------------------------------------------------------------------------- - -#define mi_decl_cache_align mi_decl_align(64) +#if (defined(__APPLE__) && defined(__aarch64__)) || defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__) +#define MI_CACHE_LINE_SIZE 128 +#define mi_decl_cache_align mi_decl_align(128) // must use a constant here +#else +#define MI_CACHE_LINE_SIZE 64 +#define mi_decl_cache_align mi_decl_align(64) // must use a constant here +#endif #if defined(_MSC_VER) #pragma warning(disable:4127) // suppress constant conditional warning (due to MI_SECURE paths) +#pragma warning(disable:4324) // structure was padded due to alignment specifier (intended for `mi_decl_cache_align`) #pragma warning(disable:26812) // unscoped enum warning #define mi_decl_forceinline __forceinline #define mi_decl_noinline __declspec(noinline) @@ -103,10 +109,6 @@ terms of the MIT license. A copy of the license can be found in the file #define __has_builtin(x) 0 #endif -#if defined(__EMSCRIPTEN__) && !defined(__wasi__) -#define __wasi__ -#endif - #define MI_UNUSED(x) (void)(x) #if (MI_DEBUG>1) #define MI_UNUSED_RELEASE(x) diff --git a/system/lib/mimalloc/include/mimalloc/heap/heap.h b/system/lib/mimalloc/include/mimalloc/heap/heap.h index 0a8e944942652..db756b4ec47fa 100644 --- a/system/lib/mimalloc/include/mimalloc/heap/heap.h +++ b/system/lib/mimalloc/include/mimalloc/heap/heap.h @@ -9,7 +9,7 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_INTERNAL_HEAP_H #include "../types.h" -#include "init.h" +#include "process.h" #include "subproc.h" void _mi_heap_init(mi_heap_t* heap, mi_thread_local_t theap, mi_subproc_t* subproc, mi_arena_id_t exclusive_arena_id, bool use_qsbr); diff --git a/system/lib/mimalloc/include/mimalloc/heap/init.h b/system/lib/mimalloc/include/mimalloc/heap/init.h deleted file mode 100644 index 3f2c6506c66b2..0000000000000 --- a/system/lib/mimalloc/include/mimalloc/heap/init.h +++ /dev/null @@ -1,27 +0,0 @@ -/* ---------------------------------------------------------------------------- -Copyright (c) 2018-2026, Microsoft Research, Daan Leijen -This is free software; you can redistribute it and/or modify it under the -terms of the MIT license. A copy of the license can be found in the file -"LICENSE" at the root of this distribution. ------------------------------------------------------------------------------*/ -#pragma once -#ifndef MI_INTERNAL_INIT_H -#define MI_INTERNAL_INIT_H - -#include "../types.h" - -mi_heap_t* _mi_subproc_heap_main(mi_subproc_t* subproc); -mi_theap_t* _mi_theap_main(void); // current thread's backing theap, without initialization -mi_page_t* _mi_page_empty_get(void); -void _mi_auto_process_init(void); -void mi_cdecl _mi_auto_process_done(void) mi_attr_noexcept; -bool _mi_preloading(void); // true while the C runtime is not initialized yet -void _mi_thread_done(mi_theap_t* theap); -mi_theap_t* _mi_thread_init(void); -mi_theap_t* _mi_thread_init_with_heap(mi_heap_t* heap); -bool _mi_is_empty_theap(const mi_theap_t* theap); - -extern mi_decl_hidden const mi_theap_t _mi_theap_empty; // read-only empty theap, initial value of the thread local default theap (in the MI_TLS_MODEL_LOCAL) -extern mi_decl_hidden mi_theap_t _mi_theap_empty_wrong; // read-only empty theap used to signal that a theap for a heap could not be allocated - -#endif diff --git a/system/lib/mimalloc/include/mimalloc/heap/page-map.h b/system/lib/mimalloc/include/mimalloc/heap/page-map.h index fd26187ac8adb..d9a10723869be 100644 --- a/system/lib/mimalloc/include/mimalloc/heap/page-map.h +++ b/system/lib/mimalloc/include/mimalloc/heap/page-map.h @@ -11,6 +11,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "../types.h" #include "../util/libc.h" #include "../util/options.h" +#include "process.h" bool _mi_page_map_init(void); mi_decl_nodiscard bool _mi_page_map_register(mi_page_t* page); @@ -24,46 +25,6 @@ mi_page_t* _mi_safe_ptr_page(const void* p); The page map maps addresses to `mi_page_t` pointers ----------------------------------------------------------- */ -#if MI_PAGE_MAP_FLAT - -// flat page-map committed on demand, using one byte per slice (64 KiB). -// single indirection and low commit, but large initial virtual reserve (4 GiB with 48 bit virtual addresses) -// used by default on <= 40 bit virtual address spaces. -extern mi_decl_hidden _Atomic(uint8_t*) _mi_page_map; -extern mi_decl_hidden _Atomic(void*) _mi_page_map_max_address; - -static inline size_t mi_page_map_index(const void* p) { - return (size_t)((uintptr_t)p >> MI_ARENA_SLICE_SHIFT); -} - -static inline uint8_t mi_page_map_at(size_t idx) { - return mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map)[idx]; -} - -static inline mi_page_t* mi_ptr_page_ex(const void* p, bool* valid) { - const size_t idx = mi_page_map_index(p); - const size_t ofs = mi_page_map_at(idx); - if (valid != NULL) { *valid = (ofs != 0); } - return (mi_page_t*)((((uintptr_t)p >> MI_ARENA_SLICE_SHIFT) + 1 - ofs) << MI_ARENA_SLICE_SHIFT); -} - -static inline mi_page_t* mi_checked_ptr_page(const void* p) { - #if MI_MIN_VABITS < MI_INTPTR_BITS - if mi_unlikely(((uintptr_t)p >> MI_MIN_VABITS) != 0) { - if (p > mi_atomic_load_ptr_relaxed(void, &_mi_page_map_max_address)) return NULL; - } - #endif - bool valid; - mi_page_t* const page = mi_ptr_page_ex(p, &valid); - return (valid ? page : NULL); -} - -static inline mi_page_t* mi_unchecked_ptr_page(const void* p) { - return mi_ptr_page_ex(p, NULL); -} - -#else - // 2-level page map: // double indirection, but low commit and low virtual reserve. // @@ -75,25 +36,32 @@ static inline mi_page_t* mi_unchecked_ptr_page(const void* p) { #define MI_PAGE_MAP_SUB_COUNT (MI_ZU(1) << MI_PAGE_MAP_SUB_SHIFT) #define MI_PAGE_MAP_SHIFT (MI_MAX_VABITS - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT) +// The page map pointer and committed entry count are stored in the (guarded) `_mi_process.page_map(_committed)`. +// Both are zero before initialization so `mi_checked_ptr_page` returns NULL for any pointer (issue #1341). typedef mi_page_t** mi_submap_t; typedef struct mi_page_map_s { - _Atomic(size_t) committed_count; // currently committed entries size_t reserved_size; // full reserved size (mi_page_map_t + submaps) mi_memid_t memid; // provenance mi_lock_t lock; // used when allocating new submaps _Atomic(mi_submap_t) submaps[1]; } mi_page_map_t; -extern mi_decl_hidden _Atomic(mi_page_map_t*) __mi_page_map; - static inline size_t mi_page_map_index(const void* p, size_t* sub_idx) { - const size_t u = (size_t)((uintptr_t)p / MI_ARENA_SLICE_SIZE); + const size_t u = (size_t)(mi_addr(p) / MI_ARENA_SLICE_SIZE); if (sub_idx != NULL) { *sub_idx = u % MI_PAGE_MAP_SUB_COUNT; } return (u / MI_PAGE_MAP_SUB_COUNT); } +// Index for looking up a (user) pointer +static inline size_t mi_page_map_lookup_index(const void* p, size_t* sub_idx) { + #if MI_CHERI + if (!__builtin_cheri_tag_get(p)) { p = NULL; } // treat untagged pointers as NULL + #endif + return mi_page_map_index(p, sub_idx); +} + static inline mi_page_map_t* mi_page_map(void) { - return mi_atomic_load_ptr_relaxed(mi_page_map_t,&__mi_page_map); + return mi_atomic_load_ptr_relaxed(mi_page_map_t,&_mi_process.page_map); } static inline mi_submap_t mi_page_map_at(const mi_page_map_t* pmap, size_t idx) { @@ -103,27 +71,27 @@ static inline mi_submap_t mi_page_map_at(const mi_page_map_t* pmap, size_t idx) static inline mi_page_t* mi_unchecked_ptr_page(const void* p) { const mi_page_map_t* pmap = mi_page_map(); size_t sub_idx; - const size_t idx = mi_page_map_index(p, &sub_idx); + const size_t idx = mi_page_map_lookup_index(p, &sub_idx); return mi_page_map_at(pmap,idx)[sub_idx]; // NULL if p==NULL } static inline mi_page_t* mi_checked_ptr_page(const void* p) { - const mi_page_map_t* pmap = mi_page_map(); size_t sub_idx; - const size_t idx = mi_page_map_index(p, &sub_idx); - const size_t committed_count = mi_atomic_load_relaxed(&pmap->committed_count); + const size_t idx = mi_page_map_lookup_index(p, &sub_idx); + const mi_page_map_t* pmap = mi_page_map(); // NULL before initialization (but then the committed count is 0) + const size_t committed_count = mi_atomic_load_relaxed(&_mi_process.page_map_committed); if mi_unlikely(idx >= committed_count) return NULL; - // #if MI_MIN_VABITS < MI_INTPTR_BITS // is still invalid if free is called before the pagemap is initialized - // if mi_unlikely(((uintptr_t)p >> MI_MIN_VABITS) != 0) { - // const size_t committed_count = mi_atomic_load_relaxed(&pmap->committed_count); - // if mi_unlikely(idx >= committed_count) return NULL; - // } - // #endif mi_submap_t const sub = mi_page_map_at(pmap,idx); if mi_unlikely(sub == NULL) return NULL; return sub[sub_idx]; } + +// Is `p` NULL given its page lookup `page`? (with an aligned page meta offset the aligned page lookup is never NULL) +#if MI_PAGE_META_IS_ALIGNED && MI_PAGE_META_ALIGNED_OFFSET > 0 +#define mi_ptr_page_is_null(p,page) ((p)==NULL) +#else +#define mi_ptr_page_is_null(p,page) ((page)==NULL) #endif #if MI_PAGE_META_IS_ALIGNED @@ -132,12 +100,12 @@ static inline mi_page_t* mi_checked_ptr_page(const void* p) { static inline mi_page_t* mi_aligned_ptr_page0(const void* p) { mi_page_t* const page_metas = (mi_page_t*)mi_align_down_ptr(p,MI_PAGE_META_ALIGNMENT); // const ptrdiff_t page_idx = ((uint8_t*)p - (uint8_t*)page_metas)/MI_ARENA_SLICE_SIZE; - const uintptr_t page_idx = ((uintptr_t)p / MI_ARENA_SLICE_SIZE) % (MI_PAGE_META_ALIGNMENT / MI_ARENA_SLICE_SIZE); + const size_t page_idx = (size_t)((mi_addr(p) / MI_ARENA_SLICE_SIZE) % (MI_PAGE_META_ALIGNMENT / MI_ARENA_SLICE_SIZE)); mi_assert_internal(page_idx <= MI_PAGE_META_ALIGNED_COUNT); #if MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_RISCV // better code on x64/x86/riscv64 - mi_page_t* const page = (mi_page_t*)((uintptr_t)page_metas | (page_idx * sizeof(mi_page_t))); + mi_page_t* const page = (mi_page_t*)(((uintptr_t)page_metas | (page_idx * sizeof(mi_page_t))) + MI_PAGE_META_ALIGNED_OFFSET); #else - mi_page_t* const page = &page_metas[page_idx]; + mi_page_t* const page = (mi_page_t*)((uint8_t*)&page_metas[page_idx] + MI_PAGE_META_ALIGNED_OFFSET); #endif return page; @@ -145,7 +113,7 @@ static inline mi_page_t* mi_aligned_ptr_page0(const void* p) { static inline mi_page_t* mi_aligned_ptr_page(const void* p) { mi_page_t* const page = mi_aligned_ptr_page0(p); - if mi_unlikely(page==NULL) return NULL; + if mi_unlikely(mi_ptr_page_is_null(p,page)) return NULL; #if MI_DEBUG mi_page_t* const cpage = mi_checked_ptr_page(p); if mi_unlikely(cpage==NULL) { @@ -159,7 +127,7 @@ static inline mi_page_t* mi_aligned_ptr_page(const void* p) { static inline mi_page_t* mi_ptr_page(const void* p) { mi_assert_internal(p==NULL || mi_is_in_heap_region(p)); - #if MI_SECURE || MI_FREE_IS_CHECKED + #if MI_FREE_IS_CHECKED return mi_checked_ptr_page(p); #elif MI_PAGE_META_IS_ALIGNED return mi_aligned_ptr_page(p); diff --git a/system/lib/mimalloc/include/mimalloc/heap/process.h b/system/lib/mimalloc/include/mimalloc/heap/process.h new file mode 100644 index 0000000000000..e15bdd00f2c73 --- /dev/null +++ b/system/lib/mimalloc/include/mimalloc/heap/process.h @@ -0,0 +1,117 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MI_INTERNAL_PROCESS_H +#define MI_INTERNAL_PROCESS_H + +#include "../types.h" +#include "../prim.h" // mi_os_mem_config_t +#include "../util/options.h" + +mi_heap_t* _mi_subproc_heap_main(mi_subproc_t* subproc); +mi_theap_t* _mi_theap_main(void); // current thread's backing theap, without initialization +mi_page_t* _mi_page_empty_get(void); +void _mi_auto_process_init(void); +void mi_cdecl _mi_auto_process_done(void) mi_attr_noexcept; +bool _mi_preloading(void); // true while the C runtime is not initialized yet +void _mi_thread_done(mi_theap_t* theap); +mi_theap_t* _mi_thread_init(void); +mi_theap_t* _mi_thread_init_with_heap(mi_heap_t* heap); +bool _mi_is_empty_theap(const mi_theap_t* theap); + +extern mi_decl_hidden const mi_theap_t _mi_theap_empty; // read-only empty theap, initial value of the thread local default theap (in the MI_TLS_MODEL_LOCAL) + +// Fork state +typedef struct mi_fork_state_s { + size_t epoch; // incremented in the child at every fork + _Atomic(size_t) pending; // set in the child until the dead theaps are abandoned + bool torn; // set in a child that may have a torn snapshot of memory (on macOS) + mi_heap_t* suspended_heap; // heap with suspended abandoned reclaim (on macOS) + size_t suspended_count[MI_BIN_COUNT]; // and its abandoned counts +} mi_fork_state_t; + + +// In secure mode, guard static process data such that guard pages fit in secure mode (twice the maximal OS page size) +#ifndef MI_SECURE_STATIC_GUARD + #if !MI_SECURE || !MI_OS_HAS_GUARD_PAGES + #define MI_SECURE_STATIC_GUARD (MI_CACHE_LINE_SIZE) // no guard pages; just a cache line + #else + #define MI_SECURE_STATIC_GUARD (2*MI_OS_PROBABLE_PAGE_SIZE) // checked at runtime + #endif +#endif + +// Output before an output function is registered (or the C runtime is initialized) is buffered (see `util/options.c`) +#ifndef MI_MAX_DELAY_OUTPUT +#define MI_MAX_DELAY_OUTPUT (16*MI_KiB) +#endif + +// Statically allocated process data. This is zero initialized and padded such that +// in secure mode it is surrounded by guard pages (see `heap/process.c:mi_process_guard_init`) +typedef struct mi_process_s { + uint8_t guard_front[MI_SECURE_STATIC_GUARD]; + + _Atomic(void*) deferred_free; // is `mi_deferred_free_fun*` (but some platforms don't support atomic function pointers) + _Atomic(void*) deferred_arg; + _Atomic(void*) out_default; // is `mi_output_fun*` + _Atomic(void*) out_arg; + _Atomic(void*) error_handler; // is `mi_error_fun*` + _Atomic(void*) error_arg; + + _Atomic(struct mi_page_map_s*) page_map; // the page map (`NULL` before initialization) (see `heap/page-map.h`) + _Atomic(size_t) page_map_committed; // committed entries in the page map + + bool process_is_initialized; // set to `true` in `mi_process_init` + bool libc_is_initialized; // `true` once the C runtime is initialized (and `false` again after process done) + bool malloc_is_redirected; // `true` if malloc is redirected to mimalloc (on Windows with `mimalloc-redirect.dll`) + _Atomic(size_t) process_done_called; // set once `mi_process_done` is called + _Atomic(size_t) subprocs_created; // total number of sub-processes created (used for `subproc_seq`) + mi_atomic_guard_t arena_purge_guard; // allow only one thread at a time to purge arenas + + mi_msecs_t clock_process_start; // clock at process initialization (see `util/stats.c`) + mi_msecs_t clock_overhead; // measured overhead of reading the clock (subtracted in `_mi_clock_end`) + int64_t clock_ticks_per_msec; // performance counter ticks per millisecond (on Windows) (0 until first use) + + _Atomic(size_t) output_buf_len; // used length of `output_buf` + _Atomic(size_t) error_count; // stop emitting errors when >= `mi_option_max_errors` + _Atomic(size_t) warning_count; // stop emitting warnings when >= `mi_option_max_warnings` + + mi_os_mem_config_t os_mem_config; // OS memory configuration + _Atomic(size_t) os_numa_node_count; // cached NUMA node count (0 until first use) + _Atomic(mi_addr_t) os_aligned_base; // next aligned address hint (see `_mi_os_get_aligned_hint`) + _Atomic(mi_addr_t) os_huge_start; // next address for huge OS pages (see `mi_os_claim_huge_pages`) + + mi_lock_t thread_locals_lock; // initialized in `_mi_thread_locals_init` (see `util/threadlocal.c`) + mi_bitmap_t* thread_locals_free; // bitmap of free thread local slots (1=free, 0=in-use) + mi_memid_t thread_locals_memid; // provenance of `thread_locals_free` + size_t thread_locals_version; // version to be able to reuse slots safely + + mi_subproc_t* subprocs; // list of all sub-processes + mi_lock_t subprocs_lock; // initialized in `_mi_subproc_main_init` (as a zero lock is invalid on some platforms) + mi_fork_state_t fork; // fork state (see `heap/subproc.c`) + + mi_profiler_t* profile_env_profiler; // the pprof profiler if `MIMALLOC_PROFILE` was set (see `profile/pprof.c`) + mi_atomic_guard_t profile_snapshot_guard; // allow only one thread at a time to take a profile snapshot + + mi_decl_cache_align mi_subproc_t subproc_main; // initial subprocess + mi_decl_cache_align mi_heap_t heap_main; // initial heap for the process + mi_decl_cache_align mi_theap_t theap_main; // initial theap for the main thread + mi_decl_cache_align mi_tld_t tld_main; // initial tld for the main thread + mi_decl_cache_align mi_theap_t theap_meta; // detached theap for meta-data allocation + mi_decl_cache_align mi_tld_t tld_meta; // detached tld for `theap_meta` + mi_decl_cache_align mi_theap_t theap_empty_wrong; // read-only empty theap used to signal that a theap for a heap could not be allocated + + // arrays last + mi_option_state_t options[_mi_option_last]; // current option values (see `util/options.c`) + char output_buf[MI_MAX_DELAY_OUTPUT+1]; // delayed output buffer (see `util/options.c`) + + // end guard + mi_decl_cache_align uint8_t guard_end[MI_SECURE_STATIC_GUARD]; +} mi_process_t; + +extern mi_decl_hidden mi_process_t _mi_process; + +#endif diff --git a/system/lib/mimalloc/include/mimalloc/heap/subproc.h b/system/lib/mimalloc/include/mimalloc/heap/subproc.h index 0b37a5dfb4d8d..9f0b5a4a2f639 100644 --- a/system/lib/mimalloc/include/mimalloc/heap/subproc.h +++ b/system/lib/mimalloc/include/mimalloc/heap/subproc.h @@ -19,6 +19,10 @@ mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id); void _mi_subprocs_unsafe_destroy_all(void); void _mi_fork_init(void); void _mi_fork_child(void); +void _mi_fork_child_zone(void); +void _mi_fork_child_try_collect(void); +size_t _mi_fork_epoch(void); +bool _mi_fork_is_torn(void); void* _mi_meta_zalloc( mi_subproc_t* subproc, size_t size, mi_memid_t* memid ); void* _mi_meta_rezalloc( mi_subproc_t* subproc, void* p, size_t newsize, mi_memid_t* memid ); diff --git a/system/lib/mimalloc/include/mimalloc/heap/theap.h b/system/lib/mimalloc/include/mimalloc/heap/theap.h index 86f42b8a82e25..3ba7464db3b71 100644 --- a/system/lib/mimalloc/include/mimalloc/heap/theap.h +++ b/system/lib/mimalloc/include/mimalloc/heap/theap.h @@ -12,6 +12,7 @@ terms of the MIT license. A copy of the license can be found in the file size_t _mi_theap_random_next(mi_theap_t* theap); void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld); +void _mi_theap_meta_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld); mi_theap_t* _mi_theap_alloc(mi_heap_t* heap, mi_tld_t* tld); mi_theap_t* _mi_theap_create(mi_heap_t* heap, mi_tld_t* tld); void _mi_theap_collect_abandon(mi_theap_t* theap); @@ -59,7 +60,7 @@ static inline mi_page_t* mi_theap_get_free_small_page(mi_theap_t* theap, size_t } static inline bool mi_theap_is_detached(mi_theap_t* theap) { - return (theap!=NULL && theap->tld->thread_id == MI_THREADID_DETACHED); + return (theap!=NULL && theap->tld->thread_id == MI_THREAD_ID_DETACHED); } #endif diff --git a/system/lib/mimalloc/include/mimalloc/internal.h b/system/lib/mimalloc/include/mimalloc/internal.h index 2f1ecf489edce..a2834db40681e 100644 --- a/system/lib/mimalloc/include/mimalloc/internal.h +++ b/system/lib/mimalloc/include/mimalloc/internal.h @@ -35,7 +35,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "heap/page-map.h" #include "heap/subproc.h" -#include "heap/init.h" +#include "heap/process.h" #include "heap/theap.h" #include "heap/heap.h" diff --git a/system/lib/mimalloc/include/mimalloc/prim.h b/system/lib/mimalloc/include/mimalloc/prim.h index 38e9499054b5f..f82c886ccbaea 100644 --- a/system/lib/mimalloc/include/mimalloc/prim.h +++ b/system/lib/mimalloc/include/mimalloc/prim.h @@ -70,8 +70,14 @@ int _mi_prim_reset(void* addr, size_t size); // Returns error code or 0 on success. On most platforms this is a no-op. int _mi_prim_reuse(void* addr, size_t size); -// Protect memory. Returns error code or 0 on success. -int _mi_prim_protect(void* addr, size_t size, bool protect); +// Set (or reset) a guard range. Returns error code or 0 on success. +// `is_committed` is the accounted commit state of the range which is never changed by the guard: +// - if `is_committed`, the range stays committed but becomes inaccessible (and its physical memory may be released); +// a reset makes the range accessible again. +// - otherwise, the range may be reserved, decommitted, or still partly committed. After a set the range is +// inaccessible until it is explicitly committed (`_mi_prim_commit`), and a reset is a no-op. +// The caller must ensure the accounted commit state does not change between a set and reset. +int _mi_prim_guard(void* addr, size_t size, bool set, bool is_committed); // Allocate huge (1GiB) pages possibly associated with a NUMA node. // `is_zero` is set to true if the memory was zero initialized (as on most OS's) diff --git a/system/lib/mimalloc/include/mimalloc/prim/os.h b/system/lib/mimalloc/include/mimalloc/prim/os.h index a2ac2d4fd9383..f9013b744459c 100644 --- a/system/lib/mimalloc/include/mimalloc/prim/os.h +++ b/system/lib/mimalloc/include/mimalloc/prim/os.h @@ -30,21 +30,24 @@ bool _mi_os_decommit(mi_subproc_t* subproc, void* addr, size_t size); void _mi_os_reuse(mi_subproc_t* subproc, void* p, size_t size); mi_decl_nodiscard bool _mi_os_commit(mi_subproc_t* subproc, void* p, size_t size, bool* is_zero); mi_decl_nodiscard bool _mi_os_commit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero, size_t stat_already_committed); -mi_decl_nodiscard bool _mi_os_protect(void* addr, size_t size); -bool _mi_os_unprotect(void* addr, size_t size); +bool _mi_os_guard(void* addr, size_t size, bool is_committed); +bool _mi_os_unguard(void* addr, size_t size, bool is_committed); bool _mi_os_purge(mi_subproc_t* subproc, void* p, size_t size); bool _mi_os_purge_ex(mi_subproc_t* subproc, void* p, size_t size, bool allow_reset, size_t stats_size, mi_commit_fun_t* commit_fun, void* commit_fun_arg); size_t _mi_os_secure_guard_page_size(void); bool _mi_os_secure_guard_page_set_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid); bool _mi_os_secure_guard_page_set_before(mi_subproc_t* subproc, void* addr, mi_memid_t memid); +bool _mi_os_secure_guard_pages_set_around(mi_subproc_t* subproc, void* addr, size_t size, mi_memid_t memid); bool _mi_os_secure_guard_page_reset_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid); -bool _mi_os_secure_guard_page_reset_before(mi_subproc_t* subproc, void* addr, mi_memid_t memid); +bool _mi_os_secure_guard_pages_set_within(void* area, size_t area_size, size_t offset, size_t size); int _mi_os_numa_node(void); int _mi_os_numa_node_count(void); void* _mi_os_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid); +void* _mi_os_alloc_with_guards(mi_subproc_t* subproc, size_t size, bool commit, bool allow_large, mi_memid_t* memid); +void _mi_os_free_with_guards(mi_subproc_t* subproc, mi_memid_t memid); void* _mi_os_alloc_aligned_at_offset(mi_subproc_t* subproc, size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_memid_t* memid); void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size); diff --git a/system/lib/mimalloc/include/mimalloc/prim/prim-tls.h b/system/lib/mimalloc/include/mimalloc/prim/prim-tls.h index 110c831dba973..4c8dcd6d6f7ab 100644 --- a/system/lib/mimalloc/include/mimalloc/prim/prim-tls.h +++ b/system/lib/mimalloc/include/mimalloc/prim/prim-tls.h @@ -9,6 +9,7 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_INTERNAL_PRIM_TLS_H #include "../types.h" +#include "../heap/process.h" #include "../heap/heap.h" #include "../heap/theap.h" #include "../util/threadlocal.h" @@ -27,7 +28,7 @@ terms of the MIT license. A copy of the license can be found in the file // Android,OpenBSD : use pthread locals (MI_TLS_MODEL_PTHREADS). todo: maybe on Android MI_TLS_MODEL_LOCAL is better? // -------------------------------------------------------------------------- -static inline mi_threadid_t _mi_prim_thread_id(void); // get a unique id for a thread +static inline mi_thread_id_t _mi_prim_thread_id(void); // get a unique id for a thread static inline mi_theap_t* _mi_theap_default(void); // the default thread local theap static inline mi_theap_t* _mi_theap_cached(void); // last used thread local theap using the _heap_ api static inline bool _mi_thread_is_initialized(void); // a thread is initialized if it has a default theap @@ -38,13 +39,13 @@ static inline mi_theap_t* _mi_page_associated_theap_peek(const mi_page_t* page // Regular exports void _mi_tls_slots_init(void); void _mi_tls_slots_done(void); -mi_threadid_t _mi_thread_id(void) mi_attr_noexcept; +mi_thread_id_t _mi_thread_id(void) mi_attr_noexcept; void _mi_theap_default_set(mi_theap_t* theap); void _mi_theap_cached_set(mi_theap_t* theap); static inline bool mi_theap_matches_thread(mi_theap_t* theap) { - const mi_threadid_t tid = _mi_thread_id(); + const mi_thread_id_t tid = _mi_thread_id(); return (theap==NULL || theap->tld==NULL || theap->tld->thread_id == tid || mi_theap_is_detached(theap)); } @@ -96,11 +97,13 @@ static inline void** mi_prim_thread_pointer(void) { return (void**)__builtin_thread_pointer(); } #elif defined(__GNUC__) && !defined(__CYGWIN__) - #if defined(__aarch64__) + #if MI_CHERI + #define MI_NO_THREAD_POINTER (1) + #elif defined(__aarch64__) static inline void** mi_prim_thread_pointer(void) { // Use a 64-bit temporary: on arm64_32 (ILP32), pointers are 32 bits but system registers are 64 bits. uint64_t tp; - #if defined(__APPLE__) // M1, issue rgb(62, 76, 62) + #if defined(__APPLE__) // M1, issue #343 __asm__ ("mrs %0, tpidrro_el0\n\tbic %0, %0, #7" : "=r" (tp)); #else __asm__ ("mrs %0, tpidr_el0" : "=r" (tp)); @@ -113,7 +116,9 @@ static inline void** mi_prim_thread_pointer(void) { __asm__ ("mv %0, tp" : "=r" (tcb)); return tcb; } - #elif defined(__arm__) + #elif defined(__arm__) /* arm32 with tpidruro */ \ + && (__ARM_ARCH >= 7 || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6KZ__) || defined(__ARM_ARCH_6ZK__)) \ + && !(defined(__ARM_ARCH_PROFILE) && __ARM_ARCH_PROFILE == 'M') static inline void** mi_prim_thread_pointer(void) { void** tcb; __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\n\tbic %0, %0, #3" : "=r" (tcb)); @@ -175,28 +180,28 @@ static inline void mi_prim_tls_slot_set(size_t slot, void* value) { // Get a unique id for the current thread. #if defined(MI_PRIM_THREAD_ID) -static inline mi_threadid_t __mi_prim_thread_id(void) { +static inline mi_thread_id_t __mi_prim_thread_id(void) { return MI_PRIM_THREAD_ID(); // used for example by CPython for a free threaded build (see python/cpython#115488) } #elif !MI_NO_THREAD_POINTER -static inline mi_threadid_t __mi_prim_thread_id(void) { +static inline mi_thread_id_t __mi_prim_thread_id(void) { #if defined(__BIONIC__) - return (mi_threadid_t)mi_prim_tls_slot(1); + return (mi_thread_id_t)mi_addr(mi_prim_tls_slot(1)); #else - return (mi_threadid_t)mi_prim_thread_pointer(); + return (mi_thread_id_t)mi_addr(mi_prim_thread_pointer()); #endif } #else // otherwise use portable C, taking the address of a thread local variable (this is still very fast on most platforms). extern mi_decl_hidden mi_decl_thread void* __mi_thread_id_helper; -static inline mi_threadid_t __mi_prim_thread_id(void) { - return (uintptr_t)&__mi_thread_id_helper; +static inline mi_thread_id_t __mi_prim_thread_id(void) { + return (mi_thread_id_t)mi_addr(&__mi_thread_id_helper); } #endif -static inline mi_threadid_t _mi_prim_thread_id(void) { - const mi_threadid_t tid = __mi_prim_thread_id(); - mi_assert_internal(tid > MI_THREADID_DETACHED); +static inline mi_thread_id_t _mi_prim_thread_id(void) { + const mi_thread_id_t tid = __mi_prim_thread_id(); + mi_assert_internal(tid > MI_THREAD_ID_DETACHED); mi_assert_internal((tid & MI_PAGE_FLAG_MASK) == 0); // bottom 2 bits are clear? return tid; } @@ -261,12 +266,9 @@ static inline mi_theap_t* __mi_theap_empty(void) { extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_default; // default theap to allocate from extern mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_cached; // theap from the last used heap -// defined in `heap/init.c`; do not use these directly -extern mi_decl_hidden bool _mi_process_is_initialized; // has mi_process_init been called? - static inline mi_theap_t* _mi_theap_default(void) { #if defined(MI_TLS_RECURSE_GUARD) - if mi_unlikely(!_mi_process_is_initialized) return _mi_theap_empty_get(); + if mi_unlikely(!_mi_process.process_is_initialized) return _mi_theap_empty_get(); #endif return __mi_theap_default; } diff --git a/system/lib/mimalloc/include/mimalloc/qsbr.h b/system/lib/mimalloc/include/mimalloc/qsbr.h index a91e8ad98dde6..fcce0fb21bbd2 100644 --- a/system/lib/mimalloc/include/mimalloc/qsbr.h +++ b/system/lib/mimalloc/include/mimalloc/qsbr.h @@ -24,15 +24,11 @@ terms of the MIT license. A copy of the license can be found in the file #include #ifndef MI_QSBR -#if defined(MI_CPYTHON) -#define MI_QSBR 1 -#else -#define MI_QSBR 0 -#endif +#define MI_QSBR 0 #endif // Create a qsbr enabled heap -mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new_qsbr(void); +mi_decl_externc mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new_qsbr(void); #if MI_QSBR // Define these for the host runtime to implement qsbr bookkeeping. diff --git a/system/lib/mimalloc/include/mimalloc/readme.md b/system/lib/mimalloc/include/mimalloc/readme.md index 1fa5832ef8338..bbd1e3b3a4eb9 100644 --- a/system/lib/mimalloc/include/mimalloc/readme.md +++ b/system/lib/mimalloc/include/mimalloc/readme.md @@ -15,6 +15,7 @@ and configuration options. - [`alloc`](alloc): Internal allocation, freeing, and page-management interfaces. - [`arena`](arena): Arena management, slice allocation, and arena-backed page interfaces. - [`heap`](heap): Heap, thread-local heap, subprocess, initialization, and page-map interfaces. + Also contains [`heap/process.h`](heap/process.h) which declares (almost) all static variables. - [`prim`](prim): Higher-level OS memory operations and primitive thread-local access. - [`profile`](profile): Sampled allocation, guarded allocation, and profiler interfaces. - [`util`](util): Runtime utilities, options, randomness, statistics, and dynamic thread-local interfaces. diff --git a/system/lib/mimalloc/include/mimalloc/types.h b/system/lib/mimalloc/include/mimalloc/types.h index ebdff4e876081..933e580f831e3 100644 --- a/system/lib/mimalloc/include/mimalloc/types.h +++ b/system/lib/mimalloc/include/mimalloc/types.h @@ -63,16 +63,23 @@ terms of the MIT license. A copy of the license can be found in the file // #define MI_STATS 1 // Define MI_SECURE to enable security mitigations -// #define MI_SECURE 1 // check invalid pointer free, guard pages around meta data, randomize arena allocation addresses (like ASLR), abort on detected meta data corruption -// #define MI_SECURE 2 // randomize relative allocation addresses within mimalloc pages -// #define MI_SECURE 3 // check buffer overflow, check double free, encode free lists (and detect corrupted free lists) -// #define MI_SECURE 4 // same as level 3 for now (`-DMI_SECURE=ON`) -// #define MI_SECURE 5 // guard page at the end of each mimalloc page (expensive!) (`-DMI_SECURE_FULL=ON`), and byte-precise buffer overflow checks. +// #define MI_SECURE 0 // no security mitigations +// #define MI_SECURE 1 // + guard pages around metadata (and separated metadata), randomized arena/OS allocation addresses, abort on EFAULT (corrupted meta-data) (`-DMI_SECURE=MINIMAL`) +// #define MI_SECURE 2 // + invalid-pointer check on free (checked lookup in page-map.h), clear free list `next` on allocation. +// #define MI_SECURE 3 // + randomize relative allocation addresses within mimalloc pages +// #define MI_SECURE 4 // + check buffer overflow (with padding), check double free, encode free lists (and detect corrupted free lists), decommit with mprotect (`-DMI_SECURE=ON`) +// #define MI_SECURE 5 // + guard page at the end of each mimalloc page (expensive!), and byte-precise buffer overflow checks (`-DMI_SECURE=FULL`) #if !defined(MI_SECURE) #define MI_SECURE 0 #endif +// Secure level 2 and higher always check for invalid pointers in `mi_free` +#if MI_SECURE>=2 && !MI_FREE_IS_CHECKED +#undef MI_FREE_IS_CHECKED +#define MI_FREE_IS_CHECKED 1 +#endif + // Define MI_DEBUG for assertion and invariant checking // #define MI_DEBUG 1 // basic assertion checks and statistics, check double free, corrupted free list, and invalid pointer free. (cmake -DMI_DEBUG=ON) // #define MI_DEBUG 2 // + internal assertion checks (cmake -DMI_DEBUG_INTERNAL=ON) @@ -118,7 +125,7 @@ terms of the MIT license. A copy of the license can be found in the file // Reserve extra padding at the end of each block to be more resilient against theap block overflows. // The padding can detect heap-block overflow on free, and provides byte-precise `mi_usable_size`. #if !defined(MI_PADDING) -#if (MI_SECURE>=3 || MI_DEBUG>=1 || (MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_TRACK_ETW)) +#if (MI_SECURE>=4 || MI_DEBUG>=1 || (MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_TRACK_ETW)) #define MI_PADDING 1 #else #define MI_PADDING 0 @@ -133,7 +140,7 @@ terms of the MIT license. A copy of the license can be found in the file // Encoded free lists allow detection of corrupted free lists // and can detect buffer overflows, modify after free, and double `free`s. -#if (MI_SECURE>=3 || MI_DEBUG>=1) +#if (MI_SECURE>=4 || MI_DEBUG>=1) #define MI_ENCODE_FREELIST 1 #endif @@ -159,23 +166,19 @@ terms of the MIT license. A copy of the license can be found in the file // Separate keeps the page info at the arena start (default) which is more secure // and reduces wasted space due to alignment and block sizes. #if !defined(MI_PAGE_META_IS_SEPARATED) -#if MI_PAGE_MAP_FLAT -#define MI_PAGE_META_IS_SEPARATED 0 -#else #define MI_PAGE_META_IS_SEPARATED 1 #endif -#endif // We can choose to page meta info aligned at the start of every MI_PAGE_META_ALIGNED_CHUNKS // This can be used to have a faster `mi_free(_small)` as we can avoid a page_map lookup. // This only works if valid pointers are passed to `mi_free` though. However, checked // free `mi_cfree` can still use the page map to validate pointers. -#if !MI_FREE_IS_CHECKED && !MI_FREE_USE_PAGEMAP +#if !MI_FREE_IS_CHECKED && !MI_FREE_USE_PAGEMAP && !MI_CHERI #if MI_PAGE_META_IS_SEPARATED #define MI_PAGE_META_IS_ALIGNED 1 #define MI_PAGE_META_ALIGNED_CHUNKS MI_SIZE_SIZE #else -#warning "cannot optimize free with alignment since the page meta data is not separated (due to MI_PAGE_MAP_FLAT?)" +#warning "cannot optimize free with alignment since the page meta data is not separated" #endif #endif @@ -183,9 +186,11 @@ terms of the MIT license. A copy of the license can be found in the file // This can be used to have a slightly faster `mi_free_small` function for specialized // cases (like language runtime systems). #if !defined(MI_PAGE_META_SMALL_IS_ALIGNED) -#if defined(MI_OPT_FREE_SMALL) && MI_OPT_FREE_SMALL==0 +// guarded can put small allocations into big blocks in a medium page (which would make `mi_free_small` fail) +// on CHERI we cannot derive the page from a (bounded) user pointer +#if (defined(MI_OPT_FREE_SMALL) && MI_OPT_FREE_SMALL==0) || MI_SECURE || MI_GUARDED || MI_CHERI #define MI_PAGE_META_SMALL_IS_ALIGNED 0 -#elif (MI_OPT_FREE_SMALL || MI_PAGE_META_IS_ALIGNED) && !MI_SECURE && !MI_GUARDED // guarded can put small allocations into big blocks in a medium page (which would make `mi_free_small` fail) +#elif (MI_OPT_FREE_SMALL || MI_PAGE_META_IS_ALIGNED) #define MI_PAGE_META_SMALL_IS_ALIGNED 1 #else #define MI_PAGE_META_SMALL_IS_ALIGNED 0 @@ -194,15 +199,17 @@ terms of the MIT license. A copy of the license can be found in the file // Configuration checks #if !MI_PAGE_META_IS_SEPARATED && MI_SECURE -#error "secure mode should use separated page infos" +#error "Secure mode should use separated page infos" #endif -#if MI_PAGE_META_SMALL_IS_ALIGNED && MI_SECURE -#error "secure mode cannot use MI_OPT_FREE_SMALL (MI_PAGE_META_SMALL_IS_ALIGNED)" +#if MI_PAGE_META_SMALL_IS_ALIGNED && (MI_SECURE || MI_GUARDED) +#error "Secure or guarded mode cannot use MI_OPT_FREE_SMALL (MI_PAGE_META_SMALL_IS_ALIGNED)" #endif -#if MI_PAGE_META_IS_SEPARATED && MI_PAGE_MAP_FLAT -#error "cannot have a flat page map with separated page infos" +#if (MI_PAGE_META_IS_ALIGNED || MI_PAGE_META_SMALL_IS_ALIGNED) && MI_CHERI +#error "Cannot have aligned page meta with CHERI" +#endif +#if MI_QSBR && MI_CHERI +#error "QSBR cannot be used with CHERI (as delayed concurrent reads would have the wrong capability)" #endif - // -------------------------------------------------------------- // Sizes of internal data-structures @@ -213,14 +220,11 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MI_ARENA_SLICE_SHIFT #ifdef MI_SMALL_PAGE_SHIFT // backward compatibility #define MI_ARENA_SLICE_SHIFT MI_SMALL_PAGE_SHIFT - #elif MI_SECURE>=5 && __APPLE__ && MI_ARCH_ARM64 + #elif MI_SECURE && MI_OS_PROBABLE_PAGE_SIZE >= 64*MI_KiB + #define MI_ARENA_SLICE_SHIFT (18) // 256 KiB as 64 KiB guard pages can be at most 1/4 of a slice (issue #1166) + #elif MI_SECURE>=5 && MI_OS_PROBABLE_PAGE_SIZE >= 16*MI_KiB #define MI_ARENA_SLICE_SHIFT (17) // 128 KiB to not waste too much due to 16 KiB guard pages - #elif MI_SECURE>=5 && defined(PAGE_SIZE) && !defined(__wasi__) // see issue #1440 - #if PAGE_SIZE >= 16*MI_KiB - #define MI_ARENA_SLICE_SHIFT (17) // 128 KiB to not waste too much due to 16 KiB guard pages - #endif - #endif - #ifndef MI_ARENA_SLICE_SHIFT + #else #define MI_ARENA_SLICE_SHIFT (13 + MI_SIZE_SHIFT) // 64 KiB (32 KiB on 32-bit) #endif #endif @@ -271,9 +275,26 @@ terms of the MIT license. A copy of the license can be found in the file // Minimal commit for a page on-demand commit #define MI_PAGE_MIN_COMMIT_SIZE (16*MI_KiB) /* MI_ARENA_SLICE_SIZE */ +// In secure mode, a guard slice is reserved in front of the arena info and around the separated page meta data +// of aligned areas and direct OS allocations (but not on WebAssembly as it cannot protect memory) +#if MI_SECURE && !defined(__wasi__) && !defined(__EMSCRIPTEN__) +#define MI_ARENA_GUARD_SLICES 1 +#else +#define MI_ARENA_GUARD_SLICES 0 +#endif + +// Is the page meta data of direct OS allocated pages separated in front of the page area? (between guard slices in secure mode) +// Otherwise it is placed at the start of the page area itself. +#if MI_PAGE_META_IS_ALIGNED || (MI_PAGE_META_IS_SEPARATED && MI_ARENA_GUARD_SLICES > 0) +#define MI_PAGE_META_OS_IS_SEPARATED 1 +#else +#define MI_PAGE_META_OS_IS_SEPARATED 0 +#endif + #if MI_PAGE_META_IS_ALIGNED #define MI_PAGE_META_ALIGNED_COUNT (MI_PAGE_META_ALIGNED_CHUNKS * MI_BCHUNK_BITS) #define MI_PAGE_META_ALIGNMENT (MI_PAGE_META_ALIGNED_COUNT * MI_ARENA_SLICE_SIZE) // 256 MiB (32 MiB on 32-bit) +#define MI_PAGE_META_ALIGNED_OFFSET (MI_ARENA_GUARD_SLICES * MI_ARENA_SLICE_SIZE) // offset of the page meta data in an aligned area #define MI_ARENA_ALIGNMENT MI_PAGE_META_ALIGNMENT #else #define MI_ARENA_ALIGNMENT MI_ARENA_SLICE_ALIGN @@ -317,7 +338,7 @@ typedef struct mi_subproc_s mi_subproc_t; typedef enum mi_memkind_e { MI_MEM_NONE, // not allocated (or static) MI_MEM_EXTERNAL, // not owned by mimalloc but provided externally (via `mi_manage_os_memory` for example) - MI_MEM_STATIC, // allocated in a static area and should not be freed (the initial main theap data for example (`heap/init.c`)) + MI_MEM_STATIC, // allocated in a static area and should not be freed (the initial main theap data for example (`heap/process.c`)) MI_MEM_OS, // allocated from the OS MI_MEM_OS_HUGE, // allocated as huge OS pages (usually 1GiB, pinned to physical memory) MI_MEM_OS_REMAP, // allocated in a remapable area (i.e. using `mremap`) @@ -362,9 +383,11 @@ typedef struct mi_memid_s { bool is_pinned; // `true` if we cannot decommit/reset/protect in this memory (e.g. when allocated using large (2Mib) or huge (1GiB) OS pages) bool initially_committed;// `true` if the memory was originally allocated as committed bool initially_zero; // `true` if the memory was originally zero initialized + bool guard_front : 1; // `true` if a page has a guard page in front of its blocks (see `arena-page.c:mi_theap_page_guards`) + bool guard_end : 1; // `true` if a page has a guard page right after its blocks } mi_memid_t; -#define MI_MEMID_INIT(kind) {{{NULL,0}}, kind, true /* pinned */, true /* committed */, false /* zero */ } +#define MI_MEMID_INIT(kind) {{{NULL,0}}, kind, true /* pinned */, true /* committed */, false /* zero */, false, false /* guard front and end */ } #define MI_MEMID_STATIC MI_MEMID_INIT(MI_MEM_STATIC) static inline bool mi_memid_is_os(mi_memid_t memid) { @@ -393,11 +416,11 @@ size_t _mi_memid_size(mi_memid_t memid); // ------------------------------------------------------ // The free lists use encoded next fields -// (Only actually encodes when MI_ENCODED_FREELIST is defined.) +// (Only actually encodes when MI_ENCODED_FREELIST is defined; on CHERI it only holds an address.) typedef uintptr_t mi_encoded_t; // thread id's -typedef size_t mi_threadid_t; +typedef size_t mi_thread_id_t; // free lists contain blocks typedef struct mi_block_s { @@ -414,13 +437,13 @@ typedef struct mi_block_s { #define MI_PAGE_FLAG_BITS (2) typedef size_t mi_page_flags_t; -// There are two special threadid's: 0 for pages that are abandoned (and not in a theap queue), +// There are two special thread_id's: 0 for pages that are abandoned (and not in a theap queue), // and 4 for abandoned & mapped threads -- abandoned-mapped pages are abandoned but also mapped // in an arena (in `mi_heap_t.arena_pages.pages_abandoned`) so these can be quickly found for reuse. // Abandoning partially used pages allows for sharing of this memory between threads (in particular if threads are blocked) -#define MI_THREADID_ABANDONED MI_ZU(0) -#define MI_THREADID_ABANDONED_MAPPED (MI_ZU(1) << MI_PAGE_FLAG_BITS) -#define MI_THREADID_DETACHED (MI_ZU(2) << MI_PAGE_FLAG_BITS) +#define MI_THREAD_ID_ABANDONED MI_ZU(0) +#define MI_THREAD_ID_ABANDONED_MAPPED (MI_ZU(1) << MI_PAGE_FLAG_BITS) +#define MI_THREAD_ID_DETACHED (MI_ZU(2) << MI_PAGE_FLAG_BITS) // Thread free list. // Points to a list of blocks that are freed by other threads. @@ -474,7 +497,7 @@ static inline mi_used_t mi_xused_used_reset(mi_used_t xused) { xused.used_alloc // Combining the `thread_free` list with an ownership bit allows a concurrent `free` to atomically // free an object and (re)claim ownership if the page was abandoned. // - If a page is not part of a theap it is called "abandoned" (`theap==NULL`) -- in -// that case the `xthreadid` is 0 or 4 (4 is for abandoned pages that +// that case the `xthread_id` is 0 or 4 (4 is for abandoned pages that // are in the `pages_abandoned` lists of an arena, these are called "mapped" abandoned pages). // - page flags are in the bottom 3 bits of `xthread_id` for the fast path in `mi_free`. // - The layout below is optimized for `alloc/free.c:mi_free` and `alloc/alloc.c:mi_page_alloc` -- do not change it without careful consideration. @@ -483,7 +506,7 @@ typedef struct mi_page_s { #if (MI_PAGE_META_IS_ALIGNED) _Atomic(struct mi_page_s*) self; // points to the actual page info (for pages that span multiple slices) #endif - _Atomic(mi_threadid_t) xthread_id; // thread this page belongs to. (= `theap->thread_id (or 0 or 4 if abandoned) | page_flags`) + _Atomic(mi_thread_id_t) xthread_id; // thread this page belongs to. (= `theap->thread_id (or 0 or 4 if abandoned) | page_flags`) mi_block_t* free; // list of available free blocks (`malloc` allocates from this list) mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`) size_t block_size; // const: size available in each block (always `>0`) @@ -493,7 +516,7 @@ typedef struct mi_page_s { uint16_t xlast_alloc; // for sampling; on 64-bit platforms it is in bits 48..63 of xused. #endif - size_t page_offset; // const: relative offset from the page to the start of the blocks + uint8_t* page_start; // const: start of the blocks (on CHERI, the capability from which all blocks are derived) uint16_t capacity; // number of blocks committed uint16_t reserved; // number of blocks reserved in memory uint16_t slice_pcommitted; // committed size in OS page sizes relative to the first arena slice of the page data (or 0 if the page is fully committed already) @@ -510,12 +533,12 @@ typedef struct mi_page_s { mi_memid_t memid; // const: provenance of the page memory #if (MI_ENCODE_FREELIST || MI_PADDING) - uintptr_t keys[MI_PAGE_KEY_COUNT]; // const: one or two random keys to encode the free lists (see `_mi_block_next`) or padding canary + mi_addr_t keys[MI_PAGE_KEY_COUNT]; // const: one or two random keys to encode the free lists (see `_mi_block_next`) or padding canary #endif #if MI_QSBR + _Atomic(mi_thread_id_t) qsbr_xthread_id; // thread id of the last owning thread (`theap->tld->thread_id`) | `MI_PAGE_QSBR_IN_QUEUE` (written only by the owning thread) struct mi_page_s* qsbr_next; struct mi_page_s* qsbr_prev; - bool qsbr_in_queue; mi_page_qsbr_t qsbr; #endif } mi_page_t; @@ -556,7 +579,7 @@ typedef struct mi_page_s { #error "mimalloc internal: define more bins" #endif -// static invariant: MI_MAX_SINGLETON_BIN >= _mi_bin(MI_LARGE_MAX_OBJ_SIZE) (See heap/init.c for the size bins) +// static invariant: MI_MAX_SINGLETON_BIN >= _mi_bin(MI_LARGE_MAX_OBJ_SIZE) (See heap/process.c for the size bins) #if (MI_LARGE_MAX_OBJ_WSIZE <= 8192) // 64 KiB #define MI_MAX_SINGLETON_BIN (48) #elif (MI_LARGE_MAX_OBJ_WSIZE <= 32768) // 256KiB @@ -655,8 +678,10 @@ struct mi_theap_s { long page_full_retain; // how many full pages can be retained per queue (before abandoning them) bool allow_page_reclaim; // `true` if this theap can reclaim abandoned pages bool allow_page_abandon; // `true` if this theap can abandon pages to reduce memory footprint - bool is_detached; // `true` if `tld->thread_id == MI_THREADID_DETACHED` + bool is_detached; // `true` if `tld->thread_id == MI_THREAD_ID_DETACHED` bool profile_never; // if `true`, this theap is never profiled (only the empty and meta theaps) + bool guard_front; // `true` if fresh pages get a guard page in front of the blocks (in secure mode, for meta theaps) + bool guard_end; // `true` if fresh pages get a guard page after the blocks (in secure mode, for meta theaps or always at level 5) // sampling size_t sample_countdown; // sample countdown in requested bytes (don't change the field order; see `internal.h:mi_theap_get_free_small_page`) @@ -802,8 +827,9 @@ typedef int64_t mi_msecs_t; // Thread local data struct mi_tld_s { - mi_threadid_t thread_id; // thread id of this thread + mi_thread_id_t thread_id; // thread id of this thread size_t thread_seq; // thread sequence id (linear count of created threads) + size_t fork_epoch; // fork epoch at creation; an older epoch means the thread does not exist in a forked child int numa_node; // thread preferred numa node mi_subproc_t* subproc; // sub-process this thread belongs to. mi_theap_t* theaps; // list of theaps in this thread (so we can abandon all when the thread terminates) diff --git a/system/lib/mimalloc/include/mimalloc/util/libc.h b/system/lib/mimalloc/include/mimalloc/util/libc.h index d5709db7c20da..a062d1100f900 100644 --- a/system/lib/mimalloc/include/mimalloc/util/libc.h +++ b/system/lib/mimalloc/include/mimalloc/util/libc.h @@ -37,7 +37,7 @@ static inline size_t mi_min(size_t x, size_t y) { return (x <= y ? x : y); } static inline size_t mi_max(size_t x, size_t y) { return (x >= y ? x : y); } // Is `x` a power of two? (0 is considered a power of two) -static inline bool mi_is_power_of_two(uintptr_t x) { +static inline bool mi_is_power_of_two(size_t x) { return ((x & (x - 1)) == 0); } @@ -48,13 +48,13 @@ static inline bool mi_alignment_is_valid(size_t alignment) { // Is a pointer aligned? static inline bool mi_is_aligned(const void* p, size_t alignment) { - return (alignment==0 || ((uintptr_t)p % alignment) == 0); + return (alignment==0 || (mi_addr(p) % alignment) == 0); } // Align upwards -static inline uintptr_t mi_align_up(uintptr_t sz, size_t alignment) { +static inline size_t mi_align_up(size_t sz, size_t alignment) { mi_assert_internal(alignment != 0); - const uintptr_t mask = alignment - 1; + const size_t mask = alignment - 1; if ((alignment & mask) == 0) { // power of two? return ((sz + mask) & ~mask); } @@ -63,15 +63,27 @@ static inline uintptr_t mi_align_up(uintptr_t sz, size_t alignment) { } } +// Align an address upwards +static inline mi_addr_t mi_addr_align_up(mi_addr_t addr, size_t alignment) { + mi_assert_internal(alignment != 0); + const mi_addr_t mask = alignment - 1; + if ((alignment & mask) == 0) { // power of two? + return ((addr + mask) & ~mask); + } + else { + return (((addr + mask)/alignment)*alignment); + } +} + // Align a pointer upwards static inline void* mi_align_up_ptr(const void* p, size_t alignment) { - return (void*)mi_align_up((uintptr_t)p, alignment); + return mi_ptr_with_addr(p, mi_addr_align_up(mi_addr(p), alignment)); } // Align down -static inline uintptr_t mi_align_down(uintptr_t sz, size_t alignment) { +static inline size_t mi_align_down(size_t sz, size_t alignment) { mi_assert_internal(alignment != 0); - const uintptr_t mask = alignment - 1; + const size_t mask = alignment - 1; if ((alignment & mask) == 0) { // power of two? return (sz & ~mask); } @@ -80,13 +92,25 @@ static inline uintptr_t mi_align_down(uintptr_t sz, size_t alignment) { } } +// Align an address downwards +static inline mi_addr_t mi_addr_align_down(mi_addr_t addr, size_t alignment) { + mi_assert_internal(alignment != 0); + const mi_addr_t mask = alignment - 1; + if ((alignment & mask) == 0) { // power of two? + return (addr & ~mask); + } + else { + return ((addr/alignment)*alignment); + } +} + // Align a pointer downwards static inline void* mi_align_down_ptr(const void* p, size_t alignment) { - return (void*)mi_align_down((uintptr_t)p, alignment); + return mi_ptr_with_addr(p, mi_addr_align_down(mi_addr(p), alignment)); } // Divide upwards: `s <= mi_divide_up(s,d)*d < s+d`. -static inline uintptr_t mi_divide_up(uintptr_t size, size_t divider) { +static inline size_t mi_divide_up(size_t size, size_t divider) { mi_assert_internal(divider != 0); return (divider == 0 ? size : ((size + divider - 1) / divider)); } diff --git a/system/lib/mimalloc/include/mimalloc/util/options.h b/system/lib/mimalloc/include/mimalloc/util/options.h index 4334dcd8df040..99873f1ea17e2 100644 --- a/system/lib/mimalloc/include/mimalloc/util/options.h +++ b/system/lib/mimalloc/include/mimalloc/util/options.h @@ -26,18 +26,25 @@ mi_block_t* _mi_block_next_is_corrupted(const mi_page_t* page, const mi_block_ Options (exposed for the debugger) ----------------------------------------------------------- */ typedef enum mi_option_init_e { - MI_OPTION_UNINIT, // not yet initialized + MI_OPTION_UNINIT, // not yet initialized (and the value is not yet set to the default) + MI_OPTION_PENDING, // the value is set (to the default) but not yet read from the environment MI_OPTION_DEFAULTED, // not found in the environment, use default value MI_OPTION_INITIALIZED // found in environment or set explicitly } mi_option_init_t; +// Static (read-only) option description typedef struct mi_option_desc_s { - long value; // the value - mi_option_init_t init; // is it initialized yet? (from the environment) + long default_value; // the default value mi_option_t option; // for debugging: the option index should match the option const char* name; // option name without `mimalloc_` prefix const char* legacy_name; // potential legacy option name } mi_option_desc_t; +// Current option value (stored in `_mi_process.options`) +typedef struct mi_option_state_s { + long value; // the value + mi_option_init_t init; // is it initialized yet? (from the environment) +} mi_option_state_t; + #endif diff --git a/system/lib/mimalloc/include/mimalloc/util/random.h b/system/lib/mimalloc/include/mimalloc/util/random.h index 458835accc14d..e1720c129e25f 100644 --- a/system/lib/mimalloc/include/mimalloc/util/random.h +++ b/system/lib/mimalloc/include/mimalloc/util/random.h @@ -11,11 +11,9 @@ terms of the MIT license. A copy of the license can be found in the file #include "../types.h" void _mi_random_init(mi_random_ctx_t* ctx); -void _mi_random_init_weak(mi_random_ctx_t* ctx); void _mi_random_reinit_if_weak(mi_random_ctx_t * ctx); void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* new_ctx); size_t _mi_random_next(mi_random_ctx_t* ctx); -size_t _mi_os_random_weak(size_t extra_seed); // ------------------------------------------------------------------- // Fast "random" shuffle diff --git a/system/lib/mimalloc/readme.md b/system/lib/mimalloc/readme.md index 0c423bb8acef1..214437c1e88f4 100644 --- a/system/lib/mimalloc/readme.md +++ b/system/lib/mimalloc/readme.md @@ -15,7 +15,7 @@ is a general purpose allocator with excellent [performance](#performance) charac Initially developed by Daan Leijen for the runtime systems of the [Koka](https://koka-lang.github.io) and [Lean](https://github.com/leanprover/lean) languages. -Latest release : `v3.6.0` (2026-10-23) recommended.\ +Latest release : `v3.6.1` (2026-10-30) recommended.\ Latest v2 release: `v2.5.2` (2026-09-12) stable, legacy.\ Latest v1 release: `v1.15.2` (2026-09-12) legacy. @@ -27,7 +27,7 @@ without code changes, for example, on dynamically linked ELF-based systems (Linu It also includes a way to dynamically override the default allocator in [Windows](#override_on_windows). Notable aspects of the design include: -- __small and consistent__: the library is about 10k LOC using simple and +- __small and consistent__: the library is about 12k LOC using simple and consistent data structures. This makes it very suitable to integrate and adapt in other projects. For runtime systems it provides hooks for a monotonic _heartbeat_ and deferred freeing (for @@ -90,6 +90,16 @@ New development is mostly on v3, while v1 and v2 are maintained with security an ### Releases +* 2026-10-30, `v3.6.1`: (v3) Many small fixes for CPython integration (thank you @DinoV), + Emscripten fixes (#1455, thank you @kleisauke) and reduced memory usage on WASI and Emscripten, + C23 free_sized(_aligned) overrides (#1470), improved fork support (#1456), + fix armv5tej compilation (#1464), fix MSVC overrides for strdup family (#1460), + faster STL allocator trait (#1468), + always use a 2-level pagemap for security and simplicity. + Experimental support for CHERI platforms. Improved Rust bindings (thank you @OscarTHZhang and @Techcable). + Add `-DMI_SECURE=MINIMAL` as improved [security](#secure-mode) with minimal (~1%) performance impact which + puts guard pages around all mimalloc meta-data (but does not encode free lists). + Move all static variables into a [`mi_process_t`](include/mimalloc/heap/proces.h) struct (that is guarded). * 2026-10-23, `v3.6.0`: (v3) add initial [profiling](#profiling) hooks and default profiler, "always on" detailed statistics per size-bin, and always-on profiling. Initial rust bindings. Better support for large OS page sizes, support a quiescent state based reclamation interfacec (for CPython), @@ -247,6 +257,22 @@ on Windows to build with the `clang-cl` compiler directly: > cmake ../.. -G "Visual Studio 17 2022" -T ClangCl ``` +## Emscripten + +You can build for WebAssembly with [Emscripten](https://emscripten.org) using `emcmake`. +Add `-pthread` to the compiler flags for a multi-threaded build. The tests run under `node`: + +``` +> emcmake cmake ../.. -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS=-pthread +> cmake --build . +> ctest +``` + +When linking with mimalloc, use `-sALLOW_MEMORY_GROWTH` (as mimalloc reserves its arena memory up front; this is +added automatically when linking with the CMake target or `pkg-config`), and `-pthread` if mimalloc was built with it. +Mimalloc reads `MIMALLOC_` options from the program environment; +under `node` you can pass the host environment with `-sNODE_HOST_ENV`, or otherwise set `ENV` in the JavaScript module. + ## Single Source @@ -409,20 +435,36 @@ OS will copy the entire 1GiB huge page (or 2MiB large page) which can cause the ## Secure Mode -_mimalloc_ can be build in secure mode by using the `-DMI_SECURE=ON` flags in `cmake`. This build enables various mitigations -to make mimalloc more robust against exploits. In particular: +_mimalloc_ can be build in secure mode by using the `-DMI_SECURE=OFF|MINIMAL|ON|FULL` flags in `cmake`. This build mode +enables various mitigations to make mimalloc more robust against exploits. +With `MINIMAL` only security mitigations are added that have minimal impact on performance (usually about 1 or 2%): + +- (v3) All internal mimalloc meta data (including the page map and static data) is surrounded by guard pages + (so a buffer overflow/underflow exploit cannot reach into the allocator's meta data). + However, free lists are not encoded at this level, and a heap block overflow can still corrupt a free list. +- (v3) The start address of OS allocated memory is randomized + (if possible), and mimalloc aborts on corrupted meta data detection. + +With `ON`, performance is impacted a bit more (usually less than 25%), but adds: -- All internal mimalloc page meta-data is surrounded by guard pages (so a buffer overflow exploit cannot reach into the metadata). - All free list pointers are - [encoded](https://github.com/microsoft/mimalloc/blob/783e3377f79ee82af43a0793910a9f2d01ac7863/include/mimalloc-internal.h#L396) + [encoded](include/mimalloc/alloc/page.h#L555) with per-page keys which is used both to prevent overwrites with a known pointer, as well as to detect heap corruption. - Double free's are detected (and ignored). +- Padding is added at the end of each heap block to detect heap block overflows (once the block is freed). +- Some use-after-free accesses are detected (if these modify the first word in the block). - The free lists are initialized in a random order and allocation randomly chooses between extension and reuse within a page to mitigate against attacks that rely on a predicable allocation order. Similarly, the larger heap blocks allocated by mimalloc from the OS are also address randomized. +- (v3) All pointers passed to `mi_free` (and others) are validated, and pointers not originating from a mimalloc allocation + are ignored. + +The `FULL` mode also adds: + - If using `-DMI_SECURE=FULL` there will also be guard pages at the end of each (64KiB) mimalloc page (thus interleaving - valid block data with inaccessible gaps). This setting is not recommended in general as it is more expensive and can lead to - reaching the maximum VMA limit on Linux systems if the heap gets too large. + valid block data with inaccessible gaps). This setting is not recommended in general as it is more expensive (and uses + more memory), and can also lead to reaching the maximum VMA limit on Linux systems if the heap gets too large. +- Byte-precise heap block overflow check (once the block is freed). As always, evaluate with care as part of an overall security strategy as all of the above are mitigations but not guarantees. @@ -431,7 +473,6 @@ As always, evaluate with care as part of an overall security strategy as all of When _mimalloc_ is built using debug mode, (`-DCMAKE_BUILD_TYPE=Debug`) (or when using `-DMI_DEBUG=ON` explicitly), various checks are done at runtime to catch development errors. -- Statistics are maintained in detail for each object size. They can be shown using `MIMALLOC_SHOW_STATS=1` at runtime. - All objects have padding at the end to detect (byte precise) heap block overflows. - Double free's, and freeing invalid heap pointers are detected. - Corrupted free-lists and some forms of use-after-free are detected. @@ -449,7 +490,8 @@ many system calls. Therefore, there are various environment variables (and optio - `MIMALLOC_GUARDED_SAMPLE_RATE=N`: Set the sample rate to `N` (by default 0). This mode places a guard page behind a suitable object about every `N` requested bytes (per thread). Since the performance in guarded mode without placing guard pages is close to release mode, this can be used to enable guard pages even in production to catch latent - buffer overflow bugs. Set the sample rate to `1` to guard every object, and to `0` to place no guard pages at all. + buffer overflow bugs (for example, by setting it to `16KiB` or higher where the performance impact will be minimal). + Set the sample rate to `1` to guard every object, and to `0` to place no guard pages at all. - `MIMALLOC_GUARDED_SAMPLE_SEED=N`: Start sampling at `N` (by default random). Can be used to reproduce a buffer overflow if needed. @@ -494,6 +536,12 @@ under your control or otherwise mixing of pointers from different heaps may occu Note: recently we also enabled static overloading on Windows. In that case you need to link with the static CRT _release_ runtime (`/MT`) and link with the static `mimalloc(-debug).obj` (to take precendence over the definitions in the CRT library). +The debug CRT (`/MTd`) is not supported. The static override covers the `malloc` family, +`_aligned_*`, `_expand`, `_msize`, `_strdup`, and `_wcsdup`. +Unfortunately, some other CRT functions that return +allocated memory, namely `_dupenv_s`, `_getcwd(NULL,..)`, `_fullpath(NULL,..)`, and `_tempnam`, +still allocate from the CRT heap and cannot be freed with `free`. Use the dynamic override +in that case (or the `mi_` equivalents like `mi_dupenv_s` where available). ## Dynamic override diff --git a/system/lib/mimalloc/src/alloc/alloc-aligned.c b/system/lib/mimalloc/src/alloc/alloc-aligned.c index 7895bee0851b0..2511a81752e00 100644 --- a/system/lib/mimalloc/src/alloc/alloc-aligned.c +++ b/system/lib/mimalloc/src/alloc/alloc-aligned.c @@ -64,7 +64,7 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap if mi_unlikely(alignment > MI_PAGE_MAX_OVERALLOC_ALIGN) { // use OS allocation for large alignments and allocate inside a singleton page (not in an arena) // This can support alignments >= MI_PAGE_ALIGN by ensuring the object can be aligned - // in the first (and single) page such that the page info is `MI_PAGE_ALIGN` bytes before it (and can be found in the _mi_page_map). + // in the first (and single) page such that the page info is `MI_PAGE_ALIGN` bytes before it (and can be found in the page map). if mi_unlikely(offset != 0) { // todo: cannot support offset alignment for very large alignments yet _mi_error_message(EOVERFLOW, "aligned allocation with a large alignment cannot be used with an alignment offset (size %zu, alignment %zu, offset %zu)\n", size, alignment, offset); @@ -87,11 +87,11 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap if (ppage!=NULL) { *ppage = page; } // .. and align within the allocation - const uintptr_t align_mask = alignment - 1; // for any x, `(x & align_mask) == (x % alignment)` - const uintptr_t poffset = ((uintptr_t)p + offset) & align_mask; - const uintptr_t adjust = (poffset == 0 ? 0 : alignment - poffset); + const size_t align_mask = alignment - 1; // for any x, `(x & align_mask) == (x % alignment)` + const size_t poffset = (size_t)((mi_addr(p) + offset) & align_mask); + const size_t adjust = (poffset == 0 ? 0 : alignment - poffset); mi_assert_internal(adjust < alignment); - void* aligned_p = (void*)((uintptr_t)p + adjust); + void* aligned_p = mi_ptr_with_addr(p, mi_addr(p) + adjust); // note: after the above allocation, the page may be abandoned now (as it became full, see `alloc/alloc-generic.c:_mi_malloc_generic`) // and we no longer own it. We should be careful to only read constant fields in the page, @@ -115,7 +115,7 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap // todo: expand padding if overallocated ? mi_assert_internal(mi_page_usable_block_size(page) >= adjust + size); - mi_assert_internal(((uintptr_t)aligned_p + offset) % alignment == 0); + mi_assert_internal((mi_addr(aligned_p) + offset) % alignment == 0); mi_assert_internal(mi_usable_size(aligned_p)>=size); mi_assert_internal(mi_usable_size(p) == mi_usable_size(aligned_p)+adjust); #if MI_DEBUG > 1 @@ -130,7 +130,7 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap mi_track_mem_defined(p, sizeof(mi_block_t)); #endif } - return aligned_p; + return mi_ptr_bounded(aligned_p, size); // exclude the bytes in front of the aligned pointer } // Generic primitive aligned allocation -- split out for better codegen @@ -159,14 +159,15 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t mi_page_t* page = NULL; void* p = _mi_theap_malloc_zero(theap, size, zero, 0,&page); if (ppage!=NULL) { *ppage = page; } - const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0; + const bool is_aligned_or_null = (mi_addr(p) & (alignment-1))==0; if mi_likely(is_aligned_or_null) { return p; } else { // this should never happen if the `mi_malloc_is_naturally_aligned` check is correct.. - // (but it can happen if this allocation just so happens to be sampled but we checked for that) - mi_assert_internal(false); // p==NULL || mi_block_ptr_is_profiled_or_guarded(_mi_page_ptr_unalign(page,p),p)); + // but it can happen if this allocation just so happens to be sampled anyways + // (with MI_SAMPLE==1 the countdown is updated lazily on a page refill, see issue #1457) + mi_assert_internal(p!=NULL && mi_block_ptr_is_sampled(_mi_page_ptr_unalign(page,p),p)); mi_free(p); } } @@ -210,17 +211,17 @@ static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, siz if mi_likely(!mi_theap_should_sample(theap,size)) #endif { - const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)` + const size_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)` const size_t padsize = size + MI_PADDING_SIZE; mi_page_t* page = mi_theap_get_free_small_page(theap, padsize, false); if mi_likely(page->free != NULL) { - const bool is_aligned = (((uintptr_t)page->free + offset) & align_mask)==0; + const bool is_aligned = ((mi_addr(page->free) + offset) & align_mask)==0; if mi_likely(is_aligned) { if (ppage!=NULL) { *ppage = page; } void* p = _mi_page_malloc_zero(theap, page, padsize, zero); // theap!=NULL mi_assert_internal(p != NULL); - mi_assert_internal(((uintptr_t)p + offset) % alignment == 0); + mi_assert_internal((mi_addr(p) + offset) % alignment == 0); mi_track_malloc(p, size, zero); return p; } @@ -345,8 +346,8 @@ static void* mi_theap_realloc_zero_aligned_at(mi_theap_t* theap, void* p, size_t if (alignment <= sizeof(uintptr_t) && offset==0) return _mi_theap_realloc_zero(theap,p,newsize,zero); if (p == NULL) return mi_theap_malloc_zero_aligned_at(theap,newsize,alignment,offset,zero,NULL); const size_t size = mi_usable_size(p); - if (newsize <= size && newsize >= (size - (size / 2)) && (((uintptr_t)p + offset) & (alignment-1)) == 0) { - return p; // reallocation still fits, is aligned and not more than 50% waste + if (newsize <= size && newsize >= (size - (size / 2)) && ((mi_addr(p) + offset) & (alignment-1)) == 0) { + return mi_ptr_bounded(p, newsize); // reallocation still fits, is aligned and not more than 50% waste } else { // note: we don't zero allocate upfront so we only zero initialize the expanded part (at the cost of calling mi_usable_size) diff --git a/system/lib/mimalloc/src/alloc/alloc-generic.c b/system/lib/mimalloc/src/alloc/alloc-generic.c index 52e0cd8c02d05..230dacf90931a 100644 --- a/system/lib/mimalloc/src/alloc/alloc-generic.c +++ b/system/lib/mimalloc/src/alloc/alloc-generic.c @@ -17,6 +17,8 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc/alloc/alloc.h" #include "mimalloc/alloc/page.h" #include "mimalloc/profile/sample-profile.h" +#include "mimalloc/heap/subproc.h" +#include "mimalloc/heap/process.h" #include "page-queue.h" #include "page.h" @@ -178,7 +180,7 @@ static inline mi_page_t* mi_page_queue_lookup_free_first(mi_theap_t* theap, mi_p mi_page_t* page = pq->first; if mi_likely(page!=NULL && mi_page_free_quick_collect(page)) { // fast path - #if (MI_SECURE>=2) // in secure mode, we extend half the time to increase randomness + #if (MI_SECURE>=3) // in secure mode, we extend half the time to increase randomness if (page->capacity < page->reserved && ((_mi_theap_random_next(theap) & 1) == 1)) { (void)_mi_page_extend_free(theap, page); // ok if this fails mi_assert_internal(mi_page_immediate_available(page)); @@ -220,7 +222,12 @@ static inline mi_page_t* mi_page_queue_find_free(mi_theap_t* theap, mi_page_queu // so their size is not always `> MI_LARGE_OBJ_SIZE_MAX`. static mi_page_t* mi_huge_page_alloc(mi_theap_t* theap, size_t size, size_t page_alignment, mi_page_queue_t* pq) { _mi_theap_qsbr_collect_delayed(theap); - const size_t block_size = _mi_os_good_alloc_size(size); + const size_t block_size = mi_bounds_size(_mi_os_good_alloc_size(size)); + // on CHERI, very large blocks need a large alignment for exact bounds: use an OS aligned page then (see `mi_page_meta_in_front_block_start`) + const size_t bounds_align = mi_bounds_align(block_size); + if mi_unlikely(bounds_align >= MI_PAGE_MAX_OVERALLOC_ALIGN) { + page_alignment = mi_max(page_alignment, mi_max(bounds_align, 2*MI_PAGE_MAX_OVERALLOC_ALIGN)); + } // mi_assert_internal(mi_bin(block_size) == MI_BIN_HUGE || page_alignment > 0); #if MI_HUGE_PAGE_ABANDON #error todo. @@ -283,23 +290,21 @@ static mi_page_t* mi_find_page(mi_theap_t* theap, size_t size, size_t huge_align ----------------------------------------------------------- */ // The program should only install a single deferred free handler before doing allocation. -static _Atomic(void*) deferred_free; // is `mi_deferred_free_fun*` (but some platforms don't support atomic function pointers) -static _Atomic(void*) deferred_arg; - +// The handler and argument are stored in `_mi_process.deferred_free/deferred_arg`. void _mi_deferred_free(mi_theap_t* theap, bool force) { theap->heartbeat++; - mi_deferred_free_fun* const fun = (mi_deferred_free_fun*)mi_atomic_load_ptr_acquire(void,&deferred_free); + mi_deferred_free_fun* const fun = (mi_deferred_free_fun*)mi_atomic_load_ptr_acquire(void,&_mi_process.deferred_free); if (fun != NULL && !theap->tld->recurse) { theap->tld->recurse = true; - void* const arg = mi_atomic_load_ptr_acquire(void,&deferred_arg); + void* const arg = mi_atomic_load_ptr_acquire(void,&_mi_process.deferred_arg); fun(force, theap->heartbeat, arg); theap->tld->recurse = false; } } void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noexcept { - mi_atomic_store_ptr_release(void,&deferred_arg, arg); - mi_atomic_store_ptr_release(void,&deferred_free, (void*)fn); + mi_atomic_store_ptr_release(void,&_mi_process.deferred_arg, arg); + mi_atomic_store_ptr_release(void,&_mi_process.deferred_free, (void*)fn); } @@ -308,7 +313,7 @@ void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noex ----------------------------------------------------------- */ static mi_theap_t* mi_theap_init(mi_theap_t* theap) { if mi_likely(mi_theap_is_initialized(theap)) return theap; - if (theap==&_mi_theap_empty_wrong) { + if (theap==&_mi_process.theap_empty_wrong) { // we were unable to allocate a theap for a first-class heap return NULL; } @@ -330,6 +335,9 @@ static mi_theap_t* mi_malloc_generic_admin(mi_theap_t* theap) theap->generic_collect_count += theap->generic_count; theap->generic_count = 0; + // after a fork, abandon the pages of threads that do not exist in the child (see `heap/subproc.c`) + _mi_fork_child_try_collect(); + // check if the profiler is enabled (unless this theap is permanently excluded, see `mi_theap_profile_never`) if (!theap->profile_never) { mi_heap_t* const heap = mi_theap_heap(theap); diff --git a/system/lib/mimalloc/src/alloc/alloc-override.c b/system/lib/mimalloc/src/alloc/alloc-override.c index 8069644fb09c3..305f9b53ffa5e 100644 --- a/system/lib/mimalloc/src/alloc/alloc-override.c +++ b/system/lib/mimalloc/src/alloc/alloc-override.c @@ -32,6 +32,18 @@ mi_decl_nodiscard size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept { return mi_usable_size(p); } +void mi_free_sized(void* p, size_t size) mi_attr_noexcept { + mi_free_size(p, size); +} + +void mi_free_aligned_sized(void* p, size_t alignment, size_t size) mi_attr_noexcept { + mi_free_size_aligned(p, size, alignment); // in mimalloc alignment is always after size for consistency +} + + +// ------------------------------------------------------ +// Override standard allocation functions +// ------------------------------------------------------ #if defined(MI_MALLOC_OVERRIDE) && !defined(_DLL) @@ -63,6 +75,7 @@ typedef void* mi_nothrow_t; #define MI_FORWARD3(fun,x,y,z) MI_FORWARD(fun) #define MI_FORWARD0(fun,x) MI_FORWARD(fun) #define MI_FORWARD02(fun,x,y) MI_FORWARD(fun) + #define MI_FORWARD03(fun,x,y,z) MI_FORWARD(fun) #else // otherwise use forwarding by calling our `mi_` function #define MI_FORWARD1(fun,x) { return fun(x); } @@ -70,6 +83,7 @@ typedef void* mi_nothrow_t; #define MI_FORWARD3(fun,x,y,z) { return fun(x,y,z); } #define MI_FORWARD0(fun,x) { fun(x); } #define MI_FORWARD02(fun,x,y) { fun(x,y); } + #define MI_FORWARD03(fun,x,y,z) { fun(x,y,z); } #endif #if defined(__APPLE__) && defined(MI_SHARED_LIB_EXPORT) && defined(MI_OSX_INTERPOSE) @@ -140,6 +154,10 @@ typedef void* mi_nothrow_t; }; #elif defined(_MSC_VER) + #if defined(_DEBUG) + // the debug CRT heap allocates with `HeapAlloc` directly but frees through `_free_base` (which we override) + #error "the static override is not supported with the debug CRT (/MTd); use /MT, the dynamic override (/MD), or build without MI_MALLOC_OVERRIDE (-DMI_OVERRIDE=OFF)" + #endif _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRT_HYBRIDPATCHABLE void* __cdecl _expand(_Pre_notnull_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Size) { return mi__expand(_Block, _Size); @@ -224,6 +242,14 @@ typedef void* mi_nothrow_t; void* __cdecl _aligned_recalloc(_Pre_maybenull_ _Post_invalid_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Count, _In_ _CRT_GUARDOVERFLOW size_t _Size, _In_ size_t _Alignment) { return mi_recalloc_aligned(_Block, _Count, _Size, _Alignment); } + _Check_return_ _ACRTIMP _CRTALLOCATOR + char* __cdecl _strdup(_In_z_ const char* _Str) { + return mi_strdup(_Str); + } + _Check_return_ _ACRTIMP _CRTALLOCATOR + wchar_t* __cdecl _wcsdup(_In_z_ const wchar_t* _Str) { + return mi_wcsdup(_Str); + } #else // On all other systems forward allocation primitives to our API mi_decl_export void* malloc(size_t size) MI_FORWARD1(mi_malloc, size) @@ -294,37 +320,56 @@ typedef void* mi_nothrow_t; // See // ------------------------------------------------------ - void _ZdlPv(void* p) MI_FORWARD0(mi_free,p) // delete - void _ZdaPv(void* p) MI_FORWARD0(mi_free,p) // delete[] - void _ZdlPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n) - void _ZdaPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n) + void _ZdlPv(void* p) MI_FORWARD0(mi_free,p) // delete + void _ZdaPv(void* p) MI_FORWARD0(mi_free,p) // delete[] - void _ZdlPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); } - void _ZdaPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); } - void _ZdlPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); } - void _ZdaPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); } + void _ZdlPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); } + void _ZdaPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); } - void _ZdlPvRKSt9nothrow_t(void* p, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free(p); } // operator delete(void*, std::nothrow_t const&) - void _ZdaPvRKSt9nothrow_t(void* p, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free(p); } // operator delete[](void*, std::nothrow_t const&) + void _ZdlPvRKSt9nothrow_t(void* p, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free(p); } // operator delete(void*, std::nothrow_t const&) + void _ZdaPvRKSt9nothrow_t(void* p, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free(p); } // operator delete[](void*, std::nothrow_t const&) void _ZdlPvSt11align_val_tRKSt9nothrow_t(void* p, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free_aligned(p,al); } // operator delete(void*, std::align_val_t, std::nothrow_t const&) void _ZdaPvSt11align_val_tRKSt9nothrow_t(void* p, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free_aligned(p,al); } // operator delete[](void*, std::align_val_t, std::nothrow_t const&) - #if (MI_SIZE_SIZE==8) || (MI_SIZE_SIZE==4 && defined(__EMSCRIPTEN__)) // pr #1257 - void* _Znwm(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit - void* _Znam(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit - void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } - void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } - void* _ZnwmSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) - void* _ZnamSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) + // the mangling of `size_t` depends on its underlying type: `y` for `unsigned long long` (64-bit MinGW), + // `m` for `unsigned long` (LP64, and also 32-bit on Apple (arm64_32) and wasm (pr #1257)), and `j` for `unsigned int`. + #if defined(_WIN64) && (MI_SIZE_SIZE==8) + void _ZdlPvy(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n) + void _ZdaPvy(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n) + void _ZdlPvySt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); } + void _ZdaPvySt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); } + void* _Znwy(size_t n) MI_FORWARD1(mi_new,n) // new + void* _Znay(size_t n) MI_FORWARD1(mi_new,n) // new[] + void* _ZnwyRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } + void* _ZnayRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } + void* _ZnwySt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) + void* _ZnaySt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) + void* _ZnwySt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } + void* _ZnaySt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } + #elif (MI_SIZE_SIZE==8) || (MI_SIZE_SIZE==4 && (defined(__APPLE__) || defined(__EMSCRIPTEN__) || defined(__wasm__))) + void _ZdlPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n) + void _ZdaPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n) + void _ZdlPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); } + void _ZdaPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); } + void* _Znwm(size_t n) MI_FORWARD1(mi_new,n) // new + void* _Znam(size_t n) MI_FORWARD1(mi_new,n) // new[] + void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } + void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } + void* _ZnwmSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) + void* _ZnamSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } #elif (MI_SIZE_SIZE==4) - void* _Znwj(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit - void* _Znaj(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit - void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } - void* _ZnajRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } - void* _ZnwjSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) - void* _ZnajSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) + void _ZdlPvj(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n) + void _ZdaPvj(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n) + void _ZdlPvjSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); } + void _ZdaPvjSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); } + void* _Znwj(size_t n) MI_FORWARD1(mi_new,n) // new + void* _Znaj(size_t n) MI_FORWARD1(mi_new,n) // new[] + void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } + void* _ZnajRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } + void* _ZnwjSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) + void* _ZnajSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) void* _ZnwjSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } void* _ZnajSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } #else @@ -332,6 +377,7 @@ typedef void* mi_nothrow_t; #endif #endif // __cplusplus + // ------------------------------------------------------ // Further Posix & Unix functions definitions // ------------------------------------------------------ @@ -384,7 +430,11 @@ void* reallocarray(void* p, size_t count, size_t size) { return mi_reallocarray // some systems define reallocarr so mark it as a weak symbol (#751) mi_decl_weak int reallocarr(void* p, size_t count, size_t size) { return mi_reallocarr(p, count, size); } -#if defined(__wasi__) +// C23 +void free_sized(void* p, size_t size) MI_FORWARD02(mi_free_sized,p,size) +void free_aligned_sized(void* p, size_t alignment, size_t size) MI_FORWARD03(mi_free_aligned_sized,p,alignment,size) + +#if defined(__wasi__) || defined(__EMSCRIPTEN__) // forward __libc interface (see PR #667) void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc, size) void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc, count, size) diff --git a/system/lib/mimalloc/src/alloc/alloc-posix.c b/system/lib/mimalloc/src/alloc/alloc-posix.c index 4ba1ccb0a10a1..0b43df8cb0ce6 100644 --- a/system/lib/mimalloc/src/alloc/alloc-posix.c +++ b/system/lib/mimalloc/src/alloc/alloc-posix.c @@ -82,6 +82,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size return p; } + mi_decl_nodiscard void* mi_reallocarray( void* p, size_t count, size_t size ) mi_attr_noexcept { // BSD size_t total; if mi_unlikely(mi_count_size_overflow(count, size, &total)) { diff --git a/system/lib/mimalloc/src/alloc/alloc.c b/system/lib/mimalloc/src/alloc/alloc.c index f3ea5e08eb484..1be26dc866679 100644 --- a/system/lib/mimalloc/src/alloc/alloc.c +++ b/system/lib/mimalloc/src/alloc/alloc.c @@ -105,7 +105,7 @@ static mi_decl_forceinline mi_decl_restrict void* mi_page_malloc_zero(mi_theap_t mi_block_t* next = mi_block_next(page,block); mi_track_mem_undefined(block,sizeof(*block)); - #if MI_SECURE + #if MI_SECURE>=2 if (!zero) block->next = 0; // don't leak internal data #endif @@ -154,9 +154,10 @@ static mi_decl_forceinline mi_decl_restrict void* mi_page_malloc_zero(mi_theap_t if (!page->free_is_zero) { mi_assert_internal(bsize%MI_SIZE_SIZE == 0); #if !MI_PADDING - return // use tail-call - #endif + return mi_ptr_bounded_data(mi_memzero_block(block,bsize), size); // use tail-call + #else mi_memzero_block(block,bsize); + #endif } else { block->next = 0; @@ -168,8 +169,7 @@ static mi_decl_forceinline mi_decl_restrict void* mi_page_malloc_zero(mi_theap_t #if MI_PADDING // && !MI_TRACK_ENABLED mi_page_block_setup_padding(page,block,size); #endif - - return block; + return mi_ptr_bounded_data(block, size - MI_PADDING_SIZE); // bounded to the requested size (see `mi_usable_size`) } // extra entries for improved efficiency in `alloc/alloc-aligned.c` (and in `alloc/alloc-generic.c:mi_malloc_generic`. @@ -490,7 +490,7 @@ void* mi_expand(void* p, size_t newsize) mi_attr_noexcept { const mi_page_t* const page = mi_ptr_page_validate(p,"mi_expand"); const size_t size = _mi_page_usable_size(page,p); if (newsize > size) return NULL; - return p; // it fits + return mi_ptr_bounded(p, newsize); // it fits #endif } @@ -528,7 +528,7 @@ static mi_decl_forceinline void* mi_theap_realloc_zero_ex(mi_theap_t* theap, voi // mi_track_resize(p,size,newsize) // if (newsize < size) { mi_track_mem_noaccess((uint8_t*)p + newsize, size - newsize); } if (pblock_size_post!=NULL) { *pblock_size_post = mi_page_block_size(page); } - return p; // reallocation still fits and not more than 50% waste + return mi_ptr_bounded(p, newsize); // reallocation still fits and not more than 50% waste } } } @@ -544,9 +544,11 @@ static mi_decl_forceinline void* mi_theap_realloc_zero_ex(mi_theap_t* theap, voi if (zero && usable > zero_start) { mi_memzero_aligned((uint8_t*)newp + zero_start, usable - zero_start); } + #if !MI_CHERI // on CHERI zero-sized blocks have zero bounds else if (newsize == 0) { ((uint8_t*)newp)[0] = 0; // work around for applications that expect zero-reallocation to be zero initialized (issue #725) } + #endif if mi_likely(p != NULL) { mi_track_mem_defined(p,copy_size); // _mi_useable_size may be too large for byte precise memory tracking.. mi_memcpy_aligned(newp, p, copy_size); diff --git a/system/lib/mimalloc/src/alloc/free.c b/system/lib/mimalloc/src/alloc/free.c index 4d6f2b4e4a679..77cfa7e21a790 100644 --- a/system/lib/mimalloc/src/alloc/free.c +++ b/system/lib/mimalloc/src/alloc/free.c @@ -134,7 +134,7 @@ static inline mi_block_t* mi_page_ptr_unalign_ex(const mi_page_t* page, const vo adjust = diff % block_size; } if (poffset!=NULL) { *poffset = adjust; } - return (mi_block_t*)((uintptr_t)p - adjust); + return mi_page_block_from_addr(page, mi_addr(p) - adjust); // don't keep the user's capability } mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) { @@ -143,12 +143,11 @@ mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) { static inline mi_block_t* mi_validate_block_from_ptr( const mi_page_t* page, const void* p ) { mi_assert(_mi_page_ptr_unalign(page,p) == (mi_block_t*)p); // should never be an interior pointer - #if MI_SECURE > 0 + #if MI_SECURE>=2 // in secure mode we always unalign to guard against free-ing interior pointers return _mi_page_ptr_unalign(page,p); #else - MI_UNUSED(page); - return (mi_block_t*)p; + return mi_page_block_from_addr(page, mi_addr(p)); // don't keep the user's capability #endif } @@ -226,9 +225,9 @@ static mi_decl_forceinline bool mi_ptr_page_is_valid_ex(const void* p, const cha mi_page_t* page; #if MI_PAGE_META_SMALL_IS_ALIGNED if (free_small) { - const uintptr_t up = mi_align_down((uintptr_t)p,MI_SMALL_PAGE_SIZE); // like this for codegen on gcc + const mi_addr_t up = mi_addr(p) & ~MI_AU(MI_SMALL_PAGE_SIZE-1); // like this for codegen on gcc if mi_unlikely(up==0 && check_p_for_null) { return false; } - page = (mi_page_t*)up; + page = (mi_page_t*)mi_ptr_with_addr(p, up); } else #endif @@ -238,8 +237,8 @@ static mi_decl_forceinline bool mi_ptr_page_is_valid_ex(const void* p, const cha #else { page = mi_ptr_page(p); } #endif - if mi_unlikely(page==NULL && check_p_for_null) { - #if MI_DEBUG + if mi_unlikely(mi_ptr_page_is_null(p,page) && check_p_for_null) { + #if MI_DEBUG || MI_CHERI // on CHERI also report in release (e.g. untagged pointers) if (p!=NULL) { _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); } #endif return false; @@ -295,8 +294,8 @@ static mi_decl_forceinline void mi_free_nonnull(void* p, mi_page_t* page, size_t mi_assert_internal(p!=NULL && page!=NULL); if (pblock_size!=NULL) { *pblock_size = mi_page_block_size(page); } - const mi_threadid_t ptid = mi_page_xthread_id(page); - const mi_threadid_t xtid = (_mi_prim_thread_id() ^ ptid); + const mi_thread_id_t ptid = mi_page_xthread_id(page); + const mi_thread_id_t xtid = (_mi_prim_thread_id() ^ ptid); if mi_likely(xtid == 0) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) == 0` // thread-local, aligned, and not a full page mi_block_t* const block = mi_validate_block_from_ptr(page,p); @@ -429,7 +428,7 @@ void mi_free_size(void* p, size_t size) mi_attr_noexcept { } void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept { - mi_assert(((uintptr_t)p % alignment) == 0); + mi_assert((mi_addr(p) % alignment) == 0); // If the alignment is smaller than the `size`, then for `size <= MI_SMALL_SIZE_MAX` // the block must be allocated within a small page and can thus be handled safely by `mi_free_size`. // (since even with over-allocation the block size will be less than 2*MI_SMALL_SIZE_MAX <= MI_SMALL_OBJ_SIZE_MAX) @@ -443,7 +442,7 @@ void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexce void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept { MI_UNUSED_RELEASE(alignment); - mi_assert(((uintptr_t)p % alignment) == 0); + mi_assert((mi_addr(p) % alignment) == 0); mi_free(p); } @@ -656,11 +655,11 @@ size_t _mi_page_usable_size(const mi_page_t* page, const void* p) mi_attr_noexce mi_assert_internal(mi_ptr_page_validate(p,"_mi_page_usable_size") == page); const mi_block_t* block = _mi_page_ptr_unalign(page, p); // for safety, always unalign if mi_likely((const void*)block==page) { - return mi_page_usable_size_of(page, block, false /* is guarded */); + return mi_ptr_bounds_limit(p, mi_page_usable_size_of(page, block, false /* is guarded */)); } else { // split out to separate routine for improved code generation - return mi_page_usable_aligned_size_of(page, p, block); + return mi_ptr_bounds_limit(p, mi_page_usable_aligned_size_of(page, p, block)); } } @@ -670,10 +669,16 @@ mi_decl_nodiscard size_t mi_usable_size(const void* p) mi_attr_noexcept { } mi_decl_nodiscard size_t mi_thread_id_of(const void* p) mi_attr_noexcept { - if (p == NULL) return 0; - const mi_page_t* const page = mi_ptr_page(p); - const mi_threadid_t tid = mi_page_thread_id(page); - return (tid <= MI_THREADID_DETACHED ? 0 : tid); + const mi_page_t* const page = mi_ptr_page_validate(p,"mi_thread_id_of"); + if (page == NULL) return 0; + #if MI_QSBR + // on qsbr we keep the last owning thread id around so we can provide it even when the page is abandoned + // (this is needed for cpython where the thread id gets restored this way inside object headers) + return mi_page_qsbr_thread_id(page); + #else + const mi_thread_id_t tid = mi_page_thread_id(page); + return (tid <= MI_THREAD_ID_DETACHED ? 0 : tid); + #endif } @@ -684,7 +689,7 @@ mi_decl_nodiscard size_t mi_thread_id_of(const void* p) mi_attr_noexcept { // This is somewhat expensive so only enabled for secure mode 4 // ------------------------------------------------------ -#if MI_SECURE>=3 && !MI_PADDING +#if MI_SECURE>=4 && !MI_PADDING // linear check if the free list contains a specific element static bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, const mi_block_t* elem, const char* list_kind) { const size_t max_count = page->capacity; // can never hold more blocks than the capacity diff --git a/system/lib/mimalloc/src/alloc/page-queue.c b/system/lib/mimalloc/src/alloc/page-queue.c index a9d8c42e3007b..986a48e7e10af 100644 --- a/system/lib/mimalloc/src/alloc/page-queue.c +++ b/system/lib/mimalloc/src/alloc/page-queue.c @@ -14,7 +14,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "page-queue.h" /* ----------------------------------------------------------- - Minimal alignment in machine words (i.e. `sizeof(void*)`) + Minimal alignment in machine words (i.e. `sizeof(size_t)`) ----------------------------------------------------------- */ #if (MI_MAX_ALIGN_SIZE > 4*MI_SIZE_SIZE) @@ -27,6 +27,16 @@ terms of the MIT license. A copy of the license can be found in the file // ok, default alignment is 1 word #endif +// Minimal block size in machine words (a block must be able to hold a pointer) +#if (MI_INTPTR_SIZE > MI_SIZE_SIZE) + #if (MI_INTPTR_SIZE > 2*MI_SIZE_SIZE) || !(defined(MI_ALIGN2W) || defined(MI_ALIGN4W)) + #error "define the minimal block size for this platform" + #endif + #define MI_MIN_BLOCK_WSIZE 2 // as on CHERI +#else + #define MI_MIN_BLOCK_WSIZE 1 +#endif + /* ----------------------------------------------------------- Bins @@ -35,22 +45,22 @@ terms of the MIT license. A copy of the license can be found in the file // Return the bin for a given field size. // Returns MI_BIN_HUGE if the size is too large. // We use `wsize` for the size in "machine word sizes", -// i.e. byte size == `wsize*sizeof(void*)`. +// i.e. byte size == `wsize*sizeof(size_t)`. static size_t mi_bin(size_t size) { size_t wsize = mi_wsize_from_size(size); -#if defined(MI_ALIGN4W) + #if defined(MI_ALIGN4W) if mi_likely(wsize <= 4) { - return (wsize <= 1 ? 1 : (wsize+1)&~1); // round to double word sizes + return (wsize <= MI_MIN_BLOCK_WSIZE ? MI_MIN_BLOCK_WSIZE : (wsize+1)&~1); // round to double word sizes } -#elif defined(MI_ALIGN2W) + #elif defined(MI_ALIGN2W) if mi_likely(wsize <= 8) { - return (wsize <= 1 ? 1 : (wsize+1)&~1); // round to double word sizes + return (wsize <= MI_MIN_BLOCK_WSIZE ? MI_MIN_BLOCK_WSIZE : (wsize+1)&~1); // round to double word sizes } -#else + #else if mi_likely(wsize <= 8) { return (wsize == 0 ? 1 : wsize); } -#endif + #endif else if mi_unlikely(wsize > MI_LARGE_MAX_OBJ_WSIZE) { return MI_BIN_HUGE; } @@ -183,12 +193,12 @@ static inline void mi_theap_queue_first_update(mi_theap_t* theap, const mi_page_ // find start slot size_t start; - if (idx<=1) { + if (idx <= MI_MIN_BLOCK_WSIZE) { start = 0; } else { // find previous size; due to minimal alignment upto 3 previous bins may need to be skipped - mi_assert_internal(pq > &theap->pages[0]); // since idx > 1 + mi_assert_internal(pq > &theap->pages[0]); // since idx > MI_MIN_BLOCK_WSIZE size_t bin = mi_bin(size); const mi_page_queue_t* prev = pq - 1; while( bin == mi_bin(prev->block_size) && prev > &theap->pages[0]) { diff --git a/system/lib/mimalloc/src/alloc/page-queue.h b/system/lib/mimalloc/src/alloc/page-queue.h index 3a7096d3b6853..906c8cd41632e 100644 --- a/system/lib/mimalloc/src/alloc/page-queue.h +++ b/system/lib/mimalloc/src/alloc/page-queue.h @@ -32,11 +32,11 @@ void mi_page_queue_move_to_back(mi_theap_t* theap, mi_page_queue_t* queue, mi_pa static inline bool mi_page_queue_is_huge(const mi_page_queue_t* pq) { - return (pq->block_size == (MI_LARGE_MAX_OBJ_SIZE+sizeof(uintptr_t))); + return (pq->block_size == (MI_LARGE_MAX_OBJ_SIZE+sizeof(size_t))); } static inline bool mi_page_queue_is_full(const mi_page_queue_t* pq) { - return (pq->block_size == (MI_LARGE_MAX_OBJ_SIZE+(2*sizeof(uintptr_t)))); + return (pq->block_size == (MI_LARGE_MAX_OBJ_SIZE+(2*sizeof(size_t)))); } static inline bool mi_page_queue_is_special(const mi_page_queue_t* pq) { diff --git a/system/lib/mimalloc/src/alloc/page.c b/system/lib/mimalloc/src/alloc/page.c index 741725707eab6..90809d2782552 100644 --- a/system/lib/mimalloc/src/alloc/page.c +++ b/system/lib/mimalloc/src/alloc/page.c @@ -77,7 +77,7 @@ static bool mi_page_is_valid_init(mi_page_t* page) { mi_assert_internal(page->heap!=NULL); mi_theap_t* const page_theap = _mi_heap_theap_peek(page->heap); mi_assert_internal(page_theap == NULL || mi_page_theap(page)==page_theap || - mi_page_theap(page)->tld == NULL || mi_page_theap(page)->tld->thread_id == MI_THREADID_DETACHED); + mi_page_theap(page)->tld == NULL || mi_page_theap(page)->tld->thread_id == MI_THREAD_ID_DETACHED); // const size_t bsize = mi_page_block_size(page); // uint8_t* start = mi_page_start(page); @@ -109,19 +109,17 @@ static bool mi_page_is_valid_init(mi_page_t* page) { return true; } -extern mi_decl_hidden bool _mi_process_is_initialized; // has mi_process_init been called? - bool _mi_page_is_valid(mi_page_t* page) { mi_assert_internal(mi_page_is_valid_init(page)); #if MI_SECURE mi_assert_internal(page->keys[0] != 0); #endif if (!mi_page_is_abandoned(page)) { - //mi_assert_internal(!_mi_process_is_initialized); + //mi_assert_internal(!_mi_process.process_is_initialized); mi_assert_internal(page->heap!=NULL); mi_theap_t* const page_theap = _mi_heap_theap_peek(page->heap); mi_assert_internal(page_theap == NULL || mi_page_theap(page)==page_theap || - mi_page_theap(page)->tld == NULL || mi_page_theap(page)->tld->thread_id == MI_THREADID_DETACHED); + mi_page_theap(page)->tld == NULL || mi_page_theap(page)->tld->thread_id == MI_THREAD_ID_DETACHED); { mi_page_queue_t* pq = mi_page_queue_of(page); mi_assert_internal(mi_page_queue_contains(pq, page)); @@ -289,7 +287,7 @@ void _mi_page_update_stats(mi_page_t* page) { void _mi_page_update_sample_countdown(mi_page_t* page) { mi_assert_internal(mi_page_is_owned(page)); - // note: the threadid may not match and even be abandoned if we free multi-threaded (so we can not always use page->theap) + // note: the thread_id may not match and even be abandoned if we free multi-threaded (so we can not always use page->theap) // adjust count down const size_t alloc_count = mi_page_alloc_count(page); @@ -736,7 +734,7 @@ void _mi_theap_collect_retired(mi_theap_t* theap, bool force) { #define MI_MIN_SLICES (2) static void mi_page_free_list_extend_secure(mi_theap_t* const theap, mi_page_t* const page, const size_t bsize, const size_t extend) { - #if (MI_SECURE < 2) + #if !(MI_SECURE>=3) mi_assert_internal(page->free == NULL); mi_assert_internal(page->local_free == NULL); #endif @@ -793,7 +791,7 @@ static void mi_page_free_list_extend_secure(mi_theap_t* const theap, mi_page_t* static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, const size_t bsize, const size_t extend) { - #if (MI_SECURE < 2) + #if !(MI_SECURE>=3) mi_assert_internal(page->free == NULL); mi_assert_internal(page->local_free == NULL); #endif @@ -821,7 +819,7 @@ static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, co ----------------------------------------------------------- */ #define MI_MAX_EXTEND_SIZE (8*1024) // heuristic, one or two OS pages seems to work well. -#if (MI_SECURE>=2) +#if (MI_SECURE>=3) #define MI_MIN_EXTEND (8*MI_SECURE) // extend at least by this many #else #define MI_MIN_EXTEND (1) @@ -834,7 +832,7 @@ static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, co // extra test in malloc? or cache effects?) bool _mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) { mi_assert_expensive(mi_page_is_valid_init(page)); - #if (MI_SECURE < 2) + #if !(MI_SECURE>=3) mi_assert(page->free == NULL); mi_assert(page->local_free == NULL); if (page->free != NULL) return true; @@ -875,13 +873,10 @@ bool _mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) { const size_t needed_size = (page->capacity + extend)*bsize; mi_assert_internal(needed_size <= page_size); size_t needed_commit = mi_align_up( mi_page_slice_offset_of(page, needed_size), mi_page_min_commit_size()); - #if MI_SECURE>=5 - // the previous alignup could extend the commit into the guard page; re-adjust if needed - const size_t page_size_commit = mi_align_up( mi_page_slice_offset_of(page, page_size), _mi_os_page_size() ); - if (needed_commit > page_size_commit) { - needed_commit = page_size_commit; + if (page->memid.guard_end && needed_commit > mi_page_guard_end_offset(page)) { + // the previous alignup could extend the commit into the guard page; re-adjust if needed + needed_commit = mi_page_guard_end_offset(page); } - #endif if (needed_commit > slice_committed) { mi_assert_internal(((needed_commit - slice_committed) % _mi_os_page_size()) == 0); if (!_mi_os_commit(mi_theap_subproc(theap), mi_page_slice_start(page) + slice_committed, needed_commit - slice_committed, NULL)) { @@ -893,11 +888,11 @@ bool _mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) { } // and append the extend the free list - if (extend < MI_MIN_SLICES || MI_SECURE<2) { //!mi_option_is_enabled(mi_option_secure)) { - mi_page_free_list_extend(page, bsize, extend ); + if (MI_SECURE>=3 && extend >= MI_MIN_SLICES) { + mi_page_free_list_extend_secure(theap, page, bsize, extend); } else { - mi_page_free_list_extend_secure(theap, page, bsize, extend); + mi_page_free_list_extend(page, bsize, extend ); } // enable the new free list page->capacity += (uint16_t)extend; diff --git a/system/lib/mimalloc/src/arena/arena-alloc.c b/system/lib/mimalloc/src/arena/arena-alloc.c index 6b652c4cd4b4a..004eedcec03bc 100644 --- a/system/lib/mimalloc/src/arena/arena-alloc.c +++ b/system/lib/mimalloc/src/arena/arena-alloc.c @@ -516,8 +516,7 @@ static void mi_arenas_try_purge(bool force, bool visit_all, mi_subproc_t* subpro if (max_arena == 0) return; // allow only one thread to purge at a time (todo: allow concurrent purging?) - static mi_atomic_guard_t purge_guard; - mi_atomic_guard(&purge_guard) + mi_atomic_guard(&_mi_process.arena_purge_guard) { // increase global expire: at most one purge per delay cycle if (arenas_expire > now) { mi_atomic_storei64_release(&subproc->purge_expire, now + (delay/10)); } diff --git a/system/lib/mimalloc/src/arena/arena-page.c b/system/lib/mimalloc/src/arena/arena-page.c index 8db26247b9544..c8d397c254b85 100644 --- a/system/lib/mimalloc/src/arena/arena-page.c +++ b/system/lib/mimalloc/src/arena/arena-page.c @@ -29,6 +29,11 @@ static bool mi_heap_has_page(mi_heap_t* heap, mi_arena_t* arena, mi_page_t* page } #endif +// Restrict a pointer to page meta data to the page meta data only +static mi_page_t* mi_page_meta_bounded(void* p) { + return (mi_page_t*)mi_ptr_bounded_data(p, sizeof(mi_page_t)); +} + mi_page_t* mi_arena_page_at_slice(mi_arena_t* arena, size_t slice_index) { mi_assert_internal(slice_index < arena->slice_count); #if MI_PAGE_META_IS_ALIGNED @@ -39,13 +44,13 @@ mi_page_t* mi_arena_page_at_slice(mi_arena_t* arena, size_t slice_index) { mi_page_t* const page = &arena->pages_meta[slice_index]; #if MI_PAGE_META_SMALL_IS_ALIGNED // pages with small blocks still have the page at the start of the slice (and set the `block_size` in pages_meta to 0) - if (page->block_size>0) return page; + if (page->block_size>0) return mi_page_meta_bounded(page); #else - return page; + return mi_page_meta_bounded(page); #endif } // fall through (for MI_PAGE_META_SMALL_IS_ALIGNED) - return (mi_page_t*)mi_arena_slice_start(arena,slice_index); + return mi_page_meta_bounded(mi_arena_slice_start(arena,slice_index)); #endif } @@ -54,8 +59,10 @@ static size_t mi_page_full_size(mi_page_t* page) { return page->memid.mem.arena.slice_count * MI_ARENA_SLICE_SIZE; } else if (mi_memid_is_os(page->memid) || page->memid.memkind == MI_MEM_EXTERNAL) { - mi_assert_internal((uint8_t*)page->memid.mem.os.base <= (uint8_t*)page); - const ptrdiff_t presize = (uint8_t*)page - (uint8_t*)page->memid.mem.os.base; + // note: the page meta data may be in front of the slices (see `mi_arenas_page_alloc_from_os`) + uint8_t* const slice_start = mi_page_slice_start(page); + mi_assert_internal((uint8_t*)page->memid.mem.os.base <= slice_start); + const ptrdiff_t presize = slice_start - (uint8_t*)page->memid.mem.os.base; mi_assert_internal((ptrdiff_t)page->memid.mem.os.size >= presize); return (presize > (ptrdiff_t)page->memid.mem.os.size ? 0 : page->memid.mem.os.size - presize); } @@ -88,12 +95,16 @@ static size_t mi_arena_pages_size(size_t slice_count, size_t* bitmap_base) { return size; } -// allocate initial arena_pages from the main heap +// allocate initial arena_pages from the main heap (or as meta data in secure mode) static mi_arena_pages_t* mi_arena_pages_alloc(mi_arena_t* arena) { const size_t slice_count = arena->slice_count; size_t bitmap_base = 0; const size_t size = mi_arena_pages_size(slice_count, &bitmap_base); + #if MI_SECURE + mi_arena_pages_t* arena_pages = (mi_arena_pages_t*)_mi_meta_zalloc_aligned(arena->subproc, size, MI_BCHUNK_SIZE, NULL); + #else mi_arena_pages_t* arena_pages = (mi_arena_pages_t*)mi_heap_zalloc_aligned(arena->subproc->heap_main, size, MI_BCHUNK_SIZE); + #endif if (arena_pages==NULL) return NULL; uint8_t* base = (uint8_t*)arena_pages + bitmap_base; mi_assert_internal(mi_is_aligned(base, MI_BCHUNK_SIZE)); @@ -146,13 +157,51 @@ static mi_arena_pages_t* mi_heap_ensure_arena_pages(mi_heap_t* heap, mi_arena_t* } +/* ----------------------------------------------------------- + Guard pages in front of and right after the blocks of a page. + These are used in secure mode for the pages of meta theaps, + and at the end of all pages at secure level 5. + The `memid.guard_front/end` flags record the guard pages of a page. +----------------------------------------------------------- */ + +typedef struct mi_page_guards_s { + size_t front; // size of the guard page in front of the blocks (or 0) + size_t end; // size of the guard page right after the blocks (or 0) +} mi_page_guards_t; + +// The guard pages a theap uses for a fresh page of `size` bytes. To limit the overhead, we only use guard pages +// if together they take at most 1/4 of the page. OS aligned pages have no front guard page. +static mi_page_guards_t mi_theap_page_guards(const mi_theap_t* theap, size_t size, bool os_align) { + const size_t gsize = _mi_os_secure_guard_page_size(); + mi_page_guards_t guards = { 0, 0 }; + if (theap->guard_front && !os_align && gsize <= size/4) { guards.front = gsize; } + if (theap->guard_end && guards.front + gsize <= size/4) { guards.end = gsize; } + return guards; +} + +static bool mi_page_has_guards(const mi_page_t* page, mi_page_guards_t guards) { + return (page->memid.guard_front == (guards.front > 0) && page->memid.guard_end == (guards.end > 0)); +} + + /* ----------------------------------------------------------- Claim abandoned pages ----------------------------------------------------------- */ -static bool mi_arena_try_claim_abandoned(size_t slice_index, mi_arena_t* arena, bool* keep_abandoned) { +typedef struct mi_arena_claim_info_s { + mi_arena_t* arena; + mi_page_guards_t guards; +} mi_arena_claim_info_t; + +static bool mi_arena_try_claim_abandoned(size_t slice_index, void* arg, bool* keep_abandoned) { + const mi_arena_claim_info_t* const info = (const mi_arena_claim_info_t*)arg; // found an abandoned page of the right size - mi_page_t* const page = mi_arena_page_at_slice(arena, slice_index); + mi_page_t* const page = mi_arena_page_at_slice(info->arena, slice_index); + // only reclaim pages with the same guard pages as a fresh page (so meta data and user pages are never mixed) + if (!mi_page_has_guards(page, info->guards)) { + *keep_abandoned = true; + return false; + } // can we claim ownership? if (!mi_page_claim_ownership(page)) { // there was a concurrent free that reclaims this page .. @@ -175,6 +224,7 @@ typedef struct mi_arena_abandoned_visit_info_s { size_t slice_count; size_t block_size; size_t bin; + mi_page_guards_t guards; } mi_arena_abandoned_visit_info_t; static bool mi_arena_try_find_abandoned_visit(mi_arena_t* arena, const void* arg, void** result) { @@ -187,8 +237,8 @@ static bool mi_arena_try_find_abandoned_visit(mi_arena_t* arena, const void* arg if (arena_pages != NULL) { size_t slice_index; mi_bitmap_t* const bitmap = arena_pages->pages_abandoned[info->bin]; - - if (mi_bitmap_try_find_and_claim(bitmap, tseq, &slice_index, &mi_arena_try_claim_abandoned, arena)) { + mi_arena_claim_info_t claim_info = { arena, info->guards }; + if (mi_bitmap_try_find_and_claim(bitmap, tseq, &slice_index, &mi_arena_try_claim_abandoned, &claim_info)) { // found an abandoned page of the right size // and claimed ownership. mi_page_t* page = mi_arena_page_at_slice(arena, slice_index); @@ -214,7 +264,7 @@ static bool mi_arena_try_find_abandoned_visit(mi_arena_t* arena, const void* arg return true; // continue } -static mi_page_t* mi_arenas_page_try_find_abandoned(mi_theap_t* theap, size_t slice_count, size_t block_size) +static mi_page_t* mi_arenas_page_try_find_abandoned(mi_theap_t* theap, size_t slice_count, size_t block_size, mi_page_guards_t guards) { mi_heap_t* const heap = mi_theap_heap(theap); const size_t tseq = theap->tld->thread_seq; @@ -235,7 +285,7 @@ static mi_page_t* mi_arenas_page_try_find_abandoned(mi_theap_t* theap, size_t sl const bool allow_large = true; const int any_numa = -1; const bool match_numa = true; - const mi_arena_abandoned_visit_info_t info = { theap, slice_count, block_size, bin }; + const mi_arena_abandoned_visit_info_t info = { theap, slice_count, block_size, bin, guards }; void* page = NULL; mi_forall_suitable_arenas(heap, req_arena, tseq, match_numa, any_numa, allow_large, &mi_arena_try_find_abandoned_visit, &info, &page); return (mi_page_t*)page; @@ -279,17 +329,8 @@ static size_t mi_page_meta_in_front_block_start(size_t block_size, bool os_align // otherwise start after the info offset = mi_page_info_size(); } - return mi_align_up(offset,MI_MAX_ALIGN_SIZE); -} - -// The usable size of a page area (excluding the guard page at the end in secure mode). -static size_t mi_arenas_page_noguard_size(size_t alloc_size) { - #if MI_SECURE>=5 - mi_assert(alloc_size > _mi_os_secure_guard_page_size()); - return alloc_size - _mi_os_secure_guard_page_size(); - #else - return alloc_size; - #endif + // on CHERI, also align such that every block has exact bounds (os_align is already aligned enough) + return mi_align_up(offset, (os_align ? MI_MAX_ALIGN_SIZE : mi_max(MI_MAX_ALIGN_SIZE, mi_bounds_align(block_size)))); } // Can we offset the first block to spread cache sets? (see `MI_PAGE_CACHE_OFFSET`) @@ -303,7 +344,7 @@ static bool mi_page_block_can_cache_offset(size_t block_size, bool os_align) { static size_t mi_page_cache_offset_index(uint8_t* slice_start, size_t slice_count, size_t noffsets) { // Fibonacci hash of the page index: the top bits spread consecutive pages evenly over the offsets - const size_t page_index = (size_t)((uintptr_t)slice_start / MI_ARENA_SLICE_SIZE) / slice_count; + const size_t page_index = (size_t)(mi_addr(slice_start) / MI_ARENA_SLICE_SIZE) / slice_count; #if MI_SIZE_BITS >= 64 const size_t hash = page_index * MI_ZU(0x9E3779B97F4A7C15); #else @@ -321,21 +362,29 @@ static size_t mi_page_small_block_cache_offset(uint8_t* slice_start, size_t slic // Cache offset for the first block to spread the blocks of different pages over the cache sets (issue #1121). // The offset fits in the unused space at the end of the page; if there is none, we skip the last block // if that wastes less than 12.5% of the page (e.g. 64 KiB blocks in a 4 MiB page, but not 512 KiB blocks). -static size_t mi_page_block_cache_offset(uint8_t* slice_start, size_t slice_count, size_t block_size, size_t block_start, bool os_align) { +static size_t mi_page_block_cache_offset(uint8_t* slice_start, size_t slice_count, size_t block_size, size_t block_start, bool os_align, size_t page_noguard_size) { if (MI_PAGE_CACHE_OFFSET == 0 || os_align) return 0; if (!mi_block_size_has_cache_offset(block_size)) return mi_page_small_block_cache_offset(slice_start, slice_count, block_size); if (!mi_page_block_can_cache_offset(block_size, os_align)) return 0; - const size_t page_noguard_size = mi_arenas_page_noguard_size(_mi_size_of_slices(slice_count)); + const size_t unit = mi_max(MI_PAGE_CACHE_OFFSET_UNIT, mi_bounds_align(block_size)); // on CHERI, keep exact bounds + if (unit > MI_PAGE_CACHE_OFFSET_MAX) return 0; mi_assert_internal(block_start + block_size <= page_noguard_size); size_t slack = (page_noguard_size - block_start) % block_size; - if (slack < MI_PAGE_CACHE_OFFSET_UNIT && block_size < page_noguard_size/8) { slack += block_size; } - const size_t noffsets = 1 + (slack < MI_PAGE_CACHE_OFFSET_MAX ? slack : MI_PAGE_CACHE_OFFSET_MAX) / MI_PAGE_CACHE_OFFSET_UNIT; - return mi_page_cache_offset_index(slice_start, slice_count, noffsets) * MI_PAGE_CACHE_OFFSET_UNIT; + if (slack < unit && block_size < page_noguard_size/8) { slack += block_size; } + const size_t noffsets = 1 + (slack < MI_PAGE_CACHE_OFFSET_MAX ? slack : MI_PAGE_CACHE_OFFSET_MAX) / unit; + return mi_page_cache_offset_index(slice_start, slice_count, noffsets) * unit; +} + +// The offset of the first block after a front guard page (of at least 4 KiB which is the maximal natural block alignment) +static size_t mi_page_guard_front_block_start(size_t guard_front, size_t block_size) { + return mi_align_up(guard_front, mi_bounds_align(block_size)); // on CHERI, keep exact bounds } // Use the separate (zero initialized) `page_meta` if available and not preferring in front of the slices, -// and otherwise put the page meta data in front of the slices. Also determines the offset of the first block. -static mi_page_t* mi_page_meta_select(uint8_t* slice_start, size_t slice_count, mi_page_t* page_meta, size_t block_size, bool os_align, size_t* block_start) { +// and otherwise put the page meta data in front of the slices (without a front guard page). +// Also determines the offset of the first block. +static mi_page_t* mi_page_meta_select(uint8_t* slice_start, size_t slice_count, mi_page_t* page_meta, size_t block_size, bool os_align, + mi_page_guards_t* guards, size_t* block_start) { mi_assert_internal(page_meta == NULL || page_meta->block_size == 0); mi_page_t* page; size_t offset; @@ -350,28 +399,30 @@ static mi_page_t* mi_page_meta_select(uint8_t* slice_start, size_t slice_count, // (note: it is not initialized yet since the slices may not be committed yet) offset = mi_page_meta_in_front_block_start(block_size, os_align); page = (mi_page_t*)slice_start; + guards->front = 0; } else { // use the separate page meta data + mi_assert_internal(guards->front == 0 || guards->front >= MI_PAGE_MAX_START_BLOCK_ALIGN2); mi_memzero_aligned(page_meta, sizeof(*page_meta)); - offset = 0; + offset = mi_page_guard_front_block_start(guards->front, block_size); page = page_meta; } - offset += mi_page_block_cache_offset(slice_start, slice_count, block_size, offset, os_align); + offset += mi_page_block_cache_offset(slice_start, slice_count, block_size, offset, os_align, _mi_size_of_slices(slice_count) - guards->end); mi_assert_internal(os_align || offset < MI_ARENA_SLICE_SIZE); // so `mi_page_slice_start` can align down the page start *block_start = offset; - return page; + return mi_page_meta_bounded(page); } #if MI_PAGE_META_IS_ALIGNED // Ensure the aligned page meta data for the slices at `slice_start` is committed. static bool mi_arena_page_meta_ensure_committed(mi_arena_t* arena, uint8_t* slice_start) { - uint8_t* const meta_slices = (uint8_t*)mi_align_down_ptr(slice_start,MI_PAGE_META_ALIGNMENT); + uint8_t* const meta_slices = (uint8_t*)mi_align_down_ptr(slice_start,MI_PAGE_META_ALIGNMENT) + MI_PAGE_META_ALIGNED_OFFSET; mi_assert_internal(meta_slices >= mi_arena_start(arena)); const size_t meta_slice_index = (meta_slices - mi_arena_start(arena)) / MI_ARENA_SLICE_SIZE; if mi_unlikely(mi_bitmap_is_clear(arena->slices_committed, meta_slice_index)) { - // try to commit all page meta slices now - const size_t meta_slice_count = mi_arena_page_meta_aligned_slice_count(); + // try to commit all page meta slices now (but not the guard slices around it) + const size_t meta_slice_count = mi_arena_page_meta_aligned_array_slices(); // the following assertion does not hold in a concurrent setting.. // mi_assert_internal(mi_bitmap_is_clearN(arena->slices_committed, meta_slice_index, meta_slice_count)); const size_t commit_size = meta_slice_count * MI_ARENA_SLICE_SIZE; @@ -384,9 +435,9 @@ static bool mi_arena_page_meta_ensure_committed(mi_arena_t* arena, uint8_t* slic } #endif -// Try to allocate slices from free space in the arena's, and determine the page meta data (`*ppage`) and the offset of the first block (`*pblock_start`). -static uint8_t* mi_arenas_page_alloc_from_slices(mi_theap_t* theap, size_t slice_count, size_t block_size, bool commit, bool allow_large, - mi_memid_t* memid, mi_arena_pages_t** parena_pages, mi_page_t** ppage, size_t* pblock_start) { +// Try to allocate slices from free space in the arena's, and return the separate page meta data (if available) in `*ppage_meta`. +static uint8_t* mi_arenas_page_alloc_from_slices(mi_theap_t* theap, size_t slice_count, bool commit, bool allow_large, + mi_memid_t* memid, mi_arena_pages_t** parena_pages, mi_page_t** ppage_meta) { mi_heap_t* const heap = mi_theap_heap(theap); mi_tld_t* const tld = theap->tld; const int numa_node = (heap->numa_node >= 0 ? heap->numa_node : tld->numa_node); @@ -413,65 +464,84 @@ static uint8_t* mi_arenas_page_alloc_from_slices(mi_theap_t* theap, size_t slice // the separate page meta data (if available) #if MI_PAGE_META_IS_ALIGNED - mi_page_t* const page_meta = mi_aligned_ptr_page0(start); + *ppage_meta = mi_aligned_ptr_page0(start); #else mi_assert_internal(MI_PAGE_META_IS_SEPARATED!=0 || arena->pages_meta == NULL); - mi_page_t* const page_meta = (arena->pages_meta == NULL ? NULL : &arena->pages_meta[memid->mem.arena.slice_index]); + *ppage_meta = (arena->pages_meta == NULL ? NULL : &arena->pages_meta[memid->mem.arena.slice_index]); #endif - *ppage = mi_page_meta_select(start, slice_count, page_meta, block_size, false, pblock_start); return start; } -#if MI_PAGE_META_IS_ALIGNED -// Allocate slices from the OS with the (committed and zero'd) page meta data at the start of the `MI_PAGE_META_ALIGNMENT` aligned area, -// and determine the page meta data (`*ppage`) and the offset of the first block (`*pblock_start`). -static uint8_t* mi_arenas_page_alloc_from_os(mi_heap_t* heap, size_t slice_count, size_t max_page_meta_count, size_t block_size, size_t block_alignment, bool os_align, bool commit, bool allow_large, - mi_memid_t* memid, mi_page_t** ppage, size_t* pblock_start) { - MI_UNUSED(allow_large); +#if MI_PAGE_META_OS_IS_SEPARATED +// Allocate slices from the OS with the (committed and zero'd) page meta data in front of the page area (returned in `*ppage_meta`). +// The OS memory is `[guard slice | page meta data | guard slice | ... | page area]` where the guard slices are only reserved in secure mode. +// With aligned page meta data, the OS memory is `MI_PAGE_META_ALIGNMENT` aligned and the page meta data is indexed by slice (see `mi_aligned_ptr_page`). +static uint8_t* mi_arenas_page_alloc_from_os(mi_heap_t* heap, size_t slice_count, size_t max_page_meta_count, size_t block_alignment, bool os_align, bool commit, bool allow_large, + mi_memid_t* memid, mi_page_t** ppage_meta) { + MI_UNUSED(allow_large); MI_UNUSED(max_page_meta_count); MI_UNUSED(os_align); mi_arena_t* const req_arena = heap->exclusive_arena; const size_t alloc_size = _mi_size_of_slices(slice_count); + const size_t page_alignment = (block_alignment < MI_PAGE_ALIGN ? MI_PAGE_ALIGN : block_alignment); + #if MI_PAGE_META_IS_ALIGNED if (block_alignment >= MI_PAGE_META_ALIGNMENT) { // if we allow alignment >= MI_PAGE_META_ALIGNMENT we need to substract 1 from a pointer // in mi_aligned_ptr_page (and test for (intptr_t)p < 0 instead of NULL). We avoid this by limiting the max alignment. _mi_warning_message("requested alignment is too large (%zu KiB)\n", block_alignment / MI_KiB); errno = EINVAL; return NULL; - // page_offset = MI_PAGE_META_ALIGNMENT; - // os_start = (uint8_t*)_mi_arena_os_alloc_aligned(heap->subproc, alloc_size + page_offset, block_alignment, MI_PAGE_META_ALIGNMENT /* align offset */, false /* commit */, false /* allow large */, req_arena, memid); } + const size_t os_alignment = MI_PAGE_META_ALIGNMENT; + #else + const size_t os_alignment = page_alignment; + #endif - // offset in the block for the page area - const size_t page_offset = (block_alignment < MI_PAGE_ALIGN ? MI_PAGE_ALIGN : block_alignment); - uint8_t* const os_start = (uint8_t*)_mi_arena_os_alloc_aligned(heap->subproc, alloc_size + page_offset, MI_PAGE_META_ALIGNMENT, 0 /* align offset */, false /* commit */, false /* allow large */, req_arena, memid); + // offset of the page area (leaving room for the guard slices around the page meta data) + const size_t page_offset = mi_align_up(_mi_size_of_slices(2*MI_ARENA_GUARD_SLICES + 1), page_alignment); + uint8_t* const os_start = (uint8_t*)_mi_arena_os_alloc_aligned(heap->subproc, alloc_size + page_offset, os_alignment, 0 /* align offset */, false /* commit */, false /* allow large */, req_arena, memid); if (os_start==NULL) return NULL; + uint8_t* const start = os_start + page_offset; - // commit page info and page - const size_t min_page_count = mi_divide_up(page_offset + MI_ARENA_SLICE_SIZE,MI_ARENA_SLICE_SIZE) + max_page_meta_count; - const size_t min_page_meta = min_page_count * sizeof(mi_page_t); - mi_assert_internal(min_page_meta < page_offset); + // commit the page meta data and page area + uint8_t* const page_meta_start = os_start + _mi_size_of_slices(MI_ARENA_GUARD_SLICES); + #if MI_PAGE_META_IS_ALIGNED + const size_t page_meta_size = (mi_divide_up(page_offset + MI_ARENA_SLICE_SIZE,MI_ARENA_SLICE_SIZE) + max_page_meta_count) * sizeof(mi_page_t); + #else + const size_t page_meta_size = sizeof(mi_page_t); + #endif + mi_assert_internal(page_meta_start + mi_align_up(page_meta_size,_mi_os_page_size()) + _mi_os_secure_guard_page_size() <= start); bool is_zero; - bool ok = _mi_arena_commit(heap->subproc,req_arena,os_start,min_page_meta,&is_zero,min_page_meta /* don't count in stats? */); + bool ok = _mi_arena_commit(heap->subproc,req_arena,page_meta_start,page_meta_size,&is_zero,page_meta_size /* don't count in stats? */); if (ok && commit) { - ok = _mi_arena_commit(heap->subproc,req_arena,os_start + page_offset,alloc_size,NULL,0); + ok = _mi_arena_commit(heap->subproc,req_arena,start,alloc_size,NULL,0); } if (!ok) { _mi_os_free(heap->subproc,os_start,alloc_size+page_offset,*memid); return NULL; } if (!is_zero && !memid->initially_zero) { - mi_memzero_aligned(os_start,min_page_meta); + mi_memzero_aligned(page_meta_start,page_meta_size); } - uint8_t* const start = os_start + page_offset; - mi_assert_internal(mi_is_aligned(start,block_alignment)); + #if MI_ARENA_GUARD_SLICES > 0 + // set guard pages around the page meta data (if not initially committed, the guards are never committed) + if (memid->initially_committed && !memid->is_pinned) { + _mi_os_secure_guard_pages_set_around(heap->subproc, page_meta_start, page_meta_size, *memid); + } + #endif + if (commit) { memid->initially_committed = true; } // so we don't commit again + + // and determine the page meta data + mi_assert_internal(mi_is_aligned(start,page_alignment)); + #if MI_PAGE_META_IS_ALIGNED mi_assert_internal(mi_is_aligned(os_start,MI_PAGE_META_ALIGNMENT)); mi_assert_internal(mi_align_down_ptr(start-1,MI_PAGE_META_ALIGNMENT) == os_start); - mi_assert_internal((uint8_t*)mi_aligned_ptr_page0(start) < os_start + min_page_meta); - memid->initially_committed = true; // so we don't commit again - *ppage = mi_page_meta_select(start, slice_count, mi_aligned_ptr_page0(start), block_size, os_align, pblock_start); + mi_assert_internal((uint8_t*)mi_aligned_ptr_page0(start) < page_meta_start + page_meta_size); + *ppage_meta = mi_aligned_ptr_page0(start); + #else + *ppage_meta = (mi_page_t*)mi_ptr_bounded_data(page_meta_start, sizeof(mi_page_t)); + #endif return start; } #else -// Allocate slices from the OS with the page meta data in front of the slices, -// and determine the page meta data (`*ppage`) and the offset of the first block (`*pblock_start`). -static uint8_t* mi_arenas_page_alloc_from_os(mi_heap_t* heap, size_t slice_count, size_t max_page_meta_count, size_t block_size, size_t block_alignment, bool os_align, bool commit, bool allow_large, - mi_memid_t* memid, mi_page_t** ppage, size_t* pblock_start) { +// Allocate slices from the OS with the page meta data in front of the slices (so there is no separate page meta data). +static uint8_t* mi_arenas_page_alloc_from_os(mi_heap_t* heap, size_t slice_count, size_t max_page_meta_count, size_t block_alignment, bool os_align, bool commit, bool allow_large, + mi_memid_t* memid, mi_page_t** ppage_meta) { MI_UNUSED(max_page_meta_count); const size_t page_alignment = MI_ARENA_SLICE_ALIGN; const size_t alloc_size = _mi_size_of_slices(slice_count); @@ -485,19 +555,18 @@ static uint8_t* mi_arenas_page_alloc_from_os(mi_heap_t* heap, size_t slice_count start = (uint8_t*)_mi_arena_os_alloc_aligned(heap->subproc, alloc_size, page_alignment, 0 /* align offset */, commit, allow_large, heap->exclusive_arena, memid); } if (start == NULL) return NULL; - // page meta data is always placed in front of the slices - *ppage = mi_page_meta_select(start, slice_count, NULL, block_size, os_align, pblock_start); + *ppage_meta = NULL; return start; } #endif -// Allocate the slices for a fresh page and determine its page meta data (`*ppage`) and the offset of the first block (`*pblock_start`). -static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_count, size_t max_page_meta_count, size_t block_size, size_t block_alignment, bool os_align, bool commit, - mi_memid_t* memid, mi_arena_pages_t** parena_pages, mi_page_t** ppage, size_t* pblock_start ) { - mi_assert_internal(parena_pages!=NULL && ppage!=NULL && pblock_start!=NULL); +// Allocate the slices for a fresh page and return its separate page meta data (if available) in `*ppage_meta`. +static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_count, size_t max_page_meta_count, size_t block_alignment, bool os_align, bool commit, bool allow_large, + mi_memid_t* memid, mi_arena_pages_t** parena_pages, mi_page_t** ppage_meta ) { + mi_assert_internal(parena_pages!=NULL && ppage_meta!=NULL); *parena_pages = NULL; + *ppage_meta = NULL; *memid = _mi_memid_none(); - const bool allow_large = (MI_SECURE < 5); // 5 = guard page at end of each arena page mi_heap_t* const heap = mi_theap_heap(theap); // try to allocate from free space in arena's @@ -506,12 +575,12 @@ static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_ !os_align && // not large alignment slice_count <= mi_arena_max_object_size()/MI_ARENA_SLICE_SIZE) // and not too large { - slice_start = mi_arenas_page_alloc_from_slices(theap, slice_count, block_size, commit, allow_large, memid, parena_pages, ppage, pblock_start); + slice_start = mi_arenas_page_alloc_from_slices(theap, slice_count, commit, allow_large, memid, parena_pages, ppage_meta); } // otherwise fall back to the OS if (slice_start == NULL) { - slice_start = mi_arenas_page_alloc_from_os(heap, slice_count, max_page_meta_count, block_size, block_alignment, os_align, commit, allow_large, memid, ppage, pblock_start); + slice_start = mi_arenas_page_alloc_from_os(heap, slice_count, max_page_meta_count, block_alignment, os_align, commit, allow_large, memid, ppage_meta); if (slice_start == NULL) return NULL; mi_heap_stat_increase(heap,pages_os_allocated,1); } @@ -529,11 +598,13 @@ static void mi_arenas_page_free_prim(mi_page_t* page); // Commit the start of the page up to and including the first block if the slices are not yet committed. // Sets `*commit_size` to the committed size (or 0 if the slices were already committed). -static bool mi_arenas_page_commit_first_block(mi_theap_t* theap, mi_memid_t memid, uint8_t* slice_start, size_t block_start, size_t block_size, size_t page_noguard_size, size_t* commit_size) { +// Never commits beyond `page_end` (the end guard page or the end of the page). +static bool mi_arenas_page_commit_first_block(mi_theap_t* theap, mi_memid_t memid, uint8_t* slice_start, size_t block_start, size_t block_size, size_t page_end, size_t* commit_size) { *commit_size = 0; if (memid.initially_committed) return true; + mi_assert_internal(!memid.guard_front); // as the front guard page would never be recommitted (see `mi_arenas_page_free_prim`) size_t csize = mi_align_up(block_start + block_size, mi_page_min_commit_size()); - if (csize > page_noguard_size) { csize = page_noguard_size; } + if (csize > page_end) { csize = page_end; } bool is_zero = false; if mi_unlikely(!_mi_arena_commit(mi_theap_subproc(theap), mi_memid_arena(memid), slice_start, csize, &is_zero, 0)) { return false; @@ -542,23 +613,17 @@ static bool mi_arenas_page_commit_first_block(mi_theap_t* theap, mi_memid_t memi return true; } -// Prepare the page memory: zero the page meta data if it is in front of the slices, set the guard page, etc. -static void mi_arenas_page_init_memory(mi_theap_t* theap, mi_page_t* page, uint8_t* slice_start, size_t slice_count, size_t block_start, size_t page_noguard_size, mi_memid_t* memid) +// Prepare the page memory: zero the page meta data if it is in front of the slices, set the guard pages, etc. +// The end guard page (if any) is at `page_end`. +static void mi_arenas_page_init_memory(mi_theap_t* theap, mi_page_t* page, uint8_t* slice_start, size_t slice_count, size_t block_start, size_t page_end, mi_memid_t* memid) { - MI_UNUSED(theap); MI_UNUSED(slice_count); MI_UNUSED(block_start); MI_UNUSED(page_noguard_size); + MI_UNUSED(slice_count); MI_UNUSED(block_start); // zero initialize the page meta data if (!memid->initially_zero && (uint8_t*)page == slice_start) { mi_memzero_aligned(page, sizeof(*page)); } - // set the guard page - #if MI_SECURE>=5 - if (memid->initially_committed) { - _mi_os_secure_guard_page_set_at(mi_theap_subproc(theap), slice_start + page_noguard_size, *memid); - } - #endif - // claimed free slices: initialize the page partly if (!memid->initially_zero && memid->initially_committed) { mi_track_mem_undefined(slice_start, slice_count * MI_ARENA_SLICE_SIZE); @@ -568,26 +633,30 @@ static void mi_arenas_page_init_memory(mi_theap_t* theap, mi_page_t* page, uint8 } #if MI_DEBUG > 1 if (memid->initially_zero && memid->initially_committed) { - if (!mi_mem_is_zero(slice_start, page_noguard_size)) { + if (!mi_mem_is_zero(slice_start, page_end)) { _mi_error_message(EFAULT, "internal error: page memory was not zero initialized.\n"); memid->initially_zero = false; if (block_start > 0) { mi_memzero_aligned(page, sizeof(*page)); } } } #endif + + // set the guard pages + mi_subproc_t* const subproc = mi_theap_subproc(theap); + if (memid->guard_front) { _mi_os_secure_guard_page_set_at(subproc, slice_start, *memid); } + if (memid->guard_end) { _mi_os_secure_guard_page_set_at(subproc, slice_start + page_end, *memid); } } // Initialize the fields of the page meta data. -static void mi_arenas_page_init_meta(mi_theap_t* theap, mi_page_t* page, uint8_t* slice_start, size_t block_start, size_t block_size, bool os_align, - size_t page_noguard_size, size_t commit_size, mi_memid_t memid) +static void mi_arenas_page_init_meta(mi_theap_t* theap, mi_page_t* page, uint8_t* slice_start, size_t block_start, size_t block_size, + size_t reserved, size_t commit_size, mi_memid_t memid) { - const size_t reserved = (os_align ? 1 : (page_noguard_size - block_start) / block_size); mi_assert_internal(reserved > 0 && reserved <= UINT16_MAX); // initialize the page start uint8_t* const start = slice_start + block_start; mi_assert_internal(start > (uint8_t*)page); - page->page_offset = start - (uint8_t*)page; + page->page_start = (uint8_t*)mi_ptr_bounded_data(start, reserved * block_size); // all blocks are derived from this // initialize page meta-data page->reserved = (uint16_t)reserved; @@ -650,45 +719,60 @@ static bool mi_arenas_page_register(mi_theap_t* theap, mi_page_t* page, mi_arena return true; } -// Allocate a fresh page -static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_count, size_t block_size, size_t block_alignment, bool commit) +// Allocate a fresh page with the given guard pages (see `mi_theap_page_guards`) +static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_count, size_t block_size, size_t block_alignment, bool commit, mi_page_guards_t guards) { const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN); const bool singleton = (os_align || block_size > MI_LARGE_MAX_OBJ_SIZE); const size_t max_page_meta_count = (singleton && slice_count > 2 ? 2 : slice_count); + const bool allow_large = (guards.front == 0 && guards.end == 0); // large OS pages cannot be protected - // allocate the slices and determine the page meta data + // allocate the slices (always committed with a front guard page, see `mi_arenas_page_commit_first_block`) mi_memid_t memid = _mi_memid_none(); mi_arena_pages_t* arena_pages = NULL; - mi_page_t* page = NULL; - size_t block_start = 0; - uint8_t* const slice_start = mi_arenas_page_alloc_fresh_area(theap,slice_count,max_page_meta_count,block_size,block_alignment,os_align,commit,&memid,&arena_pages,&page,&block_start); + mi_page_t* page_meta = NULL; + uint8_t* const slice_start = mi_arenas_page_alloc_fresh_area(theap,slice_count,max_page_meta_count,block_alignment,os_align,commit || guards.front > 0,allow_large,&memid,&arena_pages,&page_meta); if (!slice_start) return NULL; + // in a torn forked child, the free slices may still have a guard page of a page that was allocated during the fork + if (MI_SECURE > 0 && memid.memkind == MI_MEM_ARENA && !memid.is_pinned && _mi_fork_is_torn()) { + _mi_os_unguard(slice_start, _mi_size_of_slices(slice_count), true /* make accessible */); + } + + // determine the page meta data, the offset of the first block, and the end of the page + size_t block_start = 0; + mi_page_t* const page = mi_page_meta_select(slice_start, slice_count, page_meta, block_size, os_align, &guards, &block_start); + const size_t alloc_size = _mi_size_of_slices(slice_count); + const size_t reserved = (os_align ? 1 : (alloc_size - guards.end - block_start) / block_size); + const size_t page_end = (guards.end > 0 ? mi_align_up(block_start + reserved*block_size, _mi_os_page_size()) : alloc_size); // the end guard is right after the blocks + mi_assert_internal(page_end + guards.end <= alloc_size); + memid.guard_front = (guards.front > 0); + memid.guard_end = (guards.end > 0); mi_assert_internal(block_size < MI_MAX_ALIGN_SIZE || block_start % MI_MAX_ALIGN_SIZE == 0); // all blocks must be naturally aligned (as relied upon by `mi_malloc_aligned`) mi_assert_internal(block_size % mi_natural_alignment_of_size(block_size) == 0); mi_assert_internal(mi_is_aligned(slice_start + block_start, mi_natural_alignment_of_size(block_size))); + // on CHERI, all blocks must have exactly representable bounds + mi_assert_internal(mi_bounds_size(block_size) == block_size && mi_is_aligned(slice_start + block_start, mi_bounds_align(block_size))); // commit the first block (if needed) - const size_t alloc_size = _mi_size_of_slices(slice_count); - const size_t page_noguard_size = mi_arenas_page_noguard_size(alloc_size); size_t commit_size = 0; - if (!mi_arenas_page_commit_first_block(theap, memid, slice_start, block_start, block_size, page_noguard_size, &commit_size)) { + if (!mi_arenas_page_commit_first_block(theap, memid, slice_start, block_start, block_size, page_end, &commit_size)) { _mi_arenas_free(mi_theap_subproc(theap), slice_start, alloc_size, memid); return NULL; } mi_assert_internal((commit && commit_size==0) || (!commit && (commit_size <= UINT16_MAX * _mi_os_page_size()))); // initialize the page memory and meta data - mi_arenas_page_init_memory(theap, page, slice_start, slice_count, block_start, page_noguard_size, &memid); - mi_arenas_page_init_meta(theap, page, slice_start, block_start, block_size, os_align, page_noguard_size, commit_size, memid); + mi_arenas_page_init_memory(theap, page, slice_start, slice_count, block_start, page_end, &memid); + mi_arenas_page_init_meta(theap, page, slice_start, block_start, block_size, reserved, commit_size, memid); #if MI_PAGE_META_IS_ALIGNED mi_arenas_page_link_aligned_meta(page, slice_start, slice_count, max_page_meta_count); #endif mi_assert_internal(page->free==NULL); mi_assert_internal(((uint8_t*)page != slice_start) == mi_page_meta_is_separated(page)); mi_assert_internal(mi_page_slice_start(page) == slice_start); - mi_assert_internal(mi_page_size(page) <= page_noguard_size); + mi_assert_internal(mi_page_slice_offset_of(page, mi_page_size(page)) <= page_end); + mi_assert_internal(!memid.guard_end || mi_page_guard_end_offset(page) == page_end); // now we can register the page (and use `mi_arenas_page_free_prim` on error) if (!mi_arenas_page_register(theap, page, arena_pages)) return NULL; @@ -718,7 +802,8 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou static mi_page_t* mi_arenas_page_regular_alloc(mi_theap_t* theap, size_t slice_count, size_t block_size) { // 1. look for an abandoned page - mi_page_t* page = mi_arenas_page_try_find_abandoned(theap, slice_count, block_size); + const mi_page_guards_t guards = mi_theap_page_guards(theap, _mi_size_of_slices(slice_count), false); + mi_page_t* page = mi_arenas_page_try_find_abandoned(theap, slice_count, block_size, guards); if (page != NULL) { return page; // return as abandoned } @@ -728,7 +813,7 @@ static mi_page_t* mi_arenas_page_regular_alloc(mi_theap_t* theap, size_t slice_c const bool commit = (mi_page_min_commit_size() >= slice_count * MI_ARENA_SLICE_SIZE || // always commit small pages (slice_count >= _mi_slice_count_of_size(UINT16_MAX * _mi_os_page_size())) || // always commit pages too large to hold a 32-bit slice_committed (commit_on_demand == 2 && _mi_os_has_overcommit()) || (commit_on_demand == 0)); - page = mi_arenas_page_alloc_fresh(theap, slice_count, block_size, 1, commit); + page = mi_arenas_page_alloc_fresh(theap, slice_count, block_size, 1, commit, guards); if (page == NULL) return NULL; mi_assert_internal(page->memid.memkind != MI_MEM_ARENA || page->memid.mem.arena.slice_count == slice_count); @@ -744,20 +829,19 @@ static mi_page_t* mi_arenas_page_regular_alloc(mi_theap_t* theap, size_t slice_c static mi_page_t* mi_arenas_page_singleton_alloc(mi_theap_t* theap, size_t block_size, size_t block_alignment) { const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN); - #if MI_PAGE_META_IS_ALIGNED + #if MI_PAGE_META_OS_IS_SEPARATED const size_t info_size = 0; #else - const size_t info_size = (os_align ? MI_PAGE_ALIGN : mi_page_info_size()); + const size_t info_size = (os_align ? MI_PAGE_ALIGN : mi_align_up(mi_page_info_size(), mi_bounds_align(block_size))); // see `mi_page_meta_in_front_block_start` #endif // room for the cache offset of the block (committed but not touched) const size_t cache_offset_size = (mi_page_block_can_cache_offset(block_size, os_align) ? MI_PAGE_CACHE_OFFSET_MAX : 0); - #if MI_SECURE < 2 - const size_t slice_count = _mi_slice_count_of_size(info_size + cache_offset_size + block_size); - #else - const size_t slice_count = _mi_slice_count_of_size(mi_align_up(info_size + cache_offset_size + block_size, _mi_os_secure_guard_page_size()) + _mi_os_secure_guard_page_size()); - #endif + // determine the guard pages based on the page size without guard pages + const size_t size = info_size + cache_offset_size + block_size; + const mi_page_guards_t guards = mi_theap_page_guards(theap, _mi_size_of_slices(_mi_slice_count_of_size(size)), os_align); + const size_t slice_count = _mi_slice_count_of_size(mi_page_guard_front_block_start(guards.front, block_size) + size + guards.end); - mi_page_t* page = mi_arenas_page_alloc_fresh(theap, slice_count, block_size, block_alignment, true /* commit singletons always */); + mi_page_t* page = mi_arenas_page_alloc_fresh(theap, slice_count, block_size, block_alignment, true /* commit singletons always */, guards); if (page == NULL) return NULL; mi_assert(page->reserved == 1); @@ -836,13 +920,13 @@ static void mi_arenas_page_free_prim(mi_page_t* page) { // unregister page _mi_page_map_unregister(page); - // recommit guard page at the end? + // reset the guard pages // we must do this since we may later allocate large spans over this page and cannot have a guard page in between - #if MI_SECURE >= 5 - if (!page->memid.is_pinned) { - _mi_os_secure_guard_page_reset_before(mi_page_subproc(page), mi_page_slice_start(page) + mi_page_full_size(page), page->memid); - } - #endif + // (for a page committed on demand the end guard page stays decommitted and its commit bit is clear) + mi_subproc_t* const subproc = mi_page_subproc(page); + uint8_t* const slice_start = mi_page_slice_start(page); + if (page->memid.guard_front) { _mi_os_secure_guard_page_reset_at(subproc, slice_start, page->memid); } + if (page->memid.guard_end) { _mi_os_secure_guard_page_reset_at(subproc, slice_start + mi_page_guard_end_offset(page), page->memid); } // and free if (page->memid.memkind == MI_MEM_ARENA) { @@ -954,7 +1038,7 @@ void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theapx) { mi_assert_internal(mi_theap_matches_thread(current_theapx)); // mi_assert_internal(current_theap == _mi_page_associated_theap(page)); #if MI_QSBR - mi_assert_internal(!page->qsbr_in_queue && page->qsbr_next == NULL && page->qsbr_prev == NULL); + mi_assert_internal(!mi_page_qsbr_is_in_queue(page) && page->qsbr_next == NULL && page->qsbr_prev == NULL); #endif // note: somewhat expensive to update here, but might be good as then we attribute @@ -1189,7 +1273,7 @@ bool _mi_heap_visit_blocks(mi_heap_t* heap, bool abandoned_only, bool visit_bloc mi_assert(visitor!=NULL); if (visitor==NULL) return false; if (heap==NULL) { heap = mi_heap_main(); } - bool ok; + bool ok = false; mi_lock(&heap->theaps_lock) { ok = mi_heap_visit_blocks_locked(heap, abandoned_only, visit_blocks, visitor, arg); } @@ -1255,12 +1339,16 @@ typedef struct mi_heap_delete_visit_info_s { mi_theap_t* theap; } mi_heap_delete_visit_info_t; +static mi_page_t* mi_heap_area_page(const mi_heap_area_t* area) { + return mi_ptr_page(area->blocks); +} + static bool mi_heap_delete_page(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg) { MI_UNUSED(block); MI_UNUSED(block_size); MI_UNUSED(heap); const mi_heap_delete_visit_info_t* info = (const mi_heap_delete_visit_info_t*)arg; mi_heap_t* heap_target = info->heap_target; mi_theap_t* const theap = NULL; // info->theap; mi_assert_internal(mi_theap_heap(theap) == heap); - mi_page_t* const page = (mi_page_t*)area->reserved1; + mi_page_t* const page = mi_heap_area_page(area); mi_page_claim_ownership(page); // claim ownership _mi_page_qsbr_clear(page); // deletion requires an unprotected heap; destruction requires no readers @@ -1269,7 +1357,7 @@ static bool mi_heap_delete_page(const mi_heap_t* heap, const mi_heap_area_t* are } else { page->next = page->prev = NULL; // yikes.. better not to try to access this from a thread later on.. - mi_page_set_theap(page,NULL); // set threadid to abandoned + mi_page_set_theap(page,NULL); // set thread_id to abandoned } mi_assert_internal(mi_page_is_abandoned(page)); mi_assert_internal(mi_page_is_owned(page)); diff --git a/system/lib/mimalloc/src/arena/arena.c b/system/lib/mimalloc/src/arena/arena.c index ea7fb7d750eac..16234037fb528 100644 --- a/system/lib/mimalloc/src/arena/arena.c +++ b/system/lib/mimalloc/src/arena/arena.c @@ -110,14 +110,27 @@ mi_arena_id_t _mi_arena_id_none(void) { } mi_arena_t* _mi_arena_from_id(mi_arena_id_t id) { + #if MI_CHERI + // id's are just addresses (without provenance) so we look up the arena in the current subprocess + mi_arena_t* arena = NULL; + if (id != NULL) { + mi_subproc_t* const subproc = _mi_subproc(); + const size_t count = mi_arenas_get_count(subproc); + for (size_t i = 0; i < count && arena == NULL; i++) { + mi_arena_t* const candidate = mi_arena_from_index(subproc, i); + if (candidate != NULL && mi_addr(candidate) == mi_addr(id)) { arena = candidate; } + } + } + #else mi_arena_t* const arena = (mi_arena_t*)id; + #endif mi_assert_internal(arena==NULL || arena->parent==NULL); // id's should never point to sub-arena's return arena; } mi_arena_id_t mi_arena_id_from_arena(mi_arena_t* arena) { mi_assert_internal(arena==NULL || arena->parent==NULL); - return (arena==NULL ? _mi_arena_id_none() : (mi_arena_id_t)arena); + return (arena==NULL ? _mi_arena_id_none() : (mi_arena_id_t)mi_ptr_from_addr(mi_addr(arena))); // don't expose the arena capability } @@ -141,7 +154,7 @@ size_t mi_arena_min_size(void) { // fixed limit for the maximum object size in an arena static size_t mi_arena_max_fixed_object_size(void) { #if MI_PAGE_META_IS_ALIGNED - return (MI_PAGE_META_ALIGNMENT - mi_align_up(MI_PAGE_META_ALIGNED_COUNT * sizeof(mi_page_t), MI_ARENA_SLICE_SIZE)); + return (MI_PAGE_META_ALIGNMENT - _mi_size_of_slices(mi_arena_page_meta_aligned_slice_count())); #else return (MI_ARENA_MAX_SIZE - MI_ARENA_CHUNK_SIZE); // minus an initial chunk to accommodate meta info #endif @@ -177,7 +190,7 @@ void* mi_arena_area(mi_arena_id_t arena_id, size_t* size) { mi_assert_internal(_mi_size_of_slices(arena->slice_count) <= arena->total_size); *size = arena->total_size; } - return mi_arena_start(arena); + return mi_ptr_bounded_data(mi_arena_start(arena), 0); // only expose the address } @@ -481,14 +494,21 @@ static bool mi_arenas_add(mi_subproc_t* subproc, mi_arena_t* arena, mi_arena_id_ } +// Arena memory layout: +// [guard slice (secure only) | aligned page meta array (if aligned) | arena info + bitmaps | guard page (secure only) | slices...] +// Other `MI_PAGE_META_ALIGNMENT` aligned areas start with: [guard slice | aligned page meta array | guard slice] (see `mi_arena_page_meta_aligned_slice_count`). +static size_t mi_arena_info_offset(void) { + return _mi_size_of_slices(MI_ARENA_GUARD_SLICES + mi_arena_page_meta_aligned_array_slices()); +} + static mi_arena_t* mi_arena_info(void* area) { - return (mi_arena_t*)((uint8_t*)area + _mi_size_of_slices(mi_arena_page_meta_aligned_slice_count())); + return (mi_arena_t*)mi_ptr_bounded_data((uint8_t*)area + mi_arena_info_offset(), sizeof(mi_arena_t)); } static size_t mi_arena_info_slices_needed(size_t slice_count, size_t* bitmap_base) { if (slice_count == 0) slice_count = MI_BCHUNK_BITS; mi_assert_internal((slice_count % MI_BCHUNK_BITS) == 0); - const size_t base_size = _mi_size_of_slices(mi_arena_page_meta_aligned_slice_count()) + mi_align_up(sizeof(mi_arena_t), MI_BCHUNK_SIZE); + const size_t base_size = mi_arena_info_offset() + mi_align_up(sizeof(mi_arena_t), MI_BCHUNK_SIZE); const size_t bitmaps_count = 4 + MI_ARENA_BIN_COUNT; // commit, dirty, purge, pages, and abandoned const size_t bitmaps_size = bitmaps_count * mi_bitmap_size(slice_count, NULL) + mi_bbitmap_size(slice_count, NULL); // + free #if MI_PAGE_META_IS_SEPARATED && !MI_PAGE_META_IS_ALIGNED @@ -508,8 +528,9 @@ static size_t mi_arena_info_slices_needed(size_t slice_count, size_t* bitmap_bas static mi_bbitmap_t* mi_arena_bbitmap_init(mi_subproc_t* subproc, size_t slice_count, uint8_t** base) { mi_bbitmap_t* bbitmap = (mi_bbitmap_t*)(*base); - *base = (*base) + mi_bbitmap_init(subproc, bbitmap, slice_count, true /* already zero */); - return bbitmap; + const size_t size = mi_bbitmap_init(subproc, bbitmap, slice_count, true /* already zero */); + *base = (*base) + size; + return (mi_bbitmap_t*)mi_ptr_bounded_data(bbitmap, size); } @@ -539,16 +560,17 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, // } // commit & zero if needed + // the arena meta data (aligned page meta of the first area, and the arena info) is between the leading guard slice and the trailing guard page (in secure mode) + uint8_t* const meta_start = (uint8_t*)start + _mi_size_of_slices(MI_ARENA_GUARD_SLICES); + const size_t meta_size = _mi_size_of_slices(info_slices - MI_ARENA_GUARD_SLICES) - _mi_os_secure_guard_page_size(); if (!memid.initially_committed) { - size_t commit_size = _mi_size_of_slices(info_slices); - // leave a guard OS page decommitted at the end? - if (!memid.is_pinned) { commit_size -= _mi_os_secure_guard_page_size(); } + // leave the guards decommitted bool ok = false; if (commit_fun != NULL) { - ok = (*commit_fun)(true /* commit */, start, commit_size, NULL, commit_fun_arg); + ok = (*commit_fun)(true /* commit */, meta_start, meta_size, NULL, commit_fun_arg); } else { - ok = _mi_os_commit(subproc, start, commit_size, NULL); + ok = _mi_os_commit(subproc, meta_start, meta_size, NULL); } if (!ok) { _mi_warning_message("unable to commit meta-data for OS memory"); @@ -556,12 +578,12 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, } } else if (!memid.is_pinned) { - // if MI_SECURE, set a guard page at the end of the arena info + // Set guard pages before and after the arena meta data // todo: this does not respect the commit_fun as the memid is of external memory - _mi_os_secure_guard_page_set_before(subproc, (uint8_t*)start + _mi_size_of_slices(info_slices), memid); + _mi_os_secure_guard_pages_set_around(subproc, meta_start, meta_size, memid); } if (!memid.initially_zero) { - mi_memzero(start, _mi_size_of_slices(info_slices) - _mi_os_secure_guard_page_size()); + mi_memzero(meta_start, meta_size); } // init @@ -595,7 +617,7 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, arena->pages_main.pages_abandoned[i] = mi_arena_bitmap_init(slice_count, &base); } #if MI_PAGE_META_IS_SEPARATED && !MI_PAGE_META_IS_ALIGNED - arena->pages_meta = (mi_page_t*)base; + arena->pages_meta = (mi_page_t*)mi_ptr_bounded_data(base, slice_count * sizeof(mi_page_t)); base += (slice_count * sizeof(mi_page_t)); #else arena->pages_meta = NULL; @@ -616,6 +638,13 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, } mi_bbitmap_unsafe_setN(arena->slices_free, start_idx, count); } + #if MI_ARENA_GUARD_SLICES > 0 + // guard the page meta data of the other aligned areas (if not committed, the guard slices are never committed) + if (i > 0 && start_idx <= arena->slice_count && memid.initially_committed && !memid.is_pinned) { + uint8_t* const page_meta_start = (uint8_t*)start + _mi_size_of_slices(i + MI_ARENA_GUARD_SLICES); + _mi_os_secure_guard_pages_set_around(subproc, page_meta_start, _mi_size_of_slices(mi_arena_page_meta_aligned_array_slices()), memid); + } + #endif } #else mi_bbitmap_unsafe_setN(arena->slices_free, info_slices /* start */, arena->slice_count - info_slices); diff --git a/system/lib/mimalloc/src/arena/arena.h b/system/lib/mimalloc/src/arena/arena.h index 2e01fa4c84d25..b9957db244a1f 100644 --- a/system/lib/mimalloc/src/arena/arena.h +++ b/system/lib/mimalloc/src/arena/arena.h @@ -41,8 +41,8 @@ static inline size_t mi_arena_info_slices(mi_arena_t* arena) { return arena->info_slices; } -// slices reserved for page meta info at the start of aligned chunks -static inline size_t mi_arena_page_meta_aligned_slice_count(void) { +// slices of the aligned page meta data array at the start of each aligned area +static inline size_t mi_arena_page_meta_aligned_array_slices(void) { #if MI_PAGE_META_IS_ALIGNED return mi_divide_up(MI_PAGE_META_ALIGNED_COUNT * sizeof(mi_page_t), MI_ARENA_SLICE_SIZE); #else @@ -50,6 +50,15 @@ static inline size_t mi_arena_page_meta_aligned_slice_count(void) { #endif } +// slices reserved at the start of each aligned area: the page meta data array (with a guard slice in front and after in secure mode) +static inline size_t mi_arena_page_meta_aligned_slice_count(void) { + #if MI_PAGE_META_IS_ALIGNED + return (MI_ARENA_GUARD_SLICES + mi_arena_page_meta_aligned_array_slices() + MI_ARENA_GUARD_SLICES); + #else + return 0; + #endif +} + // Start of the arena memory area static inline uint8_t* mi_arena_start(mi_arena_t* arena) { return ((uint8_t*)arena->start); @@ -73,8 +82,9 @@ static inline mi_arena_t* mi_arena_from_memid(mi_memid_t memid, size_t* slice_in // initialize the bitmap for `slice_count` slices at base (and advance the base pointer) static inline mi_bitmap_t* mi_arena_bitmap_init(size_t slice_count, uint8_t** base) { mi_bitmap_t* bitmap = (mi_bitmap_t*)(*base); - *base = (*base) + mi_bitmap_init(bitmap, slice_count, true /* already zero */); - return bitmap; + const size_t size = mi_bitmap_init(bitmap, slice_count, true /* already zero */); + *base = (*base) + size; + return (mi_bitmap_t*)mi_ptr_bounded_data(bitmap, size); } typedef bool (mi_forall_arena_fun_t)(mi_arena_t* arena, const void* arg, void** result); diff --git a/system/lib/mimalloc/src/arena/bitmap.c b/system/lib/mimalloc/src/arena/bitmap.c index d1c9228df0084..800d349b81df8 100644 --- a/system/lib/mimalloc/src/arena/bitmap.c +++ b/system/lib/mimalloc/src/arena/bitmap.c @@ -248,21 +248,16 @@ static inline bool mi_bitmap_find(mi_bitmap_t* bitmap, size_t tseq, size_t n, si note: the compiler will fully inline the indirect function call -------------------------------------------------------------------------------- */ -typedef struct mi_claim_fun_data_s { - mi_arena_t* arena; -} mi_claim_fun_data_t; - static bool mi_bitmap_try_find_and_claim_visit(mi_bitmap_t* bitmap, size_t chunk_idx, size_t n, size_t* pidx, void* arg1, void* arg2) { mi_assert_internal(n==1); MI_UNUSED(n); mi_claim_fun_t* claim_fun = (mi_claim_fun_t*)arg1; - mi_claim_fun_data_t* claim_data = (mi_claim_fun_data_t*)arg2; size_t cidx; if mi_likely(mi_bchunk_try_find_and_clear(&bitmap->chunks[chunk_idx], &cidx)) { const size_t slice_index = (chunk_idx * MI_BCHUNK_BITS) + cidx; mi_assert_internal(slice_index < mi_bitmap_max_bits(bitmap)); bool keep_set = true; - if ((*claim_fun)(slice_index, claim_data->arena, &keep_set)) { + if ((*claim_fun)(slice_index, arg2, &keep_set)) { // success! mi_assert_internal(!keep_set); *pidx = slice_index; @@ -288,10 +283,9 @@ static bool mi_bitmap_try_find_and_claim_visit(mi_bitmap_t* bitmap, size_t chunk // Find a set bit in the bitmap and try to atomically clear it and claim it. // (Used to find pages in the pages_abandoned bitmaps.) mi_decl_nodiscard bool mi_bitmap_try_find_and_claim(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx, - mi_claim_fun_t* claim, mi_arena_t* arena ) + mi_claim_fun_t* claim, void* arg ) { - mi_claim_fun_data_t claim_data = { arena }; - return mi_bitmap_find(bitmap, tseq, 1, pidx, &mi_bitmap_try_find_and_claim_visit, (void*)claim, &claim_data); + return mi_bitmap_find(bitmap, tseq, 1, pidx, &mi_bitmap_try_find_and_claim_visit, (void*)claim, arg); } diff --git a/system/lib/mimalloc/src/arena/bitmap.h b/system/lib/mimalloc/src/arena/bitmap.h index fab4ec2bccc5f..3fe0ede55a8b7 100644 --- a/system/lib/mimalloc/src/arena/bitmap.h +++ b/system/lib/mimalloc/src/arena/bitmap.h @@ -189,13 +189,13 @@ static inline bool mi_bitmap_is_clear(mi_bitmap_t* bitmap, size_t idx) { } // Called once a bit is cleared to see if the memory slice can be claimed. -typedef bool (mi_claim_fun_t)(size_t slice_index, mi_arena_t* arena, bool* keep_set); +typedef bool (mi_claim_fun_t)(size_t slice_index, void* arg, bool* keep_set); // Find a set bits in the bitmap, atomically clear it, and check if `claim` returns true. // If not claimed, continue on (potentially setting the bit again depending on `keep_set`). // Returns true on success, and in that case sets the index: `0 <= *pidx <= MI_BITMAP_MAX_BITS-n`. mi_decl_nodiscard bool mi_bitmap_try_find_and_claim(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx, - mi_claim_fun_t* claim, mi_arena_t* arena ); + mi_claim_fun_t* claim, void* arg ); // Atomically clear a bit but only if it is set. Will block otherwise until the bit is set. diff --git a/system/lib/mimalloc/src/heap/heap.c b/system/lib/mimalloc/src/heap/heap.c index e49d778455c6d..5d1c59bb6a81a 100644 --- a/system/lib/mimalloc/src/heap/heap.c +++ b/system/lib/mimalloc/src/heap/heap.c @@ -45,7 +45,7 @@ void mi_heap_stats_merge_to_main(mi_heap_t* heap) { } bool _mi_heap_theap_set(mi_heap_t* heap, mi_theap_t* theap) { - mi_assert_internal((uintptr_t)theap == 1 || mi_theap_heap(theap)==heap); + mi_assert_internal(mi_addr(theap) == 1 || mi_theap_heap(theap)==heap); mi_assert_internal(!_mi_is_empty_theap(theap)); mi_assert_internal(heap->theap != 0); return _mi_thread_local_set(heap->theap,theap); @@ -103,7 +103,7 @@ mi_decl_cold mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap) mi_theap_t* theap = (mi_theap_t*)_mi_thread_local_get(heap->theap); if mi_unlikely(theap==NULL) { theap = mi_heap_init_theap(heap); - if (theap==NULL) { return (mi_theap_t*)&_mi_theap_empty_wrong; } // this will return NULL from alloc/alloc-generic.c:_mi_malloc_generic + if (theap==NULL) { return &_mi_process.theap_empty_wrong; } // this will return NULL from alloc/alloc-generic.c:_mi_malloc_generic } _mi_theap_cached_set(theap); return theap; @@ -149,10 +149,14 @@ void _mi_heap_init(mi_heap_t* heap, mi_thread_local_t theap_slot, mi_subproc_t* mi_heap_t* _mi_heap_new_for_subproc(mi_subproc_t* subproc, mi_arena_id_t exclusive_arena_id, bool is_main_heap, bool use_qsbr) { mi_assert_internal(is_main_heap ? (subproc->heap_main == NULL && subproc->parent != NULL) : subproc->heap_main != NULL); - // heap data is allocated in the current subproc - mi_heap_t* const heap_main = (is_main_heap ? subproc->parent->heap_main : subproc->heap_main); - // todo: allocate heap data in the exclusive arena ? - mi_heap_t* const heap = (mi_heap_t*)mi_heap_zalloc( heap_main, sizeof(mi_heap_t) ); + // heap data is allocated in the current subproc (or the parent for a main heap) + mi_subproc_t* const owner = (is_main_heap ? subproc->parent : subproc); + #if MI_SECURE + // in secure mode, keep heap data separate from user data (at the cost of taking a lock) + mi_heap_t* const heap = (mi_heap_t*)_mi_meta_zalloc(owner, sizeof(mi_heap_t), NULL); + #else + mi_heap_t* const heap = (mi_heap_t*)mi_heap_zalloc(owner->heap_main, sizeof(mi_heap_t)); + #endif if (heap==NULL) return NULL; // reserve a thread local slot for this heap (see also issue #1230) mi_thread_local_t theap_slot = (is_main_heap ? mi_thread_local_key_fast : _mi_thread_local_create()); @@ -192,7 +196,7 @@ mi_heap_t* mi_heap_new_qsbr(void) { // free all theaps belonging to this heap (without deleting their pages as we do this arena wise for efficiency) static void mi_heap_free_theaps(mi_heap_t* heap) { - // This can run concurrently with a thread that terminates (see `heap/init.c:mi_thread_theaps_done`), + // This can run concurrently with a thread that terminates (see `heap/process.c:mi_thread_theaps_done`), // and we need to ensure we free theaps atomically. // We first detach our theaps list from any thread local lists diff --git a/system/lib/mimalloc/src/heap/page-map.c b/system/lib/mimalloc/src/heap/page-map.c index 74954b797fa25..a0692b9d2bce2 100644 --- a/system/lib/mimalloc/src/heap/page-map.c +++ b/system/lib/mimalloc/src/heap/page-map.c @@ -16,218 +16,9 @@ static void mi_page_map_cannot_commit(void) { _mi_warning_message("unable to commit the allocation page-map on-demand\n" ); } -#if MI_PAGE_MAP_FLAT - -// The page-map contains a byte for each 64kb slice in the address space. -// For an address `a` where `ofs = _mi_page_map[a >> 16]`: -// 0 = unused -// 1 = the slice at `a & ~0xFFFF` is a mimalloc page. -// 1 < ofs <= 127 = the slice is part of a page, starting at `(((a>>16) - ofs - 1) << 16)`. -// -// 1 byte per slice => 1 TiB address space needs a 2^14 * 2^16 = 16 MiB page map. -// A full 256 TiB address space (48 bit) needs a 4 GiB page map. -// A full 4 GiB address space (32 bit) needs only a 64 KiB page map. - -// Use an initial empty page map so `free(NULL)` works even if mimalloc is not yet initialized (issue #1341) -static uint8_t mi_page_map_empty[1] = { 1 }; // mi_ptr_page(NULL) == NULL - -mi_decl_hidden mi_decl_cache_align _Atomic(uint8_t*) _mi_page_map = mi_page_map_empty; -mi_decl_hidden _Atomic(void*) _mi_page_map_max_address = NULL; -static mi_memid_t mi_page_map_memid; - -#define MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT MI_ARENA_SLICE_SIZE -static mi_bitmap_t* mi_page_map_commit; // one bit per committed 64 KiB entries - -mi_decl_nodiscard static bool mi_page_map_ensure_committed(size_t idx, size_t slice_count); - -bool _mi_page_map_init(void) { - size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_MAX_VABITS); - if (vbits == 0) { - vbits = _mi_os_virtual_address_bits(); - #if MI_ARCH_X64 // canonical address is limited to the first 128 TiB - if (vbits >= 48) { vbits = 47; } - #endif - } - if (vbits < MI_ARENA_SLICE_SHIFT) { - vbits = MI_ARENA_SLICE_SHIFT; - } - if (vbits < MI_MIN_VABITS) { // cover at least this much for a faster _mi_checked_ptr - vbits = MI_MIN_VABITS; - } - if (vbits > MI_MAX_VABITS) { // limit page map size even if more virtual addresses are available - vbits = MI_MAX_VABITS; - } - - // Allocate the page map and commit bits - mi_atomic_store_ptr_release(void, &_mi_page_map_max_address, (void*)(vbits >= MI_SIZE_BITS ? (SIZE_MAX - MI_ARENA_SLICE_SIZE + 1) : (MI_PU(1) << vbits))); - const size_t page_map_size = (MI_ZU(1) << (vbits - MI_ARENA_SLICE_SHIFT)); - const bool commit = (page_map_size <= 1*MI_MiB || mi_option_is_enabled(mi_option_pagemap_commit)); // _mi_os_has_overcommit(); // commit on-access on Linux systems? - const size_t commit_bits = mi_divide_up(page_map_size, MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT); - const size_t bitmap_size = (commit ? 0 : mi_bitmap_size(commit_bits, NULL)); - const size_t reserve_size = bitmap_size + page_map_size; - mi_subproc_t* const subproc = _mi_subproc_main(); - uint8_t* const base = (uint8_t*)_mi_os_alloc_aligned(subproc, reserve_size, 1, commit, true /* allow large */, &mi_page_map_memid); - if (base==NULL) { - _mi_error_message(ENOMEM, "unable to reserve virtual memory for the page map (%zu KiB)\n", page_map_size / MI_KiB); - return false; - } - if (mi_page_map_memid.initially_committed && !mi_page_map_memid.initially_zero) { - _mi_warning_message("internal: the page map was committed but not zero initialized!\n"); - mi_memzero_aligned(base, reserve_size); - } - if (bitmap_size > 0) { - mi_page_map_commit = (mi_bitmap_t*)base; - if (!_mi_os_commit(subproc, mi_page_map_commit, bitmap_size, NULL)) { - mi_page_map_cannot_commit(); - return false; - } - mi_bitmap_init(mi_page_map_commit, commit_bits, true); - } - mi_atomic_store_ptr_release(uint8_t,&_mi_page_map, base + bitmap_size); - - // commit the first part so NULL pointers get resolved without an access violation - if (!commit) { - if (!mi_page_map_ensure_committed(0, 1)) { - mi_page_map_cannot_commit(); - return false; - } - } - mi_atomic_load_ptr_relaxed(uint8_t, &_mi_page_map)[0] = 1; // so mi_ptr_page(NULL) == NULL - mi_assert_internal(mi_ptr_page(NULL)==NULL); - return true; -} - -void _mi_page_map_unsafe_destroy(void) { - mi_assert_internal(mi_atomic_load_ptr_relaxed(uint8_t, &_mi_page_map) != NULL); - if (mi_atomic_load_ptr_relaxed(uint8_t, &_mi_page_map) == NULL) return; - _mi_os_free_ex(_mi_subproc_main(), mi_page_map_memid.mem.os.base, mi_page_map_memid.mem.os.size, true, mi_page_map_memid); - mi_atomic_store_ptr_release(uint8_t, &_mi_page_map, NULL); - mi_page_map_commit = NULL; - mi_atomic_store_ptr_release(void, &_mi_page_map_max_address, NULL); - mi_page_map_memid = _mi_memid_none(); -} - - -static bool mi_page_map_ensure_committed(size_t idx, size_t slice_count) { - // is the page map area that contains the page address committed? - // we always set the commit bits so we can track what ranges are in-use. - // we only actually commit if the map wasn't committed fully already. - uint8_t* const page_map = mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map); - if (mi_page_map_commit != NULL) { - const size_t commit_idx = idx / MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT; - const size_t commit_idx_hi = (idx + slice_count - 1) / MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT; - for (size_t i = commit_idx; i <= commit_idx_hi; i++) { // per bit to avoid crossing over bitmap chunks - if (mi_bitmap_is_clear(mi_page_map_commit, i)) { - // this may race, in which case we do multiple commits (which is ok) - bool is_zero; - uint8_t* const start = page_map + (i * MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT); - const size_t size = MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT; - if (!_mi_os_commit(_mi_subproc_main(), start, size, &is_zero)) { - mi_page_map_cannot_commit(); - return false; - } - if (!is_zero && !mi_page_map_memid.initially_zero) { mi_memzero(start, size); } - mi_bitmap_set(mi_page_map_commit, i); - } - } - } - #if MI_DEBUG > 0 - page_map[idx] = 0; - page_map[idx+slice_count-1] = 0; - #endif - return true; -} - - -static size_t mi_page_map_get_idx(mi_page_t* page, uint8_t** page_start, size_t* slice_count) { - size_t page_size; - *page_start = mi_page_area(page, &page_size); - if (page_size > MI_LARGE_PAGE_SIZE) { page_size = MI_LARGE_PAGE_SIZE - MI_ARENA_SLICE_SIZE; } // furthest interior pointer - *slice_count = _mi_slice_count_of_size(page_size) + ((*page_start - mi_page_slice_start(page))/MI_ARENA_SLICE_SIZE); // add for large aligned blocks - return mi_page_map_index(page); -} - -bool _mi_page_map_register(mi_page_t* page) { - mi_assert_internal(page != NULL); - mi_assert_internal(mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); - mi_assert_internal(mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map) != NULL); // should be initialized before multi-thread access! - uint8_t* page_map = mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map); - if mi_unlikely(mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map) == NULL) { - if (!_mi_page_map_init()) return false; - page_map = mi_atomic_load_ptr_acquire(uint8_t,&_mi_page_map); - } - mi_assert(page_map!=NULL); - uint8_t* page_start; - size_t slice_count; - const size_t idx = mi_page_map_get_idx(page, &page_start, &slice_count); - - if (!mi_page_map_ensure_committed(idx, slice_count)) { - return false; - } - - // set the offsets - for (size_t i = 0; i < slice_count; i++) { - mi_assert_internal(i < 128); - page_map[idx + i] = (uint8_t)(i+1); - } - return true; -} - -void _mi_page_map_unregister(mi_page_t* page) { - uint8_t* const page_map = mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map); - mi_assert_internal(page_map != NULL); - if (page_map == NULL) return; - // get index and count - uint8_t* page_start; - size_t slice_count; - const size_t idx = mi_page_map_get_idx(page, &page_start, &slice_count); - // unset the offsets - mi_memzero(page_map + idx, slice_count); -} - -void _mi_page_map_unregister_range(void* start, size_t size) { - uint8_t* const page_map = mi_atomic_load_ptr_relaxed(uint8_t,&_mi_page_map); - mi_assert_internal(page_map!=NULL); - if (page_map == NULL) return; - const size_t slice_count = mi_divide_up(size, MI_ARENA_SLICE_SIZE); - const uintptr_t index = mi_page_map_index(start); - // todo: scan the commit bits and clear only those ranges? - if (!mi_page_map_ensure_committed(index, slice_count)) { // we commit the range in total; - return; - } - mi_memzero(&page_map[index], slice_count); -} - - -mi_page_t* _mi_safe_ptr_page(const void* p) { - if mi_unlikely(p >= mi_atomic_load_ptr_relaxed(void, &_mi_page_map_max_address)) return NULL; - const uintptr_t idx = mi_page_map_index(p); - if mi_unlikely(mi_page_map_commit != NULL && !mi_bitmap_is_set(mi_page_map_commit, idx/MI_PAGE_MAP_ENTRIES_PER_COMMIT_BIT)) return NULL; - const uintptr_t ofs = mi_page_map_at(idx); - if mi_unlikely(ofs == 0) return NULL; - return (mi_page_t*)((((uintptr_t)p >> MI_ARENA_SLICE_SHIFT) - ofs + 1) << MI_ARENA_SLICE_SHIFT); -} - -mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept { - return (_mi_safe_ptr_page(p) != NULL); -} - -#else - // A 2-level page map #define MI_PAGE_MAP_SUB_SIZE (MI_PAGE_MAP_SUB_COUNT * sizeof(mi_page_t*)) -// Use an initial empty page map so `free(NULL)` works even if mimalloc is not yet initialized (issue #1341) -static mi_page_map_t mi_page_map_empty = { - MI_ATOMIC_VAR_INIT(1), - sizeof(mi_page_map_t), - MI_MEMID_STATIC, - MI_LOCK_INITIALIZER, - { MI_ATOMIC_VAR_INIT(NULL) } -}; - -mi_decl_hidden mi_decl_cache_align _Atomic(mi_page_map_t*) __mi_page_map = MI_ATOMIC_VAR_INIT(&mi_page_map_empty); - static size_t mi_page_map_count_of_size(size_t size) { return (size < sizeof(mi_page_map_t) ? 0 : 1 + (size - sizeof(mi_page_map_t))/sizeof(mi_submap_t)); } @@ -236,6 +27,11 @@ static size_t mi_page_map_count_of_size(size_t size) { // return (void*)((uintptr_t)idx * MI_PAGE_MAP_SUB_COUNT * MI_ARENA_SLICE_SIZE); // } +// The page map with the authority to commit or free it (on CHERI) +static mi_page_map_t* mi_page_map_vmem(mi_page_map_t* pmap) { + return (mi_page_map_t*)mi_ptr_rederive(pmap->memid.mem.os.base, pmap); +} + mi_decl_nodiscard static mi_decl_noinline bool mi_page_map_commit_entries(mi_page_map_t* pmap, size_t required_idx) { const size_t reserved_count = mi_page_map_count_of_size(pmap->reserved_size); if mi_unlikely(required_idx >= reserved_count) { @@ -249,22 +45,21 @@ mi_decl_nodiscard static mi_decl_noinline bool mi_page_map_commit_entries(mi_pag mi_assert_internal(commit_count <= reserved_count); // note: we rely on uncommitted memory to be zero initialized on the first commit (and further concurrent commits leave the memory as is). bool is_zero; - if mi_unlikely(!_mi_os_commit(_mi_subproc_main(), pmap, commit_size, &is_zero)) { + if mi_unlikely(!_mi_os_commit(_mi_subproc_main(), mi_page_map_vmem(pmap), commit_size, &is_zero)) { mi_page_map_cannot_commit(); return false; } mi_assert_internal(is_zero || pmap->memid.initially_zero); - mi_atomic_store_release(&pmap->committed_count, commit_count); - // mi_atomic_store_release(&pmap->committed_addr, mi_page_map_addr_of_index(commit_count)); + mi_atomic_store_release(&_mi_process.page_map_committed, commit_count); return true; } mi_decl_nodiscard static bool mi_page_map_ensure_committed(mi_page_map_t* pmap, size_t idx, mi_submap_t* submap) { mi_assert_internal(submap!=NULL && *submap==NULL); - if mi_unlikely(idx >= mi_atomic_load_relaxed(&pmap->committed_count)) { - if (idx >= mi_atomic_load_acquire(&pmap->committed_count)) { + if mi_unlikely(idx >= mi_atomic_load_relaxed(&_mi_process.page_map_committed)) { + if (idx >= mi_atomic_load_acquire(&_mi_process.page_map_committed)) { if (!mi_page_map_commit_entries(pmap,idx)) return false; - mi_assert_internal(idx < mi_atomic_load_relaxed(&pmap->committed_count)); + mi_assert_internal(idx < mi_atomic_load_relaxed(&_mi_process.page_map_committed)); } } *submap = mi_atomic_load_ptr_acquire(mi_page_t*, &pmap->submaps[idx]); @@ -302,7 +97,7 @@ static bool mi_page_map_init_once(void) { mi_option_is_enabled(mi_option_pagemap_commit) || _mi_os_has_overcommit(); mi_subproc_t* const subproc = _mi_subproc_main(); mi_memid_t memid; - mi_page_map_t* const pmap = (mi_page_map_t*)_mi_os_alloc_aligned(subproc, extra_reserve_size, 1, commit, true /* allow large */, &memid); + mi_page_map_t* const pmap = (mi_page_map_t*)_mi_os_alloc_with_guards(subproc, extra_reserve_size, commit, true /* allow large */, &memid); if mi_unlikely(pmap==NULL) { _mi_error_message(ENOMEM, "unable to reserve virtual memory for the page map (%zu KiB)\n", extra_reserve_size / MI_KiB); return false; @@ -326,7 +121,7 @@ static bool mi_page_map_init_once(void) { bool is_zero; if (!_mi_os_commit(subproc,pmap,min_commit_size,&is_zero)) { mi_page_map_cannot_commit(); - _mi_os_free(subproc,pmap,extra_reserve_size,memid); + _mi_os_free_with_guards(subproc,memid); return false; }; mi_assert_internal(is_zero || memid.initially_zero); @@ -339,7 +134,7 @@ static bool mi_page_map_init_once(void) { if (!memid.initially_committed) { if (!_mi_os_commit(subproc, sub0, submap_size, NULL)) { // commit full submap (issue #1087) mi_page_map_cannot_commit(); - _mi_os_free(subproc,pmap,extra_reserve_size,memid); + _mi_os_free_with_guards(subproc,memid); return false; } } @@ -351,10 +146,10 @@ static bool mi_page_map_init_once(void) { pmap->memid = memid; pmap->reserved_size = reserve_size; mi_lock_init(&pmap->lock); - mi_atomic_store_release(&pmap->committed_count, commit_count); - // mi_atomic_store_release(&pmap->committed_addr, mi_page_map_addr_of_index(commit_count)); - mi_atomic_store_ptr_release(mi_page_t*, &pmap->submaps[0], sub0); - mi_atomic_store_ptr_release(mi_page_map_t, &__mi_page_map, pmap); + mi_atomic_store_ptr_release(mi_page_t*, &pmap->submaps[0], (mi_submap_t)mi_ptr_bounded_data(sub0, submap_size)); + // set the page map before the committed count as `mi_checked_ptr_page` only reads the page map if the count is non-zero + mi_atomic_store_ptr_release(mi_page_map_t, &_mi_process.page_map, (mi_page_map_t*)mi_ptr_bounded_data(pmap, reserve_size)); // see `mi_page_map_vmem` + mi_atomic_store_release(&_mi_process.page_map_committed, commit_count); mi_assert_internal(mi_ptr_page(NULL)==NULL); return true; } @@ -367,23 +162,39 @@ bool _mi_page_map_init(void) { return ok; } +// Sub maps are allocated with guard pages around them in secure mode +static mi_submap_t mi_page_map_alloc_submap(mi_subproc_t* subproc, mi_memid_t* memid) { + mi_submap_t sub = (mi_submap_t)_mi_os_alloc_with_guards(subproc, MI_PAGE_MAP_SUB_SIZE, true /* commit */, false /* allow large */, memid); + if (sub != NULL && !memid->initially_zero) { + mi_memzero_aligned(sub, MI_PAGE_MAP_SUB_SIZE); + } + return sub; +} + +// We do not store the memid of a sub map but can recreate it (see `_mi_os_alloc_with_guards`) +static mi_memid_t mi_page_map_submap_memid(mi_submap_t sub) { + const size_t guard_size = _mi_os_secure_guard_page_size(); + return _mi_memid_create_os((uint8_t*)sub - guard_size, _mi_os_good_alloc_size(MI_PAGE_MAP_SUB_SIZE + 2*guard_size), true, false, false); +} + void _mi_page_map_unsafe_destroy(void) { mi_page_map_t* const pmap = mi_page_map(); mi_assert_internal(pmap != NULL); - if (pmap == NULL || pmap == &mi_page_map_empty) return; + if (pmap == NULL) return; mi_subproc_t* const subproc = _mi_subproc_main(); mi_lock_done(&pmap->lock); - for (size_t idx = 1; idx < pmap->committed_count; idx++) { // skip entry 0 (as we allocate that submap at the end of the page_map) + const size_t committed_count = mi_atomic_load_relaxed(&_mi_process.page_map_committed); + mi_atomic_store_release(&_mi_process.page_map_committed, 0); // before freeing so `mi_checked_ptr_page` returns NULL + for (size_t idx = 1; idx < committed_count; idx++) { // skip entry 0 (as we allocate that submap at the end of the page_map) // free all sub-maps mi_submap_t sub = mi_page_map_at(pmap,idx); if (sub != NULL) { - mi_memid_t memid = _mi_memid_create_os(sub, MI_PAGE_MAP_SUB_SIZE, true, false, false); - _mi_os_free_ex(subproc, memid.mem.os.base, memid.mem.os.size, true, memid); + _mi_os_free_with_guards(subproc, mi_page_map_submap_memid(sub)); mi_atomic_store_ptr_release(mi_page_t*, &pmap->submaps[idx], NULL); } } - _mi_os_free_ex(subproc, pmap, pmap->reserved_size, true, pmap->memid); - mi_atomic_store_ptr_release(mi_page_map_t, &__mi_page_map, &mi_page_map_empty); + mi_atomic_store_ptr_release(mi_page_map_t, &_mi_process.page_map, NULL); + _mi_os_free_with_guards(subproc, pmap->memid); } mi_decl_nodiscard static mi_decl_noinline mi_submap_t mi_page_map_alloc_submap_at(mi_page_map_t* pmap, size_t idx) { @@ -396,8 +207,7 @@ mi_decl_nodiscard static mi_decl_noinline mi_submap_t mi_page_map_alloc_submap_a { mi_subproc_t* const subproc = _mi_subproc_main(); mi_memid_t memid; - const size_t submap_size = MI_PAGE_MAP_SUB_SIZE; - sub = (mi_submap_t)_mi_os_zalloc(subproc, submap_size, &memid); + sub = mi_page_map_alloc_submap(subproc, &memid); if (sub==NULL) { _mi_warning_message("internal error: unable to extend the page map\n"); } @@ -405,7 +215,7 @@ mi_decl_nodiscard static mi_decl_noinline mi_submap_t mi_page_map_alloc_submap_a mi_submap_t expect = NULL; if (!mi_atomic_cas_ptr_strong_acq_rel(mi_page_t*, &pmap->submaps[idx], &expect, sub)) { // another thread already allocated it.. free and continue - _mi_os_free(subproc, sub, submap_size, memid); + _mi_os_free_with_guards(subproc, memid); sub = expect; } } @@ -475,7 +285,7 @@ bool _mi_page_map_register(mi_page_t* page) { mi_assert_internal(pmap != NULL); // should be initialized before multi-thread access! if mi_unlikely(pmap == NULL) { if (!_mi_page_map_init()) return false; - pmap = mi_atomic_load_ptr_acquire(mi_page_map_t,&__mi_page_map); + pmap = mi_atomic_load_ptr_acquire(mi_page_map_t,&_mi_process.page_map); } mi_assert(pmap!=NULL); size_t slice_count; @@ -485,7 +295,6 @@ bool _mi_page_map_register(mi_page_t* page) { } void _mi_page_map_unregister(mi_page_t* page) { - mi_assert_internal(__mi_page_map != NULL); mi_assert_internal(page != NULL); mi_assert_internal(mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); mi_page_map_t* const pmap = mi_page_map(); @@ -504,7 +313,7 @@ void _mi_page_map_unregister_range(void* start, size_t size) { if mi_unlikely(pmap == NULL) return; const size_t slice_count = mi_divide_up(size, MI_ARENA_SLICE_SIZE); size_t sub_idx; - const uintptr_t idx = mi_page_map_index(start, &sub_idx); + const size_t idx = mi_page_map_index(start, &sub_idx); mi_page_map_set_range(pmap, NULL, idx, sub_idx, slice_count); // todo: avoid committing if not already committed? } @@ -516,5 +325,3 @@ mi_page_t* _mi_safe_ptr_page(const void* p) { mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept { return (_mi_safe_ptr_page(p) != NULL); } - -#endif diff --git a/system/lib/mimalloc/src/heap/init.c b/system/lib/mimalloc/src/heap/process.c similarity index 87% rename from system/lib/mimalloc/src/heap/init.c rename to system/lib/mimalloc/src/heap/process.c index aeff139821947..2658ebd89fc52 100644 --- a/system/lib/mimalloc/src/heap/init.c +++ b/system/lib/mimalloc/src/heap/process.c @@ -16,7 +16,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc/profile/sample-profile.h" #include // atexit -#include "mimalloc/heap/init.h" +#include "mimalloc/heap/process.h" // Empty page used to initialize the small free pages array static const mi_page_t mi_page_empty = { @@ -32,7 +32,7 @@ static const mi_page_t mi_page_empty = { 0, // xlast_used 0, // xlast_alloc #endif - 0, // page_offset + NULL, // page_start 0, // capacity 0, // reserved capacity 0, // slice_pcommitted @@ -51,8 +51,8 @@ static const mi_page_t mi_page_empty = { #endif #endif #if MI_QSBR - NULL, NULL, // qsbr_next, qsbr_prev - false, // qsbr_in_queue + MI_ATOMIC_VAR_INIT(0), // qsbr_xthread_id + NULL, NULL, // qsbr_next, qsbr_prev mi_init_struct_zero, // qsbr #endif }; @@ -77,7 +77,7 @@ static const mi_page_t mi_page_empty = { // Empty page queues for every bin -#define QNULL(sz) { NULL, NULL, 0, (sz)*sizeof(uintptr_t), 0 } +#define QNULL(sz) { NULL, NULL, 0, (sz)*sizeof(size_t), 0 } #define MI_PAGE_QUEUES_EMPTY \ { QNULL(1), \ QNULL( 1), QNULL( 2), QNULL( 3), QNULL( 4), QNULL( 5), QNULL( 6), QNULL( 7), QNULL( 8), /* 8 */ \ @@ -118,9 +118,11 @@ static const mi_page_t mi_page_empty = { // may lead to allocation itself on some platforms) // -------------------------------------------------------- -static mi_decl_cache_align mi_tld_t mi_tld_detached = { - MI_THREADID_DETACHED, // thread_id +// read-only detached tld of the empty theap +static mi_decl_cache_align const mi_tld_t mi_tld_empty = { + MI_THREAD_ID_DETACHED, // thread_id 0, // thread_seq + 0, // fork_epoch 0, // default numa node NULL, // subproc NULL, // theaps list @@ -136,7 +138,7 @@ extern // fix warning in C++ for const variables with internal linkage #endif mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = { MI_SMALL_PAGES_EMPTY, // direct small pages - &mi_tld_detached, // tld + (mi_tld_t*)&mi_tld_empty, // tld MI_ATOMIC_VAR_INIT(NULL), // heap MI_ATOMIC_VAR_INIT(NULL), // subproc MI_ATOMIC_VAR_INIT(1), // refcount @@ -145,6 +147,7 @@ mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = { true, // allow abandon true, // is_detached true, // profile_never: this static empty theap must never be sampled + false, false, // guard_front, guard_end ~MI_ZU(0), // sample countdown: "-1" (with a sample rate of 0, so we won't write to the empty theap with MI_SAMPLE==2) 0, 0, // sample rate, requested 0, 0, // profile rate, countdown @@ -170,16 +173,15 @@ mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = { #undef MI_STAT_COUNTER -// pre-allocate the process heap, and meta-data theap -static mi_decl_cache_align mi_heap_t mi_process_heap_main = mi_init_struct_zero; -static mi_decl_cache_align mi_theap_t mi_process_theap_meta = mi_init_struct_zero; +// pre-allocate the process data +mi_decl_hidden mi_decl_cache_align mi_process_t _mi_process = mi_init_struct_zero; -// pre-allocate the initial tld and theap for the main thread (this is not strictly needed but nice for stats) -static mi_decl_cache_align mi_tld_t mi_process_tld_main = mi_init_struct_zero; -static mi_decl_cache_align mi_theap_t mi_process_theap_main = mi_init_struct_zero; - -mi_decl_hidden mi_decl_cache_align mi_theap_t _mi_theap_empty_wrong = mi_init_struct_zero; // used for error paths -mi_decl_hidden bool _mi_process_is_initialized = false; // set to `true` in `mi_process_init`. +// In secure mode, surround the process data with guard pages to protect against overflows of adjacent static data +static void mi_process_guard_init(void) { + const size_t front = sizeof(_mi_process.guard_front); + const size_t size = offsetof(mi_process_t, guard_end) - front; + _mi_os_secure_guard_pages_set_within(&_mi_process, sizeof(_mi_process), front, size); +} mi_page_t* _mi_page_empty_get(void) { @@ -205,31 +207,30 @@ static mi_tld_t* mi_tld_init(mi_tld_t* tld, size_t tseq, mi_subproc_t* subproc); // Initialize main heap static void mi_heap_main_init_once(void) { mi_memid_t memid_static = _mi_memid_create(MI_MEM_STATIC); - mi_memcpy(&_mi_theap_empty_wrong,&_mi_theap_empty,sizeof(_mi_theap_empty_wrong)); + mi_memcpy(&_mi_process.theap_empty_wrong,&_mi_theap_empty,sizeof(_mi_process.theap_empty_wrong)); // initialize the main subprocess mi_subproc_t* subproc_main = _mi_subproc_main_init(); - // detached tld for mi_theap_empty (and theap_meta) - mi_tld_detached.memid = memid_static; - mi_tld_init(&mi_tld_detached, 0, subproc_main); + // detached tld for theap_meta + mi_tld_t* const tld_meta = &_mi_process.tld_meta; + tld_meta->thread_id = MI_THREAD_ID_DETACHED; + tld_meta->memid = memid_static; + mi_tld_init(tld_meta, 0, subproc_main); // main process heap - mi_process_heap_main.memid = memid_static; - mi_atomic_store_ptr_release(mi_heap_t,&subproc_main->heap_main,&mi_process_heap_main); - _mi_heap_init(&mi_process_heap_main,mi_thread_local_key_fast,subproc_main,0,false); + mi_heap_t* const heap_main = &_mi_process.heap_main; + heap_main->memid = memid_static; + mi_atomic_store_ptr_release(mi_heap_t,&subproc_main->heap_main,heap_main); + _mi_heap_init(heap_main,mi_thread_local_key_fast,subproc_main,0,false); // detached theap for allocating meta-data (we can allocate on this without having an initialized thread) - mi_process_theap_meta.memid = memid_static; - _mi_theap_init(&mi_process_theap_meta,&mi_process_heap_main,&mi_tld_detached); - mi_process_theap_meta.allow_page_abandon = false; // for security, don't share with other threads - mi_process_theap_meta.page_full_retain = 2; - mi_process_theap_meta.sample_rate = 0; // no sampling for meta data - mi_process_theap_meta.sample_countdown = 0; - mi_theap_profile_never(&mi_process_theap_meta); // permanently exclude from profiling, see `mi_theap_profile_never` - subproc_main->theap_meta = &mi_process_theap_meta; - - // mi_heap_theap_set(&mi_process_heap_main,&mi_process_theap_main); // set in `mi_thread_init(_theap_default)` + mi_theap_t* const theap_meta = &_mi_process.theap_meta; + theap_meta->memid = memid_static; + _mi_theap_meta_init(theap_meta,heap_main,tld_meta); + subproc_main->theap_meta = theap_meta; + + // mi_heap_theap_set(heap_main,&_mi_process.theap_main); // set in `mi_thread_init(_theap_default)` } static void mi_heap_main_init(void) { @@ -250,7 +251,7 @@ mi_heap_t* _mi_subproc_heap_main(mi_subproc_t* subproc) { } else { mi_assert_internal(false); - return &mi_process_heap_main; + return &_mi_process.heap_main; } } @@ -260,7 +261,7 @@ mi_heap_t* _mi_subproc_heap_main(mi_subproc_t* subproc) { mi_theap_t* _mi_theap_main(void) { // All subprocess main heaps share this slot; do not follow a possibly deleted default theap. - const mi_thread_local_t key = mi_process_heap_main.theap; + const mi_thread_local_t key = _mi_process.heap_main.theap; return (key==0 ? NULL : (mi_theap_t*)_mi_thread_local_get(key)); } @@ -268,7 +269,7 @@ static mi_tld_t* mi_tld_init(mi_tld_t* tld, size_t tseq, mi_subproc_t* subproc) tld->subproc = subproc; tld->theaps = NULL; mi_lock_init(&tld->theaps_lock); - if (tld->thread_id == MI_THREADID_DETACHED) { + if (tld->thread_id == MI_THREAD_ID_DETACHED) { tld->numa_node = -1; } else { @@ -276,6 +277,7 @@ static mi_tld_t* mi_tld_init(mi_tld_t* tld, size_t tseq, mi_subproc_t* subproc) tld->thread_id = _mi_prim_thread_id(); tld->is_in_threadpool = _mi_prim_thread_is_in_threadpool(); tld->thread_seq = tseq; + tld->fork_epoch = _mi_fork_epoch(); mi_atomic_increment_relaxed(&tld->subproc->thread_count); } return tld; @@ -289,7 +291,7 @@ static mi_tld_t* mi_tld_create(mi_subproc_t* subproc) { mi_memid_t memid; mi_tld_t* tld; if (_mi_subproc_is_main(subproc) && tseq==0 /* first tld */) { - tld = &mi_process_tld_main; + tld = &_mi_process.tld_main; memid = _mi_memid_create_static(tld,sizeof(*tld)); } else { @@ -306,7 +308,7 @@ static mi_tld_t* mi_tld_create(mi_subproc_t* subproc) { mi_decl_noinline static void mi_tld_free(mi_tld_t* tld) { if (tld==NULL) return; mi_atomic_decrement_relaxed(&tld->subproc->thread_count); - tld->thread_id = (mi_threadid_t)(~0); // it is best to set an invalid tid for tld_main as sometimes the same thread-id + tld->thread_id = (mi_thread_id_t)(~0); // it is best to set an invalid tid for tld_main as sometimes the same thread-id // is reused by the OS after a thread has terminated. (see issue #1287) mi_lock_done(&tld->theaps_lock); _mi_meta_free(tld->subproc, tld, tld->memid); // note: safe for static tld @@ -319,11 +321,11 @@ mi_decl_noinline static void mi_tld_free(mi_tld_t* tld) { #if MI_DEBUG || defined(MI_TLS_RECURSE_GUARD) static mi_theap_t* mi_heap_check_for_existing_theap(mi_heap_t* heap) { - const mi_threadid_t tid = _mi_thread_id(); + const mi_thread_id_t tid = _mi_thread_id(); mi_theap_t* thread_theap = NULL; mi_lock(&heap->theaps_lock) { for(mi_theap_t* theap = heap->theaps; theap != NULL; theap = theap->hnext ) { - if (theap->tld->thread_id == tid) { + if (theap->tld->thread_id == tid && theap->tld->fork_epoch == _mi_fork_epoch()) { // a dead theap in a forked child can have the same id thread_theap = theap; break; } @@ -348,7 +350,7 @@ mi_theap_t* _mi_thread_init_with_heap(mi_heap_t* heap_main) // (on macOS <= 14 for example where the loader allocates thread-local data on demand). if (heap_main==NULL) { heap_main = mi_heap_main(); - mi_assert_internal(heap_main == &mi_process_heap_main); + mi_assert_internal(heap_main == &_mi_process.heap_main); } mi_assert_internal(heap_main!=NULL); @@ -366,8 +368,8 @@ mi_theap_t* _mi_thread_init_with_heap(mi_heap_t* heap_main) mi_tld_t* tld = mi_tld_create(heap_main->subproc); if (tld==NULL) return NULL; // out-of-memory on tld allocation // allocate and initialize the theap for the main heap - if (tld==&mi_process_tld_main) { - theap = &mi_process_theap_main; // initial theap is pre-allocated + if (tld==&_mi_process.tld_main) { + theap = &_mi_process.theap_main; // initial theap is pre-allocated theap->memid = _mi_memid_create_static(theap,sizeof(*theap)); } else { @@ -387,6 +389,7 @@ mi_theap_t* _mi_thread_init_with_heap(mi_heap_t* heap_main) mi_assert_internal(heap_main==NULL || heap_theap == theap); MI_UNUSED_RELEASE(heap_theap); mi_subproc_stat_increase(mi_theap_subproc(theap), threads, 1); // or theap stats and wait for merge? + _mi_fork_child_try_collect(); // in a forked child, abandon dead theaps as they may have the same thread id as this thread // _mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id()); return theap; } @@ -529,11 +532,9 @@ void mi_thread_set_in_threadpool(void) mi_attr_noexcept { // Process init and done // -------------------------------------------------------- -static bool os_preloading = true; // true until this module is initialized - // Returns true if this module has not been initialized; Don't use C runtime routines until it returns false. bool mi_decl_noinline _mi_preloading(void) { - return os_preloading; + return !_mi_process.libc_is_initialized; } // Returns true if mimalloc was redirected @@ -543,7 +544,7 @@ mi_decl_nodiscard bool mi_is_redirected(void) mi_attr_noexcept { // Called once by the process loader from `src/prim/prim.c` before `main` is called. void _mi_auto_process_init(void) { - os_preloading = false; + _mi_process.libc_is_initialized = true; mi_process_init(); @@ -585,6 +586,7 @@ static void mi_process_init_once(void) { _mi_options_init(); // read environment (if possible) _mi_stats_init(); // start timer _mi_os_init(); // primitive dependent + mi_process_guard_init(); // after os_init so the OS page size is known mi_heap_main_init(); // before page_map_init so stats are working _mi_page_map_init(); // todo: this could fail.. should we abort in that case? @@ -593,7 +595,7 @@ static void mi_process_init_once(void) { // the following can potentially allocate (on freeBSD for pthread keys) _mi_tls_slots_init(); // pthread key create _mi_thread_locals_init(); // pthread key create - _mi_process_is_initialized = true; // before `mi_process_setup_auto_thread_done` so `_mi_theap_default` returns the current theap with `MI_TLS_RECURSE_GUARD` + _mi_process.process_is_initialized = true; // before `mi_process_setup_auto_thread_done` so `_mi_theap_default` returns the current theap with `MI_TLS_RECURSE_GUARD` mi_process_setup_auto_thread_done(); // after the above mi_thread_init so it can add the current theap _mi_fork_init(); // register fork handlers (can allocate) @@ -682,30 +684,25 @@ static void mi_process_unwind_once(void) // Called when the process is done static void mi_process_done_once(void) { // only shutdown if we were initialized - if (!_mi_process_is_initialized) return; - // ensure we are called once - static bool process_done = false; - if (process_done) return; - process_done = true; + if (!_mi_process.process_is_initialized) return; const size_t thread_count = mi_atomic_load_acquire(&_mi_subproc_main()->thread_count); if (thread_count == 1 || !_mi_prim_process_is_killed()) { // can we safely take locks? mi_process_unwind_once(); - _mi_verbose_message("process done: 0x%zx\n", mi_process_tld_main.thread_id); + _mi_verbose_message("process done: 0x%zx\n", _mi_process.tld_main.thread_id); } _mi_subproc_main_done(); _mi_allocator_done(); - os_preloading = true; // don't call the C runtime anymore + _mi_process.libc_is_initialized = false; // don't call the C runtime anymore } // Call when the process is done (cdecl as it is used with `at_exit` on some platforms) void mi_cdecl mi_process_done(void) mi_attr_noexcept { // we don't use `mi_atomic_do_once` here to avoid deadlocks if threads are killed due to a forced exit (issue #1377) - static _Atomic(size_t) mi_process_is_done; size_t expected = 0; - if (mi_atomic_cas_strong_acq_rel(&mi_process_is_done, &expected, 1)) { + if (mi_atomic_cas_strong_acq_rel(&_mi_process.process_done_called, &expected, 1)) { mi_process_done_once(); } } diff --git a/system/lib/mimalloc/src/heap/readme.md b/system/lib/mimalloc/src/heap/readme.md index 66d4b2b778432..96d4506973713 100644 --- a/system/lib/mimalloc/src/heap/readme.md +++ b/system/lib/mimalloc/src/heap/readme.md @@ -2,7 +2,7 @@ - [`heap.c`](heap.c): First-class heap creation, destruction, ownership, and management. - [`theap.c`](theap.c): Thread-local heap instances and page collection. -- [`init.c`](init.c): Process and thread initialization, shutdown, and initial allocator state. +- [`process.c`](process.c): Process and thread initialization, shutdown, and initial allocator state. - [`subproc.c`](subproc.c): Subprocess allocation domains, metadata allocation, and subprocess lifecycle management. - [`page-map.c`](page-map.c): Address-to-page mapping to safely map any pointer to a valid mimalloc page (or `NULL`). diff --git a/system/lib/mimalloc/src/heap/subproc.c b/system/lib/mimalloc/src/heap/subproc.c index a55b7182473bf..3b8edb59ea9a8 100644 --- a/system/lib/mimalloc/src/heap/subproc.c +++ b/system/lib/mimalloc/src/heap/subproc.c @@ -13,13 +13,11 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc/heap/page-map.h" #include "mimalloc/profile/sample-profile.h" #include "mimalloc/heap/subproc.h" +#include "mimalloc/heap/process.h" +#include "mimalloc/alloc/page.h" #include "mimalloc/util/threadlocal.h" -// pre-allocate the main subprocess structure. -static mi_decl_cache_align mi_subproc_t mi_process_subproc_main = mi_init_struct_zero; -static mi_subproc_t* mi_subprocs = NULL; -static mi_lock_t mi_subprocs_lock = MI_LOCK_INITIALIZER; - +// the list of sub-processes is in `_mi_process.subprocs` (and the main subprocess is statically allocated in `_mi_process`) /* ----------------------------------------------------------- Meta-data allocation @@ -99,11 +97,11 @@ bool _mi_meta_is_meta_page(const mi_subproc_t* subproc, const mi_page_t* page) { ----------------------------------------------------------- */ mi_subproc_t* _mi_subproc_main(void) { - return &mi_process_subproc_main; + return &_mi_process.subproc_main; } bool _mi_subproc_is_main(mi_subproc_t* subproc) { - return (subproc == &mi_process_subproc_main); + return (subproc == &_mi_process.subproc_main); } mi_subproc_t* _mi_subproc(void) { @@ -145,18 +143,17 @@ mi_subproc_id_t mi_subproc_current(void) { static mi_subproc_t* mi_subproc_init(mi_subproc_t* subproc, mi_subproc_t* parent) { - static _Atomic(size_t) subproc_total_count; subproc->parent = parent; - subproc->subproc_seq = mi_atomic_increment_relaxed(&subproc_total_count); + subproc->subproc_seq = mi_atomic_increment_relaxed(&_mi_process.subprocs_created); mi_stats_header_init(&subproc->stats); mi_lock_init(&subproc->arena_reserve_lock); mi_lock_init(&subproc->heaps_lock); mi_lock_init(&subproc->theap_meta_lock); - mi_lock(&mi_subprocs_lock) { + mi_lock(&_mi_process.subprocs_lock) { // push on subproc list - subproc->next = mi_subprocs; - if (mi_subprocs!=NULL) { mi_subprocs->prev = subproc; } - mi_subprocs = subproc; + subproc->next = _mi_process.subprocs; + if (_mi_process.subprocs!=NULL) { _mi_process.subprocs->prev = subproc; } + _mi_process.subprocs = subproc; } return subproc; } @@ -175,7 +172,7 @@ mi_subproc_id_t mi_subproc_new(void) { _mi_meta_free(parent, subproc, memid); return _mi_subproc_to_id(NULL); } - theap_meta->memid = memid; + theap_meta->memid = theap_memid; // init subproc mi_subproc_init(subproc,parent); @@ -192,9 +189,8 @@ mi_subproc_id_t mi_subproc_new(void) { // init meta theap mi_assert_internal(parent->theap_meta!=NULL); mi_assert_internal(parent->theap_meta->tld!=NULL); - mi_assert_internal(parent->theap_meta->tld->thread_id == MI_THREADID_DETACHED); - _mi_theap_init(theap_meta,heap_main,parent->theap_meta->tld /* detached tld */); - mi_theap_profile_never(theap_meta); // permanently exclude from profiling, see `mi_theap_profile_never` + mi_assert_internal(parent->theap_meta->tld->thread_id == MI_THREAD_ID_DETACHED); + _mi_theap_meta_init(theap_meta,heap_main,parent->theap_meta->tld /* detached tld */); subproc->theap_meta = theap_meta; return _mi_subproc_to_id(subproc); @@ -211,10 +207,10 @@ static void mi_subproc_unsafe_destroy(mi_subproc_t* subproc, bool acquire_subpro if (subproc==NULL) return; // remove from the subproc list - mi_lock_maybe(&mi_subprocs_lock, acquire_subprocs_lock) { + mi_lock_maybe(&_mi_process.subprocs_lock, acquire_subprocs_lock) { if (subproc->next!=NULL) { subproc->next->prev = subproc->prev; } if (subproc->prev!=NULL) { subproc->prev->next = subproc->next; } - else { mi_assert_internal(mi_subprocs==subproc); mi_subprocs = subproc->next; } + else { mi_assert_internal(_mi_process.subprocs==subproc); _mi_process.subprocs = subproc->next; } } // destroy all subproc heaps @@ -239,7 +235,7 @@ static void mi_subproc_unsafe_destroy(mi_subproc_t* subproc, bool acquire_subpro if (!_mi_subproc_is_main(subproc)) { // merge stats back into the main subproc - _mi_stats_merge_into(&mi_process_subproc_main.stats, &subproc->stats); + _mi_stats_merge_into(&_mi_process.subproc_main.stats, &subproc->stats); } // remove associated arenas @@ -265,22 +261,22 @@ static void mi_subproc_unsafe_destroy(mi_subproc_t* subproc, bool acquire_subpro void mi_subproc_destroy(mi_subproc_id_t subproc_id) { mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); - if (subproc==NULL || subproc==&mi_process_subproc_main) return; + if (subproc==NULL || subproc==&_mi_process.subproc_main) return; mi_subproc_unsafe_destroy(subproc, true /* take lock */); } void _mi_subprocs_unsafe_destroy_all(void) { - mi_lock(&mi_subprocs_lock) { - mi_subproc_t* subproc = mi_subprocs; + mi_lock(&_mi_process.subprocs_lock) { + mi_subproc_t* subproc = _mi_process.subprocs; while (subproc!=NULL) { mi_subproc_t* next = subproc->next; - if (subproc!=&mi_process_subproc_main) { - mi_subproc_unsafe_destroy(subproc, false /* take mi_subprocs lock */); + if (subproc!=&_mi_process.subproc_main) { + mi_subproc_unsafe_destroy(subproc, false /* take subprocs lock */); } subproc = next; } } - mi_subproc_unsafe_destroy(&mi_process_subproc_main, true /* take mi_subprocs lock */); + mi_subproc_unsafe_destroy(&_mi_process.subproc_main, true /* take subprocs lock */); } @@ -321,69 +317,152 @@ bool mi_subproc_visit_heaps(mi_subproc_id_t subproc_id, mi_heap_visit_fun* visit mi_subproc_t* _mi_subproc_main_init(void) { - mi_lock_init(&mi_subprocs_lock); - mi_memid_t memid = _mi_memid_create_static(&mi_process_subproc_main,sizeof(mi_subproc_t)); - mi_process_subproc_main.memid = memid; - mi_subproc_init(&mi_process_subproc_main,NULL); - return &mi_process_subproc_main; + mi_lock_init(&_mi_process.subprocs_lock); + mi_subproc_t* const subproc = &_mi_process.subproc_main; + subproc->memid = _mi_memid_create_static(subproc,sizeof(mi_subproc_t)); + mi_subproc_init(subproc,NULL); + return subproc; } void _mi_subproc_main_done(void) { - mi_lock_done(&mi_subprocs_lock); + mi_lock_done(&_mi_process.subprocs_lock); } /* ----------------------------------------------------------- Fork support - `fork()` is only safe if no other thread is executing inside mimalloc - at the time of the fork; all other threads must be quiesced at a safe - point outside mimalloc (as is done for example by CPython's stop-the-world). - The child then inherits consistent data structures with no locks held - (as no thread was inside mimalloc) so we only need to abandon the pages - of the threads that do not exist in the child. + In a forked child only the forking thread exists, so the theaps of all other + threads are dead. The fork handler only marks this and the pages of the dead + theaps are abandoned later on the generic allocation path. This is required as + other threads may have been inside mimalloc at the time of the fork, leaving their + data structures inconsistent; a child that only calls `exec` or `_exit` (as POSIX + requires for multi-threaded programs) then never touches them. + If a child keeps allocating, it should only fork when all other threads are + quiesced at a safe point outside mimalloc (as is done for example by CPython's + stop-the-world). If a dead thread held a lock at the fork, its pages are leaked. + + On macOS, the child can see a torn snapshot of memory that other threads wrote + concurrently with the fork (across separate mappings). In particular, the thread + free lists of the pages of the forking thread, and abandoned pages, may be torn. + libSystem allocates in the child before the pthread child handlers run, so the malloc + zone `reinit_lock` (which runs before that) drops (and leaks) the thread free lists of + the forking thread, and suspends reclaiming abandoned pages until the pthread child + handler. (So on macOS the zone override is required for fork support.) + It also marks the child as torn, since free arena slices may still have a guard + page of a page that was allocated concurrently with the fork (see `arena-page.c`). ----------------------------------------------------------- */ -// Abandon the pages of theaps whose threads do not exist in the child. + +size_t _mi_fork_epoch(void) { + return _mi_process.fork.epoch; +} + +bool _mi_fork_is_torn(void) { + return _mi_process.fork.torn; +} + +static bool mi_fork_theap_is_dead(const mi_theap_t* theap) { + const mi_tld_t* const tld = theap->tld; + return (tld != NULL && tld->thread_id != MI_THREAD_ID_DETACHED && tld->fork_epoch != _mi_process.fork.epoch); +} + +// Abandon the pages of dead theaps in a heap. // The theap and tld memory itself is not freed as the (dead) thread lists may be inconsistent. -// We need to take no locks as at this point only the child process runs in a single thread. -static void mi_fork_child_abandon_theaps(void) { - mi_theap_t* const theap_default = _mi_theap_default(); - mi_tld_t* const tld_current = (mi_theap_is_initialized(theap_default) ? theap_default->tld : NULL); - mi_theap_t* dead = NULL; - for (mi_subproc_t* subproc = mi_subprocs; subproc != NULL; subproc = subproc->next) { - for (mi_heap_t* heap = subproc->heaps; heap != NULL; heap = heap->next) { - mi_theap_t* theap = heap->theaps; - while (theap != NULL) { - mi_theap_t* const next = theap->hnext; - mi_tld_t* const tld = theap->tld; - if (tld != NULL && tld != tld_current && tld->thread_id != MI_THREADID_DETACHED) { - // remove from the heap list and push on the dead list - if (theap->hnext != NULL) { theap->hnext->hprev = theap->hprev; } - if (theap->hprev != NULL) { theap->hprev->hnext = theap->hnext; } - else { heap->theaps = theap->hnext; } - theap->hprev = NULL; - theap->hnext = dead; - dead = theap; +static bool mi_fork_heap_abandon_dead(mi_heap_t* heap) { + if (!mi_lock_try_acquire(&heap->theaps_lock)) return false; + mi_theap_t* theap = heap->theaps; + while (theap != NULL) { + mi_theap_t* const next = theap->hnext; + if (mi_fork_theap_is_dead(theap)) { + if (theap->hnext != NULL) { theap->hnext->hprev = theap->hprev; } + if (theap->hprev != NULL) { theap->hprev->hnext = theap->hnext; } + else { heap->theaps = theap->hnext; } + theap->hnext = NULL; + theap->hprev = NULL; + theap->tld = NULL; + _mi_theap_abandon_owned_pages(theap); + _mi_stats_merge_into(&heap->stats, &theap->stats); + mi_atomic_store_ptr_release(mi_heap_t, &theap->heap, NULL); + } + theap = next; + } + mi_lock_release(&heap->theaps_lock); + return true; +} + +// Called at thread initialization and regularly on the generic allocation path. +// Retried later if a lock is busy. +void _mi_fork_child_try_collect(void) { + if mi_likely(mi_atomic_load_relaxed(&_mi_process.fork.pending) == 0) return; + if (mi_atomic_exchange_acq_rel(&_mi_process.fork.pending, 0) == 0) return; + bool done = false; + if (mi_lock_try_acquire(&_mi_process.subprocs_lock)) { + done = true; + for (mi_subproc_t* subproc = _mi_process.subprocs; subproc != NULL; subproc = subproc->next) { + if (mi_lock_try_acquire(&subproc->heaps_lock)) { + for (mi_heap_t* heap = subproc->heaps; heap != NULL; heap = heap->next) { + if (!mi_fork_heap_abandon_dead(heap)) { done = false; } } - theap = next; + mi_lock_release(&subproc->heaps_lock); + } + else { + done = false; } } + mi_lock_release(&_mi_process.subprocs_lock); } - while (dead != NULL) { - mi_theap_t* const next = dead->hnext; - mi_heap_t* const heap = mi_theap_heap(dead); - dead->hnext = NULL; - dead->tld = NULL; - _mi_theap_abandon_owned_pages(dead); - _mi_stats_merge_into(&heap->stats, &dead->stats); - mi_atomic_store_ptr_release(mi_heap_t, &dead->heap, NULL); - dead = next; + if (!done) { mi_atomic_store_release(&_mi_process.fork.pending, 1); } +} + +#if defined(__APPLE__) +// Called on macOS in the child from the malloc zone `reinit_lock`. +void _mi_fork_child_zone(void) { + _mi_process.fork.torn = true; + mi_theap_t* const theap_default = _mi_theap_default(); + if (!mi_theap_is_initialized(theap_default)) return; + + // suspend reclaiming abandoned pages (by allocation) of the default heap + mi_heap_t* const heap = mi_theap_heap(theap_default); + for (size_t bin = 0; bin < MI_BIN_COUNT; bin++) { + _mi_process.fork.suspended_count[bin] = mi_atomic_exchange_relaxed(&heap->abandoned_count[bin], 0); + } + _mi_process.fork.suspended_heap = heap; + + // drop the thread free lists of the pages of the forking thread + mi_tld_t* const tld = theap_default->tld; + if (!mi_lock_try_acquire(&tld->theaps_lock)) return; + for (mi_theap_t* theap = tld->theaps; theap != NULL; theap = theap->tnext) { + for (size_t bin = 0; bin <= MI_BIN_FULL; bin++) { + for (mi_page_t* page = theap->pages[bin].first; page != NULL; page = page->next) { + const mi_thread_free_t tf = mi_atomic_load_relaxed(&page->xthread_free); + mi_atomic_store_relaxed(&page->xthread_free, mi_tf_create(NULL, mi_tf_is_owned(tf))); + } + } + } + mi_lock_release(&tld->theaps_lock); +} + +// Resume reclaiming abandoned pages (adding as the count may have changed in the meantime) +static void mi_fork_child_resume_reclaim(void) { + mi_heap_t* const heap = _mi_process.fork.suspended_heap; + if (heap == NULL) return; + _mi_process.fork.suspended_heap = NULL; + for (size_t bin = 0; bin < MI_BIN_COUNT; bin++) { + mi_atomic_add_relaxed(&heap->abandoned_count[bin], _mi_process.fork.suspended_count[bin]); } } +#endif +// Called in the child process right after a fork. void _mi_fork_child(void) { - mi_fork_child_abandon_theaps(); + #if defined(__APPLE__) + mi_fork_child_resume_reclaim(); + #endif + _mi_process.fork.epoch++; + mi_theap_t* const theap = _mi_theap_default(); + if (mi_theap_is_initialized(theap)) { theap->tld->fork_epoch = _mi_process.fork.epoch; } // the forking thread survives + mi_atomic_store_release(&_mi_process.fork.pending, 1); } // Called at process initialization. diff --git a/system/lib/mimalloc/src/heap/theap.c b/system/lib/mimalloc/src/heap/theap.c index 8266bd01cf7ba..d52e89438394a 100644 --- a/system/lib/mimalloc/src/heap/theap.c +++ b/system/lib/mimalloc/src/heap/theap.c @@ -13,6 +13,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc/alloc/page.h" #include "mimalloc/arena/arena-alloc.h" #include "mimalloc/profile/sample-guarded.h" +#include "mimalloc/profile/sample-profile.h" #include "mimalloc/heap/heap.h" #include "mimalloc/heap/theap.h" @@ -222,7 +223,7 @@ static void mi_theap_options_init(mi_theap_t* theap, mi_heap_t* heap) { theap->allow_page_reclaim = (mi_heap_reclaim_on_free(heap) >= 0); theap->allow_page_abandon = (mi_option_get(mi_option_page_full_retain) >= 0); // nowadays qsbr pages can be abandoned: && !mi_heap_qsbr_is_enabled(heap)); theap->page_full_retain = mi_option_get_clamp(mi_option_page_full_retain, -1, 32); - theap->is_detached = (theap->tld->thread_id == MI_THREADID_DETACHED); + theap->is_detached = (theap->tld->thread_id == MI_THREAD_ID_DETACHED); } // todo: make order of parameters consistent (but would that break compat with CPython?) @@ -240,6 +241,8 @@ void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld) mi_atomic_store_ptr_release(mi_subproc_t,&theap->subproc,heap->subproc); mi_assert_internal(theap->stats.size == sizeof(mi_stats_t)); mi_theap_options_init(theap, heap); + theap->guard_front = false; + theap->guard_end = (MI_SECURE >= 5); // 5 = guard page at the end of each page if (theap->tld->is_in_threadpool) { // if we run as part of a thread pool it is better to not arbitrarily reclaim abandoned pages into our theap. @@ -302,10 +305,22 @@ void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld) } } +// Initialize a detached theap for meta data allocation +void _mi_theap_meta_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld) { + _mi_theap_init(theap, heap, tld); + theap->allow_page_abandon = false; // for security, don't share with other threads + theap->page_full_retain = 2; + theap->sample_rate = 0; // no sampling for meta data + theap->sample_countdown = 0; + theap->guard_front = (MI_SECURE > 0); + theap->guard_end = (MI_SECURE > 0); + mi_theap_profile_never(theap); // permanently exclude from profiling, see `mi_theap_profile_never` +} + mi_theap_t* _mi_theap_alloc(mi_heap_t* heap, mi_tld_t* tld) { mi_assert_internal(tld!=NULL); mi_assert_internal(heap!=NULL); - mi_assert_internal(tld->thread_id == MI_THREADID_DETACHED || _mi_thread_id() == tld->thread_id); + mi_assert_internal(tld->thread_id == MI_THREAD_ID_DETACHED || _mi_thread_id() == tld->thread_id); // mi_assert_internal(_mi_heap_theap_peek(heap)==NULL); // don't access thread locals as this is called on thread init // allocate and initialize a theap @@ -598,11 +613,11 @@ void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page) { const size_t ubsize = mi_page_usable_block_size(page); area->reserved = page->reserved * bsize; area->committed = page->capacity * bsize; - area->blocks = mi_page_start(page); + area->blocks = mi_ptr_bounded_data(mi_page_start(page), area->reserved); area->used = mi_page_used(page); // number of blocks in use (#553) area->block_size = ubsize; area->full_block_size = bsize; - area->reserved1 = page; + area->reserved1 = NULL; } static void mi_get_fast_divisor(size_t divisor, uint64_t* magic, size_t* shift) { @@ -636,7 +651,7 @@ bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi // optimize page with one block if (page->capacity == 1) { mi_assert_internal(mi_page_used(page) == 1 && page->free == NULL); - return visitor(heap, area, pstart, ubsize, arg); + return visitor(heap, area, mi_ptr_bounded_data(pstart, ubsize), ubsize, arg); } mi_assert(bsize <= UINT32_MAX); @@ -644,7 +659,7 @@ bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi if (mi_page_used(page) == page->capacity) { uint8_t* block = pstart; for (size_t i = 0; i < page->capacity; i++) { - if (!visitor(heap, area, block, ubsize, arg)) return false; + if (!visitor(heap, area, mi_ptr_bounded_data(block, ubsize), ubsize, arg)) return false; block += bsize; } return true; @@ -652,13 +667,13 @@ bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi // create a bitmap of free blocks. #define MI_MAX_BLOCKS (MI_SMALL_PAGE_SIZE / sizeof(void*)) - uintptr_t free_map[MI_MAX_BLOCKS / MI_INTPTR_BITS]; - const uintptr_t bmapsize = mi_divide_up(page->capacity, MI_INTPTR_BITS); - memset(free_map, 0, bmapsize * sizeof(intptr_t)); - if (page->capacity % MI_INTPTR_BITS != 0) { + size_t free_map[MI_MAX_BLOCKS / MI_SIZE_BITS]; + const size_t bmapsize = mi_divide_up(page->capacity, MI_SIZE_BITS); + memset(free_map, 0, bmapsize * sizeof(size_t)); + if (page->capacity % MI_SIZE_BITS != 0) { // mark left-over bits at the end as free - size_t shift = (page->capacity % MI_INTPTR_BITS); - uintptr_t mask = (UINTPTR_MAX << shift); + size_t shift = (page->capacity % MI_SIZE_BITS); + size_t mask = (SIZE_MAX << shift); free_map[bmapsize - 1] = mask; } @@ -681,9 +696,9 @@ bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi size_t blockidx = mi_fast_divide(offset, magic, shift); mi_assert_internal(blockidx == offset / bsize); mi_assert_internal(blockidx < MI_MAX_BLOCKS); - size_t bitidx = (blockidx / MI_INTPTR_BITS); - size_t bit = blockidx - (bitidx * MI_INTPTR_BITS); - free_map[bitidx] |= ((uintptr_t)1 << bit); + size_t bitidx = (blockidx / MI_SIZE_BITS); + size_t bit = blockidx - (bitidx * MI_SIZE_BITS); + free_map[bitidx] |= ((size_t)1 << bit); } mi_assert_internal(page->capacity == (free_count + mi_page_used(page))); @@ -695,26 +710,26 @@ bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi for (size_t i = 0; i < bmapsize; i++) { if (free_map[i] == 0) { // every block is in use - for (size_t j = 0; j < MI_INTPTR_BITS; j++) { + for (size_t j = 0; j < MI_SIZE_BITS; j++) { #if MI_DEBUG>1 used_count++; #endif - if (!visitor(heap, area, block, ubsize, arg)) return false; + if (!visitor(heap, area, mi_ptr_bounded_data(block, ubsize), ubsize, arg)) return false; block += bsize; } } else { // visit the used blocks in the mask - uintptr_t m = ~free_map[i]; + size_t m = ~free_map[i]; while (m != 0) { #if MI_DEBUG>1 used_count++; #endif size_t bitidx = mi_ctz(m); - if (!visitor(heap, area, block + (bitidx * bsize), ubsize, arg)) return false; + if (!visitor(heap, area, mi_ptr_bounded_data(block + (bitidx * bsize), ubsize), ubsize, arg)) return false; m &= m - 1; // clear least significant bit } - block += bsize * MI_INTPTR_BITS; + block += bsize * MI_SIZE_BITS; } } mi_assert_internal(mi_page_used(page) == used_count); diff --git a/system/lib/mimalloc/src/prim/emscripten/prim.c b/system/lib/mimalloc/src/prim/emscripten/prim.c index 9d97a5859ed6b..3c2868077ec48 100644 --- a/system/lib/mimalloc/src/prim/emscripten/prim.c +++ b/system/lib/mimalloc/src/prim/emscripten/prim.c @@ -9,7 +9,8 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc/prim.h" -#include "mimalloc/heap/init.h" +#include "mimalloc/util/libc.h" +#include "mimalloc/heap/process.h" #include // sched_yield #include // getentropy @@ -60,6 +61,7 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config) { extern void emmalloc_free(void*); int _mi_prim_free(void* addr, size_t size) { + MI_UNUSED(size); emmalloc_free(addr); return 0; } @@ -116,8 +118,8 @@ int _mi_prim_reuse(void* addr, size_t size) { return 0; } -int _mi_prim_protect(void* addr, size_t size, bool protect) { - MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(protect); +int _mi_prim_guard(void* addr, size_t size, bool set, bool is_committed) { + MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(set); MI_UNUSED(is_committed); return 0; } @@ -180,11 +182,23 @@ void _mi_prim_out_stderr( const char* msg) { // Environment //---------------------------------------------------------------- +// Under node, host environment variables are only visible when linking with `-sNODE_HOST_ENV=1`. +// (Define `MI_NO_GETENV` to exclude this code) +extern char** environ; + int _mi_prim_getenv(const char* name, char* result, size_t result_size) { - // For code size reasons, do not support environ customization for now. - MI_UNUSED(name); - MI_UNUSED(result); - MI_UNUSED(result_size); + if (name==NULL) return -1; + const size_t len = _mi_strlen(name); + if (len == 0) return -1; + char** env = environ; + if (env == NULL) return -1; // not yet initialized; try again later + for (int i = 0; i < 10000 && env[i] != NULL; i++) { + const char* s = env[i]; + if (_mi_strnicmp(name, s, len) == 0 && s[len] == '=') { // case insensitive + if (!_mi_strlcpy(result, s + len + 1, result_size)) return -1; + return 1; + } + } return 0; // not found } diff --git a/system/lib/mimalloc/src/prim/os.c b/system/lib/mimalloc/src/prim/os.c index c9debd50f1035..fdb3f0732c03b 100644 --- a/system/lib/mimalloc/src/prim/os.c +++ b/system/lib/mimalloc/src/prim/os.c @@ -24,7 +24,8 @@ terms of the MIT license. A copy of the license can be found in the file #endif #endif -static mi_os_mem_config_t mi_os_mem_config = { +// Default OS memory configuration (used until `_mi_os_init`) +static const mi_os_mem_config_t mi_os_mem_config_default = { 4096, // page size 0, // large page size (usually 2MiB) 4096, // allocation granularity @@ -36,26 +37,33 @@ static mi_os_mem_config_t mi_os_mem_config = { false // has transparent huge pages? (if true we purge in (aligned) large page size chunks only to not fragment such pages) }; +// The OS memory configuration in the process data (zero until `_mi_os_init`) +static inline const mi_os_mem_config_t* mi_os_mem_config(void) { + const mi_os_mem_config_t* const config = &_mi_process.os_mem_config; + return (mi_likely(config->page_size != 0) ? config : &mi_os_mem_config_default); +} + bool _mi_os_has_overcommit(void) { - return mi_os_mem_config.has_overcommit; + return mi_os_mem_config()->has_overcommit; } bool _mi_os_has_virtual_reserve(void) { - return mi_os_mem_config.has_virtual_reserve; + return mi_os_mem_config()->has_virtual_reserve; } bool _mi_os_has_thp(void) { - return mi_os_mem_config.has_transparent_huge_pages; + return mi_os_mem_config()->has_transparent_huge_pages; } // OS (small) page size size_t _mi_os_page_size(void) { - return mi_os_mem_config.page_size; + return mi_os_mem_config()->page_size; } // if large OS pages are supported (2 or 4MiB), then return the size, otherwise return the small page size (4KiB) size_t _mi_os_large_page_size(void) { - return (mi_os_mem_config.large_page_size != 0 ? mi_os_mem_config.large_page_size : _mi_os_page_size()); + const size_t lpsize = mi_os_mem_config()->large_page_size; + return (lpsize != 0 ? lpsize : _mi_os_page_size()); } // minimal purge size. Can be larger than the page size if transparent huge pages are enabled. @@ -65,7 +73,7 @@ size_t _mi_os_minimal_purge_size(void) { // set by user return mi_align_up(minsize, _mi_os_page_size()); } - else if (mi_os_mem_config.has_transparent_huge_pages && mi_option_get(mi_option_allow_thp) == 2) { + else if (mi_os_mem_config()->has_transparent_huge_pages && mi_option_get(mi_option_allow_thp) == 2) { // don't break up THP pages; // we don't do this by default as that can lead to increased memory usage (see issue #1282). // on the other hand, breaking up THP pages can lead to performance degradation and perhaps @@ -85,15 +93,16 @@ size_t _mi_os_guard_page_size(void) { } size_t _mi_os_virtual_address_bits(void) { - const size_t vbits = mi_os_mem_config.virtual_address_bits; + const size_t vbits = mi_os_mem_config()->virtual_address_bits; mi_assert(vbits <= MI_MAX_VABITS); return vbits; } bool _mi_os_canuse_large_page(size_t size, size_t alignment) { // if we have access, check the size and alignment requirements - if (mi_os_mem_config.large_page_size == 0) return false; - return ((size % mi_os_mem_config.large_page_size) == 0 && (alignment % mi_os_mem_config.large_page_size) == 0); + const size_t lpsize = mi_os_mem_config()->large_page_size; + if (lpsize == 0) return false; + return ((size % lpsize) == 0 && (alignment % lpsize) == 0); } // round to a good OS allocation size (bounded by max 12.5% waste) @@ -109,7 +118,13 @@ size_t _mi_os_good_alloc_size(size_t size) { } void _mi_os_init(void) { - _mi_prim_mem_init(&mi_os_mem_config); + mi_os_mem_config_t config = mi_os_mem_config_default; + _mi_prim_mem_init(&config); + mi_assert(config.page_size != 0); + _mi_process.os_mem_config = config; // publish at once + if (_mi_os_page_size() > MI_ARENA_SLICE_SIZE) { + _mi_error_message(ENOTSUP, "the OS page size (%zu KiB) is larger than the arena slice size (%zu KiB); increase MI_ARENA_SLICE_SHIFT\n", _mi_os_page_size()/MI_KiB, (size_t)(MI_ARENA_SLICE_SIZE/MI_KiB)); + } } @@ -131,42 +146,40 @@ bool _mi_os_commit(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero // (otherwise an initial large allocation of say 2TiB has a 50% chance to include (known) addresses // in the middle of the 2TiB - 6TiB address range (see issue #372)) -#define MI_HINT_BASE ((uintptr_t)2 << 40) // 2TiB start -#define MI_HINT_AREA ((uintptr_t)4 << 40) // upto (2+4) 6TiB (since before win8 there is "only" 8TiB available to processes) -#define MI_HINT_MAX ((uintptr_t)30 << 40) // wrap after 30TiB (area after 32TiB is used for huge OS pages) +#define MI_HINT_BASE (MI_AU(2) << 40) // 2TiB start +#define MI_HINT_AREA (MI_AU(4) << 40) // upto (2+4) 6TiB (since before win8 there is "only" 8TiB available to processes) +#define MI_HINT_MAX (MI_AU(30) << 40) // wrap after 30TiB (area after 32TiB is used for huge OS pages) void* _mi_os_get_aligned_hint(size_t try_alignment, size_t sze) { - static mi_decl_cache_align _Atomic(uintptr_t) aligned_base; // = 0 - // todo: perhaps only do alignment hints if THP is enabled? - if (try_alignment <= mi_os_mem_config.alloc_granularity || try_alignment > 16*MI_GiB) return NULL; - if (mi_os_mem_config.virtual_address_bits < 46) return NULL; // < 64TiB virtual address space + if (try_alignment <= mi_os_mem_config()->alloc_granularity || try_alignment > 16*MI_GiB) return NULL; + if (mi_os_mem_config()->virtual_address_bits < 46) return NULL; // < 64TiB virtual address space size_t req_size = sze + _mi_os_page_size(); // always reserve a bit more to create virtual gaps between hinted blocks. req_size += (try_alignment - 1); // ensure we can align in the requested size req_size = mi_align_up(req_size, _mi_os_large_page_size()); - #if (MI_SECURE>=1) + #if MI_SECURE if (req_size > 32*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(MI_HINT_AREA / 1<<34) = 1/256 #endif - uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, req_size); + mi_addr_t hint = mi_atomic_add_acq_rel(&_mi_process.os_aligned_base, req_size); if (hint == 0 || hint > MI_HINT_MAX) { // wrap or initialize - uintptr_t init = MI_HINT_BASE; - #if (MI_SECURE>=1 || !MI_DEBUG) // security: randomize start of aligned allocations unless in debug mode + mi_addr_t init = MI_HINT_BASE; + #if (MI_SECURE || !MI_DEBUG) // security: randomize start of aligned allocations unless in debug mode mi_theap_t* const theap = _mi_theap_default(); // don't use `mi_theap_get_default()` as that can cause allocation recursively (issue #1267) if (!mi_theap_is_initialized(theap)) return NULL; // no hint as we lack randomness at this point - const uintptr_t r = _mi_theap_random_next(theap); - init = init + ((MI_MiB * ((r>>17) & 0x3FFFFF)) % MI_HINT_AREA); // (randomly 22 bits)* 1MiB == 0 to 4TiB + const size_t r = _mi_theap_random_next(theap); + init = init + ((MI_AU(MI_MiB) * ((r>>17) & 0x3FFFFF)) % MI_HINT_AREA); // (randomly 22 bits)* 1MiB == 0 to 4TiB #endif - uintptr_t expected = hint + req_size; - mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init); - hint = mi_atomic_add_acq_rel(&aligned_base, req_size); // this may still give 0 or > MI_HINT_MAX but that is ok, it is a hint after all + mi_addr_t expected = hint + req_size; + mi_atomic_cas_strong_acq_rel(&_mi_process.os_aligned_base, &expected, init); + hint = mi_atomic_add_acq_rel(&_mi_process.os_aligned_base, req_size); // this may still give 0 or > MI_HINT_MAX but that is ok, it is a hint after all if (hint==0) return NULL; } - const uintptr_t hint_align = mi_align_up(hint,try_alignment); + const mi_addr_t hint_align = mi_addr_align_up(hint,try_alignment); mi_assert_internal(hint_align + sze < hint + req_size); - return (void*)hint_align; + return mi_ptr_from_addr(hint_align); } #else void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) { @@ -180,66 +193,77 @@ void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) { Guard page allocation ----------------------------------------------------------- */ -// In secure mode, return the size of a guard page, otherwise 0 +// Return the size of a guard page, (or 0 if security is disabled or memory cannot be protected) size_t _mi_os_secure_guard_page_size(void) { - #if MI_SECURE > 0 + #if MI_SECURE && MI_OS_HAS_GUARD_PAGES // no memory protection on WebAssembly return _mi_os_guard_page_size(); #else return 0; #endif } -// In secure mode, try to decommit an area and output a warning if this fails. -bool _mi_os_secure_guard_page_set_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid) { - if (addr == NULL) return true; - #if MI_SECURE > 0 - bool ok = false; - if (!memid.is_pinned) { - mi_arena_t* const arena = mi_memid_arena(memid); - if (arena != NULL && arena->commit_fun != NULL) { - ok = (*(arena->commit_fun))(false /* decommit */, addr, _mi_os_secure_guard_page_size(), NULL, arena->commit_fun_arg); - } - else { - ok = _mi_os_decommit(subproc, addr, _mi_os_secure_guard_page_size()); - } +// Set or reset a guard page. The memid `initially_committed` is the accounted commit state +// of the guard page which must not change between a set and reset (see `_mi_prim_guard`). +static bool mi_os_secure_guard_page(void* addr, mi_memid_t memid, bool set) { + const size_t size = _mi_os_secure_guard_page_size(); + if (addr == NULL || size == 0 || memid.is_pinned) return true; + mi_assert_internal(mi_is_aligned(addr, _mi_os_page_size())); + bool ok; + mi_arena_t* const arena = mi_memid_arena(memid); + if (arena != NULL && arena->commit_fun != NULL) { + // decommit on a set, and only recommit on a reset if accounted as committed + ok = (set || memid.initially_committed ? (*(arena->commit_fun))(!set /* commit */, addr, size, NULL, arena->commit_fun_arg) : true); + } + else { + ok = (set ? _mi_os_guard(addr, size, memid.initially_committed) : _mi_os_unguard(addr, size, memid.initially_committed)); } if (!ok) { - _mi_error_message(EINVAL, "secure level %d, but failed to commit guard page (at %p of size %zu)\n", MI_SECURE, addr, _mi_os_secure_guard_page_size()); + _mi_error_message(EINVAL, "secure level %d, but failed to %s guard page (at %p of size %zu)\n", MI_SECURE, (set ? "set" : "reset"), addr, size); } return ok; - #else - MI_UNUSED(subproc); MI_UNUSED(memid); - return true; - #endif } -// In secure mode, try to decommit an area and output a warning if this fails. +// Make an area inaccessible and output a warning if this fails. +bool _mi_os_secure_guard_page_set_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid) { + MI_UNUSED(subproc); + return mi_os_secure_guard_page(addr, memid, true); +} + +// Make an area inaccessible and output a warning if this fails. bool _mi_os_secure_guard_page_set_before(mi_subproc_t* subproc, void* addr, mi_memid_t memid) { return _mi_os_secure_guard_page_set_at(subproc, (uint8_t*)addr - _mi_os_secure_guard_page_size(), memid); } -// In secure mode, try to recommit an area -bool _mi_os_secure_guard_page_reset_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid) { - if (addr == NULL) return true; - #if MI_SECURE > 0 - if (!memid.is_pinned) { - mi_arena_t* const arena = mi_memid_arena(memid); - if (arena != NULL && arena->commit_fun != NULL) { - return (*(arena->commit_fun))(true, addr, _mi_os_secure_guard_page_size(), NULL, arena->commit_fun_arg); - } - else { - return _mi_os_commit(subproc, addr, _mi_os_secure_guard_page_size(), NULL); - } - } - #else - MI_UNUSED(subproc); MI_UNUSED(memid); - #endif - return true; +// Set guard pages right before `addr` and right after `addr + size` (rounded up to the OS page size). +bool _mi_os_secure_guard_pages_set_around(mi_subproc_t* subproc, void* addr, size_t size, mi_memid_t memid) { + const bool ok_before = _mi_os_secure_guard_page_set_before(subproc, addr, memid); + const bool ok_after = _mi_os_secure_guard_page_set_at(subproc, mi_align_up_ptr((uint8_t*)addr + size, _mi_os_page_size()), memid); + return (ok_before && ok_after); } -// In secure mode, try to recommit an area -bool _mi_os_secure_guard_page_reset_before(mi_subproc_t* subproc, void* addr, mi_memid_t memid) { - return _mi_os_secure_guard_page_reset_at(subproc, (uint8_t*)addr - _mi_os_secure_guard_page_size(), memid); +// Make an area accessible again (if it is accounted as committed) +bool _mi_os_secure_guard_page_reset_at(mi_subproc_t* subproc, void* addr, mi_memid_t memid) { + MI_UNUSED(subproc); + return mi_os_secure_guard_page(addr, memid, false); +} + +// Set guard pages right before `area + offset` and right after `area + offset + size` (rounded to OS pages) +// within a padded static `area`. Outputs a warning if the guard pages do not fit (or cannot be set). +bool _mi_os_secure_guard_pages_set_within(void* area, size_t area_size, size_t offset, size_t size) { + const size_t gsize = _mi_os_secure_guard_page_size(); + if (gsize == 0) return true; + const mi_addr_t start = mi_addr(area); + const mi_addr_t before = mi_addr_align_down(start + offset, _mi_os_page_size()) - gsize; + const mi_addr_t after = mi_addr_align_up(start + offset + size, _mi_os_page_size()); + if (before < start || after + gsize > start + area_size) { + _mi_warning_message("secure level %d, but unable to set guard pages around static data at %p as the OS page size (%zu KiB) is too large\n", MI_SECURE, area, gsize / MI_KiB); + return false; + } + mi_memid_t memid = _mi_memid_create(MI_MEM_STATIC); // not pinned so it can be protected + memid.initially_committed = true; + const bool ok_before = mi_os_secure_guard_page(mi_ptr_with_addr(area, before), memid, true); + const bool ok_after = mi_os_secure_guard_page(mi_ptr_with_addr(area, after), memid, true); + return (ok_before && ok_after); } @@ -321,7 +345,7 @@ static void* mi_os_prim_alloc_at(mi_subproc_t* subproc, void* hint_addr, size_t if (try_alignment == 0) { try_alignment = 1; } // avoid 0 to ensure there will be no divide by zero when aligning // try to align along large OS page size for larger allocations - const size_t large_page_size = mi_os_mem_config.large_page_size; + const size_t large_page_size = mi_os_mem_config()->large_page_size; if (large_page_size > 0 && hint_addr == NULL && size >= 8*large_page_size && mi_is_power_of_two(try_alignment) && try_alignment < large_page_size) { try_alignment = large_page_size; } @@ -368,7 +392,7 @@ static void* mi_os_prim_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t const bool try_direct_alloc = true; #else // try a direct allocation if the alignment is below the default, or less than or equal to 1/4 fraction of the size. - const bool try_direct_alloc = (alignment <= mi_os_mem_config.alloc_granularity || alignment <= size/4); + const bool try_direct_alloc = (alignment <= mi_os_mem_config()->alloc_granularity || alignment <= size/4); #endif bool os_is_large = false; @@ -397,7 +421,7 @@ static void* mi_os_prim_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t if (size >= (SIZE_MAX - alignment)) return NULL; // overflow const size_t over_size = size + alignment; - if (!mi_os_mem_config.has_partial_free) { // win32 virtualAlloc cannot free parts of an allocated block + if (!mi_os_mem_config()->has_partial_free) { // win32 virtualAlloc cannot free parts of an allocated block // over-allocate uncommitted (virtual) memory p = mi_os_prim_alloc(subproc, over_size, 1 /*alignment*/, false /* commit? */, false /* allow_large */, &os_is_large, &os_is_zero); if (p == NULL) return NULL; @@ -505,6 +529,25 @@ void* _mi_os_zalloc(mi_subproc_t* subproc, size_t size, mi_memid_t* memid) { return mi_os_ensure_zero(subproc, p, size, memid); } +// Allocate OS memory with (in secure mode) a guard page right before and after it. +// The memory can be freed using `_mi_os_free` with the returned `memid`. +void* _mi_os_alloc_with_guards(mi_subproc_t* subproc, size_t size, bool commit, bool allow_large, mi_memid_t* memid) { + const size_t guard_size = _mi_os_secure_guard_page_size(); + size = mi_align_up(size, _mi_os_page_size()); + uint8_t* const base = (uint8_t*)_mi_os_alloc_aligned(subproc, size + 2*guard_size, 1, commit, allow_large, memid); + if (base == NULL) return NULL; + uint8_t* const p = base + guard_size; + if (memid->initially_committed) { // (if not committed, the guard pages are never committed) + _mi_os_secure_guard_pages_set_around(subproc, p, size, *memid); + } + return p; +} + +// Free memory allocated by `_mi_os_alloc_with_guards` (including the guard pages). +void _mi_os_free_with_guards(mi_subproc_t* subproc, mi_memid_t memid) { + _mi_os_free(subproc, memid.mem.os.base, memid.mem.os.size, memid); +} + /* ----------------------------------------------------------- OS aligned allocation with an offset. This is used for large alignments > MI_BLOCK_ALIGNMENT_MAX. We use a large mimalloc @@ -700,30 +743,27 @@ bool _mi_os_purge(mi_subproc_t* subproc, void* p, size_t size) { } -// Protect a region in memory to be not accessible. -static bool mi_os_protectx(void* addr, size_t size, bool protect) { +// Set (or reset) a guard range (see `_mi_prim_guard` for the meaning of `is_committed`). +static bool mi_os_guardx(void* addr, size_t size, bool set, bool is_committed) { // page align conservatively within the range size_t csize = 0; void* start = mi_os_page_align_area_conservative(addr, size, &csize); if (csize == 0) return false; - /* - if (_mi_os_is_huge_reserved(addr)) { - _mi_warning_message("cannot mprotect memory allocated in huge OS pages\n"); - } - */ - int err = _mi_prim_protect(start,csize,protect); + int err = _mi_prim_guard(start, csize, set, is_committed); if (err != 0) { - _mi_warning_message("cannot %s OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", (protect ? "protect" : "unprotect"), err, err, start, csize); + _mi_warning_message("cannot %s guard OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", (set ? "set" : "reset"), err, err, start, csize); } return (err == 0); } -bool _mi_os_protect(void* addr, size_t size) { - return mi_os_protectx(addr, size, true); +// Make a range inaccessible +bool _mi_os_guard(void* addr, size_t size, bool is_committed) { + return mi_os_guardx(addr, size, true, is_committed); } -bool _mi_os_unprotect(void* addr, size_t size) { - return mi_os_protectx(addr, size, false); +// Make a range accessible again (if it is accounted as committed) +bool _mi_os_unguard(void* addr, size_t size, bool is_committed) { + return mi_os_guardx(addr, size, false, is_committed); } @@ -735,10 +775,7 @@ and possibly associated with a specific NUMA node. (use `numa_node>=0`) #define MI_HUGE_OS_PAGE_SIZE (MI_GiB) -#if (MI_SIZE_SIZE >= 8) -// To ensure proper alignment, use our own area for huge OS pages -static mi_decl_cache_align _Atomic(uintptr_t) mi_huge_start; // = 0 - +#if (MI_SIZE_SIZE >= 8) && !MI_CHERI // on CHERI the separately mapped huge pages cannot be covered by one capability // Claim an aligned address range for huge pages static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) { if (total_size != NULL) *total_size = 0; @@ -748,30 +785,30 @@ static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) { return NULL; } - uintptr_t start = 0; - uintptr_t end = 0; - uintptr_t huge_start = mi_atomic_load_relaxed(&mi_huge_start); + mi_addr_t start = 0; + mi_addr_t end = 0; + mi_addr_t huge_start = mi_atomic_load_relaxed(&_mi_process.os_huge_start); do { start = huge_start; if (start == 0) { // Initialize the start address after the 32TiB area - start = ((uintptr_t)32 << 40); // 32TiB virtual start address (after addresses returned by _mi_os_get_aligned_hint) - #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode + start = (MI_AU(32) << 40); // 32TiB virtual start address (after addresses returned by _mi_os_get_aligned_hint) + #if (MI_SECURE || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode mi_theap_t* const theap = _mi_theap_default(); // don't use `mi_theap_get_default()` as that can cause allocation recursively (issue #1267) if (mi_theap_is_initialized(theap)) { // todo: or no hint at all if we lack randomness? - const uintptr_t r = _mi_theap_random_next(theap); - start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x0FFF)); // (randomly 12bits)*1GiB == between 0 to 4TiB + const size_t r = _mi_theap_random_next(theap); + start = start + (MI_AU(MI_HUGE_OS_PAGE_SIZE) * ((r>>17) & 0x0FFF)); // (randomly 12bits)*1GiB == between 0 to 4TiB } else { - _mi_warning_message("failed to randomize the start address of huge pages allocation (%zu bytes at %p)", size, (void*)start); + _mi_warning_message("failed to randomize the start address of huge pages allocation (%zu bytes at %p)", size, mi_ptr_from_addr(start)); } #endif } end = start + size; - } while (!mi_atomic_cas_weak_acq_rel(&mi_huge_start, &huge_start, end)); + } while (!mi_atomic_cas_weak_acq_rel(&_mi_process.os_huge_start, &huge_start, end)); if (total_size != NULL) *total_size = size; - return (uint8_t*)start; + return (uint8_t*)mi_ptr_from_addr(start); } #else static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) { @@ -871,10 +908,8 @@ static void mi_os_free_huge_os_pages(mi_subproc_t* subproc, void* p, size_t size Support NUMA aware allocation -----------------------------------------------------------------------------*/ -static _Atomic(size_t) mi_numa_node_count; // = 0 // cache the node count - int _mi_os_numa_node_count(void) { - size_t count = mi_atomic_load_acquire(&mi_numa_node_count); + size_t count = mi_atomic_load_acquire(&_mi_process.os_numa_node_count); if mi_unlikely(count == 0) { long ncount = mi_option_get(mi_option_use_numa_nodes); // given explicitly? if (ncount > 0 && ncount < INT_MAX) { @@ -885,7 +920,7 @@ int _mi_os_numa_node_count(void) { if (n == 0 || n > INT_MAX) { count = 1; } else { count = n; } } - mi_atomic_store_release(&mi_numa_node_count, count); // save it + mi_atomic_store_release(&_mi_process.os_numa_node_count, count); // save it if (count>1) { _mi_verbose_message("using %zd numa regions\n", count); } } mi_assert_internal(count > 0 && count <= INT_MAX); @@ -903,7 +938,7 @@ static int mi_os_numa_node_get(void) { } int _mi_os_numa_node(void) { - if mi_likely(mi_atomic_load_relaxed(&mi_numa_node_count) == 1) { + if mi_likely(mi_atomic_load_relaxed(&_mi_process.os_numa_node_count) == 1) { return 0; } else { @@ -917,7 +952,7 @@ int _mi_os_numa_node(void) { -----------------------------------------------------------------------------*/ #if 0 mi_decl_export void* mi_os_alloc(size_t size, bool commit, size_t* full_size) { - return mi_os_alloc_aligned(size, mi_os_mem_config.alloc_granularity, commit, NULL, full_size); + return mi_os_alloc_aligned(size, mi_os_mem_config()->alloc_granularity, commit, NULL, full_size); } static void* mi_os_alloc_aligned_ex(size_t size, size_t alignment, bool commit, bool allow_large, bool* is_committed, bool* is_pinned, void** base, size_t* full_size) { diff --git a/system/lib/mimalloc/src/prim/osx/alloc-override-zone.c b/system/lib/mimalloc/src/prim/osx/alloc-override-zone.c index 00ba0fa88ad26..6602efa64024b 100644 --- a/system/lib/mimalloc/src/prim/osx/alloc-override-zone.c +++ b/system/lib/mimalloc/src/prim/osx/alloc-override-zone.c @@ -7,6 +7,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc/prim/os.h" +#include "mimalloc/heap/subproc.h" #if defined(MI_MALLOC_OVERRIDE) #if !defined(__APPLE__) @@ -193,11 +194,12 @@ static boolean_t intro_zone_locked(malloc_zone_t* zone) { } // Required whenever the zone advertises version >= 9: macOS calls this from the -// atfork_child handler (_malloc_fork_child) without a NULL check. mimalloc keeps -// no zone-level locks that need reinitializing after fork, so a no-op is safe. +// atfork_child handler (_malloc_fork_child) without a NULL check. // Leaving it NULL makes the forked child jump to address 0 and crash in fork(). +// This runs in the child before the other libSystem child handlers (that allocate). static void intro_reinit_lock(malloc_zone_t* zone) { MI_UNUSED(zone); + _mi_fork_child_zone(); } diff --git a/system/lib/mimalloc/src/prim/prim-tls.c b/system/lib/mimalloc/src/prim/prim-tls.c index ddaa5d4576fb2..3cb1e17cb8758 100644 --- a/system/lib/mimalloc/src/prim/prim-tls.c +++ b/system/lib/mimalloc/src/prim/prim-tls.c @@ -31,8 +31,8 @@ mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_cached = (mi_theap_t*)&_mi_ mi_decl_hidden mi_decl_thread void* __mi_thread_id_helper = NULL; -mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { - const mi_threadid_t tid = _mi_prim_thread_id(); +mi_thread_id_t _mi_thread_id(void) mi_attr_noexcept { + const mi_thread_id_t tid = _mi_prim_thread_id(); mi_assert_internal( (tid & MI_PAGE_FLAG_MASK) == 0 ); // mimalloc reserves the bottom 2 bits return tid; } diff --git a/system/lib/mimalloc/src/prim/prim.c b/system/lib/mimalloc/src/prim/prim.c index 17ab7ad8bfe59..1fc1a3a036923 100644 --- a/system/lib/mimalloc/src/prim/prim.c +++ b/system/lib/mimalloc/src/prim/prim.c @@ -5,7 +5,7 @@ terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ -#include "mimalloc/heap/init.h" +#include "mimalloc/heap/process.h" #include "mimalloc/prim.h" diff --git a/system/lib/mimalloc/src/prim/unix/prim.c b/system/lib/mimalloc/src/prim/unix/prim.c index 794d2d070f43f..d55d87fb3853e 100644 --- a/system/lib/mimalloc/src/prim/unix/prim.c +++ b/system/lib/mimalloc/src/prim/unix/prim.c @@ -26,7 +26,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc/prim/os.h" #include "mimalloc/util/libc.h" #include "mimalloc/util/options.h" -#include "mimalloc/heap/init.h" +#include "mimalloc/heap/process.h" #include // mmap #include // sysconf, sleep @@ -264,7 +264,7 @@ static size_t unix_detect_virtual_address_bits(void) { #endif { if (probe.key != -1) { // If a key is unknown to the kernel, its key field will be cleared to -1. - return (MI_SIZE_BITS - mi_clz((uintptr_t)probe.value)); + return (MI_SIZE_BITS - mi_clz((size_t)probe.value)); } } #endif @@ -593,22 +593,28 @@ int _mi_prim_reuse(void* start, size_t size) { return 0; } +// Release the physical memory of a range (while keeping it mapped) +static int unix_madvise_release(void* start, size_t size) { + #if defined(__APPLE__) && defined(MADV_FREE_REUSABLE) + // use MADV_FREE_REUSABLE as it does immediate rss accounting (issue #1097) + int err = unix_madvise(start, size, MADV_FREE_REUSABLE); + if (err) { err = unix_madvise(start, size, MADV_DONTNEED); } + return err; + #else + // use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE) + return unix_madvise(start, size, MADV_DONTNEED); + #endif +} + int _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) { int err = 0; #if 1 - #if defined(__APPLE__) && defined(MADV_FREE_REUSABLE) - // decommit on macOS: use MADV_FREE_REUSABLE as it does immediate rss accounting (issue #1097) - err = unix_madvise(start, size, MADV_FREE_REUSABLE); - if (err) { err = unix_madvise(start, size, MADV_DONTNEED); } - #else - // decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE) - err = unix_madvise(start, size, MADV_DONTNEED); - #endif - #if !MI_DEBUG && MI_SECURE<=2 - *needs_recommit = false; - #else + err = unix_madvise_release(start, size); + #if MI_DEBUG || MI_SECURE>=4 *needs_recommit = true; mprotect(start, size, PROT_NONE); + #else + *needs_recommit = false; #endif #else // decommit: use mmap with MAP_FIXED and PROT_NONE to discard the existing memory (and reduce rss) @@ -649,13 +655,28 @@ int _mi_prim_reset(void* start, size_t size) { return err; } -int _mi_prim_protect(void* start, size_t size, bool protect) { +static int unix_mprotect(void* start, size_t size, bool protect) { int err = mprotect(start, size, protect ? PROT_NONE : (PROT_READ | PROT_WRITE)); if (err != 0) { err = errno; } unix_mprotect_hint(err); return err; } +int _mi_prim_guard(void* start, size_t size, bool set, bool is_committed) { + if (set) { + // `mprotect` also works on reserved memory, so committed or not, we release the physical memory and protect + unix_madvise_release(start, size); // best effort + return unix_mprotect(start, size, true); + } + else if (is_committed) { + const int err = unix_mprotect(start, size, false); + return (err != 0 ? err : _mi_prim_reuse(start, size)); + } + else { + return 0; // stays inaccessible until committed + } +} + //--------------------------------------------- @@ -1015,13 +1036,20 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) { #ifndef GRND_NONBLOCK #define GRND_NONBLOCK (1) #endif - static _Atomic(uintptr_t) no_getrandom; // = 0 + #ifndef GRND_INSECURE + #define GRND_INSECURE (4) + #endif + static _Atomic(size_t) no_getrandom; // = 0 if (mi_atomic_load_acquire(&no_getrandom)==0) { ssize_t ret = syscall(SYS_getrandom, buf, buf_len, GRND_NONBLOCK); + if (ret < 0 && errno == EAGAIN) { + // early during boot the entropy pool may not be initialized yet; use the best available randomness (Linux 5.6+) + ret = syscall(SYS_getrandom, buf, buf_len, GRND_INSECURE); + } if (ret > 0) { mi_msan_unpoison(buf, (size_t)ret); } if (ret >= 0) return (buf_len == (size_t)ret); - if (errno != ENOSYS) return false; - mi_atomic_store_release(&no_getrandom, (uintptr_t)1); // don't call again, and fall back to /dev/urandom + if (errno == ENOSYS) { mi_atomic_store_release(&no_getrandom, (size_t)1); } // don't call again + // and fall back to /dev/urandom (which does not block early during boot) } #endif int flags = O_RDONLY; diff --git a/system/lib/mimalloc/src/prim/wasi/prim.c b/system/lib/mimalloc/src/prim/wasi/prim.c index 1946cfc43409f..f46ae2d2bb776 100644 --- a/system/lib/mimalloc/src/prim/wasi/prim.c +++ b/system/lib/mimalloc/src/prim/wasi/prim.c @@ -11,11 +11,11 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc/prim.h" #include "mimalloc/prim/os.h" #include "mimalloc/util/libc.h" -#include "mimalloc/heap/init.h" +#include "mimalloc/heap/process.h" #include // fputs #include // getenv -#include // sbrk, sleep +#include // sbrk, sleep, getentropy //--------------------------------------------- // Initialize @@ -115,7 +115,7 @@ static void* mi_prim_mem_grow(size_t size, size_t try_alignment) { return NULL; } */ - mi_assert_internal( p == NULL || try_alignment == 0 || (uintptr_t)p % try_alignment == 0 ); + mi_assert_internal( p == NULL || try_alignment == 0 || mi_addr(p) % try_alignment == 0 ); return p; } @@ -155,8 +155,8 @@ int _mi_prim_reuse(void* addr, size_t size) { return 0; } -int _mi_prim_protect(void* addr, size_t size, bool protect) { - MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(protect); +int _mi_prim_guard(void* addr, size_t size, bool set, bool is_committed) { + MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(set); MI_UNUSED(is_committed); return 0; } @@ -270,7 +270,7 @@ int _mi_prim_getenv(const char* name, char* result, size_t result_size) { //---------------------------------------------------------------- bool _mi_prim_random_buf(void* buf, size_t buf_len) { - return false; + return (getentropy(buf, buf_len) == 0); // uses `random_get` (and buf_len <= 256) } diff --git a/system/lib/mimalloc/src/prim/windows/prim.c b/system/lib/mimalloc/src/prim/windows/prim.c index 4eb1dde4577bb..2ff305ee6540a 100644 --- a/system/lib/mimalloc/src/prim/windows/prim.c +++ b/system/lib/mimalloc/src/prim/windows/prim.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -12,11 +12,11 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc/prim/os.h" #include "mimalloc/util/libc.h" #include "mimalloc/util/options.h" -#include "mimalloc/heap/init.h" +#include "mimalloc/heap/process.h" #include // fputs, stderr #include // atexit -// xbox has no console IO and cannot use LoadLibrary or GetModuleHandle +// xbox has no console IO (and might not be able to use GetModuleHandle) #if !defined(WINAPI_FAMILY_PARTITION) || WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP | WINAPI_PARTITION_SYSTEM) #define MI_WIN_DESKTOP 1 #endif @@ -94,7 +94,7 @@ typedef BOOL (__stdcall* PGetVersionExW)(LPOSVERSIONINFOW lpVersionInformation); // Load a library static HMODULE mi_win_loadlibrary(const wchar_t* library) { - #if MI_WIN_DESKTOP + #if 1 // used to be `MI_WIN_DESKTOP`, see #1340 and #1467 return LoadLibraryW(library); #else return LoadPackagedLibrary(library, 0); @@ -139,7 +139,7 @@ static bool win_enable_large_os_pages_once(size_t* large_page_size) err = GetLastError(); if (ok) { TOKEN_PRIVILEGES tp; - ok = LookupPrivilegeValue(NULL, TEXT("SeLockMemoryPrivilege"), &tp.Privileges[0].Luid); + ok = LookupPrivilegeValueW(NULL, L"SeLockMemoryPrivilege", &tp.Privileges[0].Luid); err = GetLastError(); if (ok) { tp.PrivilegeCount = 1; @@ -444,7 +444,7 @@ int _mi_prim_reuse(void* addr, size_t size) { return 0; } -int _mi_prim_protect(void* addr, size_t size, bool protect) { +static int win_protect(void* addr, size_t size, bool protect) { #if !MI_WIN_DESKTOP && WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP) ULONG oldprotect = 0; BOOL ok = VirtualProtectFromApp(addr, size, protect ? PAGE_NOACCESS : PAGE_READWRITE, &oldprotect); @@ -455,9 +455,22 @@ int _mi_prim_protect(void* addr, size_t size, bool protect) { return (ok ? 0 : (int)GetLastError()); } - -//--------------------------------------------- -// Huge page allocation +int _mi_prim_guard(void* addr, size_t size, bool set, bool is_committed) { + if (is_committed) { + // protecting requires committed memory; on a set we first release the physical memory + // (best effort, as `MEM_RESET` fails on image memory, like the guard pages in our static data) + if (set) { VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE); } + return win_protect(addr, size, set); + } + else if (set) { + // decommitting also works for (partly) reserved memory + bool needs_recommit; + return _mi_prim_decommit(addr, size, &needs_recommit); + } + else { + return 0; // stays inaccessible until committed + } +} //--------------------------------------------- static void* _mi_prim_alloc_huge_os_pagesx(void* hint_addr, size_t size, int numa_node) @@ -567,15 +580,16 @@ size_t _mi_prim_numa_node_count(void) { //---------------------------------------------------------------- static mi_msecs_t mi_to_msecs(LARGE_INTEGER t) { - static LARGE_INTEGER mfreq; // = 0 - if (mfreq.QuadPart == 0LL) { + int64_t ticks_per_msec = _mi_process.clock_ticks_per_msec; + if (ticks_per_msec == 0) { LARGE_INTEGER f; if (QueryPerformanceFrequency(&f)) { - mfreq.QuadPart = f.QuadPart/1000LL; + ticks_per_msec = f.QuadPart/1000LL; } - if (mfreq.QuadPart == 0) mfreq.QuadPart = 1; + if (ticks_per_msec == 0) { ticks_per_msec = 1; } + _mi_process.clock_ticks_per_msec = ticks_per_msec; } - return (mi_msecs_t)(t.QuadPart / mfreq.QuadPart); + return (mi_msecs_t)(t.QuadPart / ticks_per_msec); } mi_msecs_t _mi_prim_clock_now(void) { @@ -1220,10 +1234,8 @@ extern "C" { #if defined(MI_SHARED_LIB) && !defined(MI_WIN_NOREDIRECT) #define MI_PRIM_HAS_ALLOCATOR_INIT 1 - static bool mi_redirected = false; // true if malloc redirects to mi_malloc - bool _mi_is_redirected(void) { - return mi_redirected; + return _mi_process.malloc_is_redirected; } #ifdef __cplusplus @@ -1232,10 +1244,10 @@ extern "C" { mi_decl_export void _mi_redirect_entry(DWORD reason) { // called on redirection; careful as this may be called before DllMain if (reason == DLL_PROCESS_ATTACH) { - mi_redirected = true; + _mi_process.malloc_is_redirected = true; } else if (reason == DLL_PROCESS_DETACH) { - mi_redirected = false; + _mi_process.malloc_is_redirected = false; } else if (reason == DLL_THREAD_DETACH) { // mi_debug_out("redirect thread detach\n"); diff --git a/system/lib/mimalloc/src/profile/pprof.c b/system/lib/mimalloc/src/profile/pprof.c index 4b925f38ced9f..97f16874e1a6f 100644 --- a/system/lib/mimalloc/src/profile/pprof.c +++ b/system/lib/mimalloc/src/profile/pprof.c @@ -28,6 +28,7 @@ format generation). #include "mimalloc/util/options.h" #include "mimalloc/util/stats.h" #include "mimalloc/profile/pprof.h" +#include "mimalloc/heap/process.h" // _mi_process.profile_env_profiler // --------------------------------------------------------------------------- // Internal API @@ -39,7 +40,7 @@ typedef struct mi_callstack_s mi_callstack_t; static bool mi_locations_init(mi_heap_t* heap, mi_locations_t* locations); static void mi_locations_done(mi_heap_t* heap, mi_locations_t* locations); -static mi_location_t* mi_locations_find_or_insert(mi_heap_t* heap, mi_locations_t* locations, mi_threadid_t thread_id, const mi_callstack_t* callstack, size_t callstack_hash); +static mi_location_t* mi_locations_find_or_insert(mi_heap_t* heap, mi_locations_t* locations, mi_thread_id_t thread_id, const mi_callstack_t* callstack, size_t callstack_hash); // --------------------------------------------------------------------------- // Types @@ -66,7 +67,7 @@ struct mi_callstack_s { // `hash`/`thread_id`/`callstack` which allows a lock-free lookup of locations in the hash table. struct mi_location_s { _Atomic(struct mi_location_s*) next; // next location in the same hash bucket (for chaining on collision); - mi_threadid_t thread_id; + mi_thread_id_t thread_id; mi_callstack_t callstack; size_t hash; // hash of the thread_id and callstack int64_t alloc_count; // only ever written by the single (allocating) owning thread: not atomic @@ -128,9 +129,8 @@ static int64_t mi_prim_time_secs(void); // wall-clock seconds since the Unix ep #if MI_PROFILE_USE_BUILTIN // todo: also disable profiling in secure mode? static bool mi_is_elevated_process(void); -static mi_profiler_t* mi_profile_env_profiler; // NULL if `MIMALLOC_PROFILE` was not set (or initialization failed) -// Called once at process initialization (see `mi_process_init` in `heap/init.c`). +// Called once at process initialization (see `mi_process_init` in `heap/process.c`). void _mi_pprof_profiler_init(void) { #if MI_PROFILE if (mi_is_elevated_process()) return; // don't let an untrusted environment influence a privileged process @@ -142,19 +142,19 @@ void _mi_pprof_profiler_init(void) { const size_t time_interval_secs = mi_option_get_clamp(mi_option_profile_time_interval,0,24*60*60L); // 0 by default (no automatic interval snapshots) mi_profiler_t* profiler = mi_pprof_profiler_new(sample_rate, fname, alloc_interval_size, inuse_interval_size, time_interval_secs); if (profiler == NULL) return; - mi_profile_env_profiler = profiler; + _mi_process.profile_env_profiler = profiler; _mi_verbose_message("pprof profiler initialized with base file name: %s\n", fname); mi_profile(profiler); // attach to the main sub-process (and any current/future heaps in it) mi_profiler_start(profiler); #endif } -// Called once at process termination (see `mi_process_done` in `heap/init.c`). +// Called once at process termination (see `mi_process_done` in `heap/process.c`). void _mi_pprof_profiler_done(void) { #if MI_PROFILE - mi_profiler_t* profiler = mi_profile_env_profiler; + mi_profiler_t* profiler = _mi_process.profile_env_profiler; if (profiler == NULL) return; - mi_profile_env_profiler = NULL; + _mi_process.profile_env_profiler = NULL; mi_profiler_stop(profiler); _mi_verbose_message("pprof profiler done\n"); mi_profiler_snapshot(profiler); // final snapshot so short-lived processes still get a complete profile @@ -183,7 +183,7 @@ static mi_decl_noinline mi_location_t* mi_location_get(mi_pprof_profiler_t* prof const size_t depth = mi_prim_backtrace(frames, MI_MAX_BACKTRACE_DEPTH, &callstack_hash); if (depth==0) return NULL; mi_callstack_t callstack = { depth, frames }; - const mi_threadid_t thread_id = _mi_thread_id(); + const mi_thread_id_t thread_id = _mi_thread_id(); mi_location_t* loc = mi_locations_find_or_insert(prof->profile_heap, &prof->locations, thread_id, &callstack, callstack_hash); return loc; } @@ -442,7 +442,7 @@ void mi_pprof_profiler_delete(mi_profiler_t* profiler) { #define MI_FNV_PRIME ((size_t)0x01000193UL) // FNV-1a 32-bit prime #endif -static size_t mi_location_hash(mi_threadid_t thread_id, size_t callstack_hash) { +static size_t mi_location_hash(mi_thread_id_t thread_id, size_t callstack_hash) { size_t hash = MI_FNV_OFFSET_BASIS; const size_t prime = MI_FNV_PRIME; hash = (hash ^ (size_t)thread_id) * prime; @@ -480,7 +480,7 @@ static void mi_locations_done(mi_heap_t* heap, mi_locations_t* locations) { } // Match a location's identifying fields; these never change after publication. -static bool mi_location_matches(const mi_location_t* loc, size_t hash, mi_threadid_t thread_id, const mi_callstack_t* callstack) { +static bool mi_location_matches(const mi_location_t* loc, size_t hash, mi_thread_id_t thread_id, const mi_callstack_t* callstack) { return (loc->hash == hash && loc->thread_id == thread_id && loc->callstack.count == callstack->count && (callstack->count == 0 || mi_memcmp(loc->callstack.frames, callstack->frames, callstack->count * sizeof(void*)) == 0)); @@ -493,7 +493,7 @@ static bool mi_location_matches(const mi_location_t* loc, size_t hash, mi_thread // is a lock-free, read-only traversal using relaxed loads; only inserting a // genuinely new location uses a CAS (and even then, only ever races against // other threads inserting *different* locations into the same bucket). -static mi_location_t* mi_locations_find_or_insert(mi_heap_t* heap, mi_locations_t* locations, mi_threadid_t thread_id, const mi_callstack_t* callstack, size_t callstack_hash) { +static mi_location_t* mi_locations_find_or_insert(mi_heap_t* heap, mi_locations_t* locations, mi_thread_id_t thread_id, const mi_callstack_t* callstack, size_t callstack_hash) { if (locations->buckets == NULL) return NULL; const size_t hash = mi_location_hash(thread_id, callstack_hash); const size_t idx = hash % locations->bucket_count; @@ -672,7 +672,7 @@ static void mi_pprof_profiler_snapshot(mi_pprof_profiler_t* prof) { // Basic backtraces // --------------------------------------------------------------------------- -#if MI_HAS_EXECINFOH || MI_HAS_LIBUNWINDH +#if MI_HAS_EXECINFOH || MI_HAS_UNWINDH static size_t mi_backtrace_hash(void** frames, size_t depth) { size_t hash = MI_FNV_OFFSET_BASIS; for (size_t i = 0; i < depth; i++) { @@ -710,20 +710,30 @@ static mi_decl_noinline size_t mi_prim_backtrace(void** buffer, size_t max_depth *hash = mi_backtrace_hash(buffer, depth); // no platform-provided hash, compute our own return depth; } -#elif MI_HAS_LIBUNWINDH -#define UNW_LOCAL_ONLY -#include // unw_backtrace +#elif MI_HAS_UNWINDH +#include // _Unwind_Backtrace (from the compiler's unwinder: libgcc_s or LLVM libunwind) +typedef struct mi_unwind_state_s { + void** buffer; + size_t max_depth; + size_t skip; + size_t depth; +} mi_unwind_state_t; + +static _Unwind_Reason_Code mi_unwind_frame(struct _Unwind_Context* ctx, void* arg) { + mi_unwind_state_t* const st = (mi_unwind_state_t*)arg; + const uintptr_t ip = (uintptr_t)_Unwind_GetIP(ctx); + if (ip == 0) return _URC_END_OF_STACK; + if (st->skip > 0) { st->skip--; return _URC_NO_REASON; } + st->buffer[st->depth++] = (void*)ip; + return (st->depth >= st->max_depth ? _URC_END_OF_STACK : _URC_NO_REASON); +} + static mi_decl_noinline size_t mi_prim_backtrace(void** buffer, size_t max_depth, size_t* hash) { // used as a fallback on platforms without `execinfo.h` (e.g. musl-based Linux like Alpine) - const size_t skip = 1 + MI_BACKTRACE_SKIP_FRAMES; // +1 for `mi_prim_backtrace`'s own frame - if (max_depth <= skip) { *hash = 0; return 0; } - if (max_depth > INT_MAX) { max_depth = INT_MAX; } - const size_t raw_depth = (size_t)unw_backtrace(buffer, (int)(max_depth - skip)); - if (raw_depth <= skip) { *hash = 0; return 0; } - const size_t depth = raw_depth - skip; - memmove(buffer, buffer + skip, depth * sizeof(void*)); - *hash = mi_backtrace_hash(buffer, depth); // no platform-provided hash, compute our own - return depth; + mi_unwind_state_t st = { buffer, max_depth, 1 + MI_BACKTRACE_SKIP_FRAMES /* +1 for `mi_prim_backtrace`'s own frame */, 0 }; + if (max_depth > 0) { _Unwind_Backtrace(&mi_unwind_frame, &st); } + *hash = mi_backtrace_hash(buffer, st.depth); // no platform-provided hash, compute our own + return st.depth; } #else static size_t mi_prim_backtrace(void** buffer, size_t max_depth, size_t* hash) { @@ -755,7 +765,7 @@ typedef struct { bool failed; // set if an allocation failed; the mapping list is then incomplete } mi_pprof_modules_t; -static void mi_pprof_modules_add(mi_heap_t* heap, mi_pprof_modules_t* mods, uintptr_t start, uintptr_t end, const char* path) { +mi_decl_maybe_unused static void mi_pprof_modules_add(mi_heap_t* heap, mi_pprof_modules_t* mods, uintptr_t start, uintptr_t end, const char* path) { if (mods->failed) return; if (mods->count == mods->capacity) { const size_t new_capacity = (mods->capacity == 0 ? 16 : mods->capacity * 2); @@ -1091,7 +1101,7 @@ static bool mi_pprof_snapshot_text(mi_pprof_profiler_t* prof, const char* fname) (uint64_t)inuse_count, (uint64_t)inuse_bytes, (uint64_t)loc->alloc_count, (uint64_t)loc->alloc_bytes); for (size_t j = 0; j < loc->callstack.count; j++) { - mi_fprintf(f, " 0x%tx", (uintptr_t)loc->callstack.frames[j]); + mi_fprintf(f, " 0x%tx", (size_t)(uintptr_t)loc->callstack.frames[j]); } fprintf(f, "\n"); } diff --git a/system/lib/mimalloc/src/profile/sample-guarded.c b/system/lib/mimalloc/src/profile/sample-guarded.c index a1d8b34823658..e59b54d794bc6 100644 --- a/system/lib/mimalloc/src/profile/sample-guarded.c +++ b/system/lib/mimalloc/src/profile/sample-guarded.c @@ -49,6 +49,8 @@ void _mi_theap_guarded_init(mi_theap_t* theap) { static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size, size_t* usable_size) { // todo: we can still make padding work by moving it out of the guard page area mi_page_t* const page = mi_ptr_page(block); + if mi_unlikely(page==NULL) return NULL; // should not happen, but avoids gcc -Wstringop-overflow warnings + block = (mi_block_t*)mi_ptr_rederive(mi_page_start(page), block); // `block` is bounded to the requested size mi_page_set_has_interior_pointers(page, true); block->next = MI_BLOCK_TAG_GUARDED; @@ -61,13 +63,13 @@ static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size, size_t mi_free(block); return NULL; } - uint8_t* guard_page = (uint8_t*)block + block_size - os_page_size; + uint8_t* guard_page = (uint8_t*)mi_page_vmem_ptr(page, (uint8_t*)block + block_size - os_page_size); // note: the alignment of the guard page relies on blocks being os_page_size aligned which // is ensured in `mi_arena_page_alloc_fresh`. mi_assert_internal(mi_is_aligned(block, os_page_size)); mi_assert_internal(mi_is_aligned(guard_page, os_page_size)); if (!page->memid.is_pinned && mi_is_aligned(guard_page, os_page_size)) { - const bool ok = _mi_os_protect(guard_page, os_page_size); + const bool ok = _mi_os_guard(guard_page, os_page_size, true /* committed */); if mi_unlikely(!ok) { _mi_warning_message("failed to set a guard page behind an object (object %p of size %zu)\n", block, block_size); } @@ -136,7 +138,7 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE); } #endif - return p; + return mi_ptr_bounded_data(p, size - MI_PADDING_SIZE); } @@ -154,17 +156,17 @@ void _mi_page_block_unguard(mi_page_t* page, mi_block_t* block, void* p) { const size_t psize = _mi_os_page_size(); mi_assert_internal(bsize > psize); if (page->memid.is_pinned) return; // no guard page was set (see `mi_block_ptr_set_guarded`) - void* gpage = (uint8_t*)block + bsize - psize; + void* gpage = mi_page_vmem_ptr(page, (uint8_t*)block + bsize - psize); mi_assert_internal(mi_is_aligned(gpage, psize)); - _mi_os_unprotect(gpage, psize); + _mi_os_unguard(gpage, psize, true /* committed */); } // unguard a whole page (called from `mi_heap_destroy`) void _mi_page_unguard_all(mi_page_t* page) { if mi_likely(!mi_page_has_interior_pointers(page) || page->memid.is_pinned) return; - uint8_t* const start = mi_page_start(page); + uint8_t* const start = (uint8_t*)mi_page_vmem_ptr(page, mi_page_start(page)); const size_t psize = mi_page_committed(page); - _mi_os_unprotect(start,psize); // unprotect all at once as we cannot know which blocks are guarded + _mi_os_unguard(start, psize, true /* committed */); // unguard all at once as we cannot know which blocks are guarded } #else void _mi_page_block_unguard(mi_page_t* page, mi_block_t* block, void* p) { diff --git a/system/lib/mimalloc/src/profile/sample-profile.c b/system/lib/mimalloc/src/profile/sample-profile.c index ec1bf1c3fcaba..4291493f4bd07 100644 --- a/system/lib/mimalloc/src/profile/sample-profile.c +++ b/system/lib/mimalloc/src/profile/sample-profile.c @@ -179,7 +179,7 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* th // Set up the profiled block as an interior pointer so the interior "slow" path is taken on mi_free (where we catch it to call on_free) mi_page_set_has_interior_pointers(page, true); block->next = MI_BLOCK_TAG_PROFILED; - p = (uint8_t*)block + user_offset; + p = mi_ptr_bounded((uint8_t*)block + user_offset, req_size); // exclude the profile data in front mi_profiler_sample_data_t* const sample_data = (mi_profiler_sample_data_t*)((uint8_t*)block + sample_data_offset); sample_data->user_data_size = sample_user_data_size; @@ -235,9 +235,9 @@ mi_decl_export bool mi_heap_profile(mi_heap_t* heap, mi_profiler_t* profiler) { } mi_decl_export void mi_heap_profile_disable(mi_heap_t* heap) { - static mi_profiler_t mi_nosample_profiler = { NULL, 0, MI_SAMPLE_RATE_MAX, NULL, NULL, NULL, NULL }; + static const mi_profiler_t mi_nosample_profiler = { NULL, 0, MI_SAMPLE_RATE_MAX, NULL, NULL, NULL, NULL }; mi_heap_set_profiler(heap,NULL); // first clear any previous one - mi_heap_set_profiler(heap,&mi_nosample_profiler); // and then install the nosample one + mi_heap_set_profiler(heap,(mi_profiler_t*)&mi_nosample_profiler); // and then install the nosample one } mi_decl_export bool mi_subproc_profile(mi_subproc_id_t subproc_id, mi_profiler_t* profiler) { @@ -282,8 +282,7 @@ bool mi_profiler_stop(mi_profiler_t* profiler) { void mi_profiler_snapshot(mi_profiler_t* profiler) { if (profiler==NULL) return; if (profiler->on_snapshot==NULL) return; - static mi_atomic_guard_t guard; - mi_atomic_guard(&guard) { + mi_atomic_guard(&_mi_process.profile_snapshot_guard) { (*profiler->on_snapshot)(profiler); } } diff --git a/system/lib/mimalloc/src/static.c b/system/lib/mimalloc/src/static.c index edf493c6b423d..5d1bed62bcc71 100644 --- a/system/lib/mimalloc/src/static.c +++ b/system/lib/mimalloc/src/static.c @@ -55,7 +55,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "alloc/page-queue.c" #include "heap/heap.c" -#include "heap/init.c" +#include "heap/process.c" #include "heap/page-map.c" #include "heap/subproc.c" #include "heap/theap.c" diff --git a/system/lib/mimalloc/src/util/libc.c b/system/lib/mimalloc/src/util/libc.c index ef4d22966fd00..5242ab18cbda1 100644 --- a/system/lib/mimalloc/src/util/libc.c +++ b/system/lib/mimalloc/src/util/libc.c @@ -199,17 +199,17 @@ int _mi_getenv(const char* name, char* result, size_t result_size) { // If it returns `true`, the caller should call `_mi_atomic_once_release` after performing the action. // Other threads (than the initial thread that entered) will block until `_mi_atomic_once_release` has been called. bool _mi_atomic_once_enter(mi_atomic_once_t* once) { - const uintptr_t once_tid = mi_atomic_load_acquire(&once->tid); + const size_t once_tid = mi_atomic_load_acquire(&once->tid); if mi_likely(once_tid == 1) { return false; // already executed } - const mi_threadid_t current_tid = _mi_thread_id(); + const mi_thread_id_t current_tid = _mi_thread_id(); if (once_tid == current_tid) { return false; // recursive invocation; don't block on ourselves } mi_lock_acquire(&once->lock); - uintptr_t expected = 0; + size_t expected = 0; if (mi_atomic_cas_strong_acq_rel(&once->tid, &expected, current_tid)) { // could use atomic_load/store as well return true; // should execute and release } @@ -441,7 +441,7 @@ int _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) { uintmax_t x = 0; if (c == 'x' || c == 'u') { if (numtype == 'z') x = va_arg(args, size_t); - else if (numtype == 't') x = va_arg(args, uintptr_t); // unsigned ptrdiff_t + else if (numtype == 't') x = va_arg(args, size_t); // unsigned ptrdiff_t else if (numtype == 'L') x = va_arg(args, unsigned long long); else if (numtype == 'l') x = va_arg(args, unsigned long); else if (numtype == '8') x = va_arg(args, uint64_t); @@ -450,13 +450,13 @@ int _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) { } else if (c == 'p') { void* const p = va_arg(args, void*); - x = (uintptr_t)p; + x = mi_addr(p); mi_outs("0x", &out, end); start = out; width = (width >= 2 ? width - 2 : 0); } if (width == 0 && (c == 'x' || c == 'p')) { - if (c == 'p') { width = 2 * (x <= UINT32_MAX ? 4 : ((x >> 16) <= UINT32_MAX ? 6 : sizeof(void*))); } + if (c == 'p') { width = 2 * (x <= UINT32_MAX ? 4 : ((x >> 16) <= UINT32_MAX ? 6 : sizeof(size_t))); } if (width == 0) { width = 2; } if (alignright) { fill = '0'; } } @@ -465,7 +465,7 @@ int _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) { else if (c == 'i' || c == 'd') { // signed intmax_t x = 0; - if (numtype == 'z') x = va_arg(args, intptr_t ); + if (numtype == 'z') x = va_arg(args, ptrdiff_t); // signed size_t else if (numtype == 't') x = va_arg(args, ptrdiff_t); else if (numtype == 'L') x = va_arg(args, long long); else if (numtype == 'l') x = va_arg(args, long); diff --git a/system/lib/mimalloc/src/util/options.c b/system/lib/mimalloc/src/util/options.c index 6489ffbbcf5a6..2db2487688ca2 100644 --- a/system/lib/mimalloc/src/util/options.c +++ b/system/lib/mimalloc/src/util/options.c @@ -9,13 +9,11 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc/prim/prim-tls.h" #include "mimalloc/alloc/page.h" #include "mimalloc/util/options.h" +#include "mimalloc/heap/process.h" #include // stdin/stdout #include // abort -static long mi_max_error_count = 16; // stop outputting errors after this (use < 0 for no limit) -static long mi_max_warning_count = 16; // stop outputting warnings after this (use < 0 for no limit) - static void mi_add_stderr_output(void); int mi_version(void) { @@ -86,12 +84,8 @@ int mi_version(void) { #endif #ifndef MI_DEFAULT_PAGEMAP_COMMIT -#if defined(__APPLE__) && MI_PAGE_MAP_FLAT // when overloading malloc, we still get mixed pointers sometimes on macOS; this avoids a bad access -#define MI_DEFAULT_PAGEMAP_COMMIT 1 -#else #define MI_DEFAULT_PAGEMAP_COMMIT 0 #endif -#endif #ifndef MI_DEFAULT_PAGE_MAX_RECLAIM #define MI_DEFAULT_PAGE_MAX_RECLAIM (-1) // unlimited @@ -119,95 +113,97 @@ int mi_version(void) { #endif #ifndef MI_DEFAULT_STATS_MERGE_THRESHOLD -#define MI_DEFAULT_STATS_MERGE_THRESHOLD (512*1024) // 512 KiB +#define MI_DEFAULT_STATS_MERGE_THRESHOLD (4*1024*1024) // 4 MiB #endif #ifndef MI_DEFAULT_ARENA_PURGE_IMMEDIATE_SIZE #define MI_DEFAULT_ARENA_PURGE_IMMEDIATE_SIZE ((MI_SIZE_SIZE * MI_LARGE_PAGE_SIZE) / MI_KiB) // 32 MiB (in KiB) #endif -// Static options -static mi_option_desc_t mi_options[_mi_option_last] = +// Static option descriptions (the current values are in `_mi_process.options`) +static const mi_option_desc_t mi_option_descs[_mi_option_last] = { // stable options #if MI_DEBUG || defined(MI_SHOW_ERRORS) - { 1, MI_OPTION_UNINIT, MI_OPTION(show_errors) }, + { 1, MI_OPTION(show_errors) }, #else - { 0, MI_OPTION_UNINIT, MI_OPTION(show_errors) }, + { 0, MI_OPTION(show_errors) }, #endif - { 0, MI_OPTION_UNINIT, MI_OPTION(show_stats) }, - { MI_DEFAULT_VERBOSE, MI_OPTION_UNINIT, MI_OPTION(verbose) }, + { 0, MI_OPTION(show_stats) }, + { MI_DEFAULT_VERBOSE, MI_OPTION(verbose) }, // some of the following options are experimental and not all combinations are allowed. - { 1, MI_OPTION_UNINIT, MI_OPTION(deprecated_eager_commit) }, - { MI_DEFAULT_ARENA_EAGER_COMMIT, - MI_OPTION_UNINIT, MI_OPTION_LEGACY(arena_eager_commit,eager_region_commit) }, // eager commit arena's? 2 is used to enable this only on an OS that has overcommit (i.e. linux) - { 1, MI_OPTION_UNINIT, MI_OPTION_LEGACY(purge_decommits,reset_decommits) }, // purge decommits memory (instead of reset) (note: on linux this uses MADV_DONTNEED for decommit) - { MI_DEFAULT_ALLOW_LARGE_OS_PAGES, - MI_OPTION_UNINIT, MI_OPTION_LEGACY(allow_large_os_pages,large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's - { MI_DEFAULT_RESERVE_HUGE_OS_PAGES, - MI_OPTION_UNINIT, MI_OPTION(reserve_huge_os_pages) }, // per 1GiB huge pages - {-1, MI_OPTION_UNINIT, MI_OPTION(reserve_huge_os_pages_at) }, // reserve huge pages at node N - { MI_DEFAULT_RESERVE_OS_MEMORY, - MI_OPTION_UNINIT, MI_OPTION(reserve_os_memory) }, // reserve N KiB OS memory in advance (use `option_get_size`) - { 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_segment_cache) }, // cache N segments per thread - { 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_page_reset) }, // reset page memory on free - { 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_abandoned_page_purge) }, - { 0, MI_OPTION_UNINIT, MI_OPTION(deprecated_segment_reset) }, // reset segment memory on free (needs eager commit) - { 1, MI_OPTION_UNINIT, MI_OPTION(deprecated_eager_commit_delay) }, - { 1000,MI_OPTION_UNINIT, MI_OPTION_LEGACY(purge_delay,reset_delay) }, // purge delay in milli-seconds - { 0, MI_OPTION_UNINIT, MI_OPTION(use_numa_nodes) }, // 0 = use available numa nodes, otherwise use at most N nodes. - { 0, MI_OPTION_UNINIT, MI_OPTION_LEGACY(disallow_os_alloc,limit_os_alloc) }, // 1 = do not use OS memory for allocation (but only reserved arenas) - { 100, MI_OPTION_UNINIT, MI_OPTION(os_tag) }, // only apple specific for now but might serve more or less related purpose - { 32, MI_OPTION_UNINIT, MI_OPTION(max_errors) }, // maximum errors that are output - { 32, MI_OPTION_UNINIT, MI_OPTION(max_warnings) }, // maximum warnings that are output - { 10, MI_OPTION_UNINIT, MI_OPTION(deprecated_max_segment_reclaim)}, // max. percentage of the abandoned segments to be reclaimed per try. - { 0, MI_OPTION_UNINIT, MI_OPTION(destroy_on_exit)}, // release all OS memory on process exit; careful with dangling pointer or after-exit frees! - { MI_DEFAULT_ARENA_RESERVE, MI_OPTION_UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time (=1GiB) (use `option_get_size`) - { 4, MI_OPTION_UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's - { 1, MI_OPTION_UNINIT, MI_OPTION_LEGACY(deprecated_purge_extend_delay, decommit_extend_delay) }, - { MI_DEFAULT_DISALLOW_ARENA_ALLOC, MI_OPTION_UNINIT, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's) - { 400, MI_OPTION_UNINIT, MI_OPTION(retry_on_oom) }, // windows only: retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. - { 1, MI_OPTION_UNINIT, MI_OPTION(deprecated_visit_abandoned) }, - { 0, MI_OPTION_UNINIT, MI_OPTION(guarded_min) }, // only used when building with MI_GUARDED: minimal rounded object size for guarded objects - { MI_GiB, MI_OPTION_UNINIT, MI_OPTION(guarded_max) }, // only used when building with MI_GUARDED: maximal rounded object size for guarded objects - { 0, MI_OPTION_UNINIT, MI_OPTION(unsafe_guarded_precise) }, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0) - { MI_DEFAULT_GUARDED_SAMPLE_RATE, - MI_OPTION_UNINIT, MI_OPTION(guarded_sample_rate)}, // guard an object in the min/max range about every N requested bytes (=0) - { 0, MI_OPTION_UNINIT, MI_OPTION(guarded_sample_seed)}, - { 10000, MI_OPTION_UNINIT, MI_OPTION(generic_collect) }, // collect theaps every N (=10000) generic allocation calls - { 1, MI_OPTION_UNINIT, MI_OPTION_LEGACY(page_reclaim_on_free, abandoned_reclaim_on_free) },// reclaim abandoned (small) pages on a free: -1 = disable completely, 0 = only reclaim into the originating theap, 1 = reclaim on free across theaps - { MI_DEFAULT_PAGE_FULL_RETAIN, - MI_OPTION_UNINIT, MI_OPTION(page_full_retain) }, // number of (small) pages to retain in the free page queues - { 4, MI_OPTION_UNINIT, MI_OPTION(page_max_candidates) }, // max search to find a best page candidate - { 0, MI_OPTION_UNINIT, MI_OPTION(max_vabits) }, // max virtual address space bits - { MI_DEFAULT_PAGEMAP_COMMIT, - MI_OPTION_UNINIT, MI_OPTION(pagemap_commit) }, // commit the full pagemap upfront? - { 0, MI_OPTION_UNINIT, MI_OPTION(page_commit_on_demand) }, // commit pages on-demand (2 disables this only on overcommit systems (like Linux)) - { MI_DEFAULT_PAGE_MAX_RECLAIM, - MI_OPTION_UNINIT, MI_OPTION(page_max_reclaim) }, // don't reclaim (small) pages of the same originating theap if we already own N pages in that size class - { MI_DEFAULT_PAGE_CROSS_THREAD_MAX_RECLAIM, - MI_OPTION_UNINIT, MI_OPTION(page_cross_thread_max_reclaim) }, // don't reclaim (small) pages across threads if we already own N pages in that size class - { MI_DEFAULT_ALLOW_THP, - MI_OPTION_UNINIT, MI_OPTION(allow_thp) }, // allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the process. - { 0, MI_OPTION_UNINIT, MI_OPTION_LEGACY(arena_purge_min_size, minimal_purge_size) }, // set minimal purge size (in KiB) (=0). Using 0 resolves to either 64 (or 2048 if THP is enabled). - { MI_DEFAULT_ARENA_MAX_OBJECT_SIZE, - MI_OPTION_UNINIT, MI_OPTION(arena_max_object_size) }, // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit). - { 0, MI_OPTION_UNINIT, MI_OPTION(arena_is_numa_local) }, // associate local numa node with an initial arena allocation - { MI_DEFAULT_STATS_MERGE_ON_COLLECT, - MI_OPTION_UNINIT, MI_OPTION(stats_merge_on_collect) }, // on each theap collect, stats are merged with the parent heap - { MI_GiB, MI_OPTION_UNINIT, MI_OPTION(profile_alloc_interval) },// N bytes (=1 GiB default) between automatic dumps of the `MIMALLOC_PROFILE` heap profiler (use `option_get_size`); 0 disables automatic dumping - { 0, MI_OPTION_UNINIT, MI_OPTION(profile_inuse_interval) }, // N bytes between automatic dumps of the `MIMALLOC_PROFILE` heap profiler whenever in-use bytes grow by that amount (use `option_get_size`); 0 disables (default) - { 0, MI_OPTION_UNINIT, MI_OPTION(profile_time_interval) }, // N seconds between automatic dumps of the `MIMALLOC_PROFILE` heap profiler; 0 disables (default) - { 512*MI_KiB, MI_OPTION_UNINIT, MI_OPTION(profile_sample_rate) }, // sample rate in bytes for the `MIMALLOC_PROFILE` heap profiler (use `option_get_size`) (=512 KiB) - { MI_DEFAULT_STATS_MERGE_THRESHOLD, - MI_OPTION_UNINIT, MI_OPTION(stats_merge_threshold) }, // on a page stats update, merge theap stats into the heap if N bytes were freed/allocated; 0 disables (=512 KiB) - { MI_DEFAULT_ARENA_PURGE_IMMEDIATE_SIZE, - MI_OPTION_UNINIT, MI_OPTION(arena_purge_immediate_size) }, // ranges above this size are purged immediately + { 1, MI_OPTION(deprecated_eager_commit) }, + { MI_DEFAULT_ARENA_EAGER_COMMIT, MI_OPTION_LEGACY(arena_eager_commit,eager_region_commit) }, // eager commit arena's? 2 is used to enable this only on an OS that has overcommit (i.e. linux) + { 1, MI_OPTION_LEGACY(purge_decommits,reset_decommits) }, // purge decommits memory (instead of reset) (note: on linux this uses MADV_DONTNEED for decommit) + { MI_DEFAULT_ALLOW_LARGE_OS_PAGES, MI_OPTION_LEGACY(allow_large_os_pages,large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's + { MI_DEFAULT_RESERVE_HUGE_OS_PAGES, MI_OPTION(reserve_huge_os_pages) }, // per 1GiB huge pages + {-1, MI_OPTION(reserve_huge_os_pages_at) }, // reserve huge pages at node N + { MI_DEFAULT_RESERVE_OS_MEMORY, MI_OPTION(reserve_os_memory) }, // reserve N KiB OS memory in advance (use `option_get_size`) + { 0, MI_OPTION(deprecated_segment_cache) }, // cache N segments per thread + { 0, MI_OPTION(deprecated_page_reset) }, // reset page memory on free + { 0, MI_OPTION(deprecated_abandoned_page_purge) }, + { 0, MI_OPTION(deprecated_segment_reset) }, // reset segment memory on free (needs eager commit) + { 1, MI_OPTION(deprecated_eager_commit_delay) }, + { 1000, MI_OPTION_LEGACY(purge_delay,reset_delay) }, // purge delay in milli-seconds + { 0, MI_OPTION(use_numa_nodes) }, // 0 = use available numa nodes, otherwise use at most N nodes. + { 0, MI_OPTION_LEGACY(disallow_os_alloc,limit_os_alloc) }, // 1 = do not use OS memory for allocation (but only reserved arenas) + { 100, MI_OPTION(os_tag) }, // only apple specific for now but might serve more or less related purpose + { 32, MI_OPTION(max_errors) }, // maximum errors that are output + { 32, MI_OPTION(max_warnings) }, // maximum warnings that are output + { 10, MI_OPTION(deprecated_max_segment_reclaim)}, // max. percentage of the abandoned segments to be reclaimed per try. + { 0, MI_OPTION(destroy_on_exit)}, // release all OS memory on process exit; careful with dangling pointer or after-exit frees! + { MI_DEFAULT_ARENA_RESERVE, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time (=1GiB) (use `option_get_size`) + { 4, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's + { 1, MI_OPTION_LEGACY(deprecated_purge_extend_delay, decommit_extend_delay) }, + { MI_DEFAULT_DISALLOW_ARENA_ALLOC, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's) + { 400, MI_OPTION(retry_on_oom) }, // windows only: retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. + { 1, MI_OPTION(deprecated_visit_abandoned) }, + { 0, MI_OPTION(guarded_min) }, // only used when building with MI_GUARDED: minimal rounded object size for guarded objects + { MI_GiB, MI_OPTION(guarded_max) }, // only used when building with MI_GUARDED: maximal rounded object size for guarded objects + { 0, MI_OPTION(unsafe_guarded_precise) }, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0) + { MI_DEFAULT_GUARDED_SAMPLE_RATE, MI_OPTION(guarded_sample_rate)}, // guard an object in the min/max range about every N requested bytes (=0) + { 0, MI_OPTION(guarded_sample_seed)}, + { 10000, MI_OPTION(generic_collect) }, // collect theaps every N (=10000) generic allocation calls + { 0, MI_OPTION_LEGACY(page_reclaim_on_free, abandoned_reclaim_on_free) },// reclaim abandoned (small) pages on a free: -1 = disable completely, 0 (default) = only reclaim into the originating theap, 1 = reclaim on free across theaps + { MI_DEFAULT_PAGE_FULL_RETAIN, MI_OPTION(page_full_retain) }, // number of (small) pages to retain in the free page queues + { 4, MI_OPTION(page_max_candidates) }, // max search to find a best page candidate + { 0, MI_OPTION(max_vabits) }, // max virtual address space bits + { MI_DEFAULT_PAGEMAP_COMMIT, MI_OPTION(pagemap_commit) }, // commit the full pagemap upfront? + { 0, MI_OPTION(page_commit_on_demand) }, // commit pages on-demand (2 disables this only on overcommit systems (like Linux)) + { MI_DEFAULT_PAGE_MAX_RECLAIM, MI_OPTION(page_max_reclaim) }, // don't reclaim (small) pages of the same originating theap if we already own N pages in that size class + { MI_DEFAULT_PAGE_CROSS_THREAD_MAX_RECLAIM, MI_OPTION(page_cross_thread_max_reclaim) }, // don't reclaim (small) pages across threads if we already own N pages in that size class + { MI_DEFAULT_ALLOW_THP, MI_OPTION(allow_thp) }, // allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the process. + { 0, MI_OPTION_LEGACY(arena_purge_min_size, minimal_purge_size) }, // set minimal purge size (in KiB) (=0). Using 0 resolves to either 64 (or 2048 if THP is enabled). + { MI_DEFAULT_ARENA_MAX_OBJECT_SIZE, MI_OPTION(arena_max_object_size) }, // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit). + { 0, MI_OPTION(arena_is_numa_local) }, // associate local numa node with an initial arena allocation + { MI_DEFAULT_STATS_MERGE_ON_COLLECT, MI_OPTION(stats_merge_on_collect) }, // on each theap collect, stats are merged with the parent heap + { MI_GiB, MI_OPTION(profile_alloc_interval) },// N bytes (=1 GiB default) between automatic dumps of the `MIMALLOC_PROFILE` heap profiler (use `option_get_size`); 0 disables automatic dumping + { 0, MI_OPTION(profile_inuse_interval) }, // N bytes between automatic dumps of the `MIMALLOC_PROFILE` heap profiler whenever in-use bytes grow by that amount (use `option_get_size`); 0 disables (default) + { 0, MI_OPTION(profile_time_interval) }, // N seconds between automatic dumps of the `MIMALLOC_PROFILE` heap profiler; 0 disables (default) + { 512*MI_KiB, MI_OPTION(profile_sample_rate) }, // sample rate in bytes for the `MIMALLOC_PROFILE` heap profiler (use `option_get_size`) (=512 KiB) + { MI_DEFAULT_STATS_MERGE_THRESHOLD, MI_OPTION(stats_merge_threshold) }, // on a page stats update, merge theap stats into the heap if N bytes were freed/allocated; 0 disables (=4 MiB) + { MI_DEFAULT_ARENA_PURGE_IMMEDIATE_SIZE, MI_OPTION(arena_purge_immediate_size) }, // ranges above this size are purged immediately }; -static void mi_option_init(mi_option_desc_t* desc); +static void mi_option_init(mi_option_t option, mi_option_state_t* state); + +static const mi_option_desc_t* mi_option_desc(mi_option_t option) { + const mi_option_desc_t* desc = &mi_option_descs[option]; + mi_assert(desc->option == option); // index should match the option + return desc; +} + +// Get the option state with the value set (to the default if not yet set) (but not reading from the environment) +static mi_option_state_t* mi_option_state(mi_option_t option) { + mi_option_state_t* state = &_mi_process.options[option]; + if mi_unlikely(state->init == MI_OPTION_UNINIT) { + state->value = mi_option_desc(option)->default_value; + state->init = MI_OPTION_PENDING; + } + return state; +} // option value is stored in KiB so it can fit a `long` // todo: use mi_ssize_t for options? @@ -231,8 +227,6 @@ void _mi_options_init(void) { mi_option_t option = (mi_option_t)i; long l = mi_option_get(option); MI_UNUSED(l); // initialize } - mi_max_error_count = mi_option_get(mi_option_max_errors); - mi_max_warning_count = mi_option_get(mi_option_max_warnings); #if MI_GUARDED if (mi_option_get(mi_option_guarded_sample_rate) > 0) { if (mi_option_is_enabled(mi_option_allow_large_os_pages)) { @@ -275,9 +269,8 @@ mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_ // show options for (int i = 0; i < _mi_option_last; i++) { mi_option_t option = (mi_option_t)i; - long l = mi_option_get(option); MI_UNUSED(l); // possibly initialize - mi_option_desc_t* desc = &mi_options[option]; - _mi_fprintf(out, arg, "option '%s': %ld %s\n", desc->name, desc->value, (mi_option_has_size_in_kib(option) ? "KiB" : (mi_option_has_size_in_bytes(option) ? "B" : ""))); + long l = mi_option_get(option); // possibly initialize + _mi_fprintf(out, arg, "option '%s': %ld %s\n", mi_option_desc(option)->name, l, (mi_option_has_size_in_kib(option) ? "KiB" : (mi_option_has_size_in_bytes(option) ? "B" : ""))); } // show build configuration @@ -316,22 +309,19 @@ mi_decl_export void mi_options_print(void) mi_attr_noexcept { long _mi_option_get_fast(mi_option_t option) { mi_assert(option >= 0 && option < _mi_option_last); - mi_option_desc_t* desc = &mi_options[option]; - mi_assert(desc->option == option); // index should match the option - //mi_assert(desc->init != MI_OPTION_UNINIT); - return desc->value; + return mi_option_state(option)->value; } mi_decl_nodiscard long mi_option_get(mi_option_t option) { mi_assert(option >= 0 && option < _mi_option_last); if (option < 0 || option >= _mi_option_last) return 0; - mi_option_desc_t* desc = &mi_options[option]; - mi_assert(desc->option == option); // index should match the option - if mi_unlikely(desc->init == MI_OPTION_UNINIT) { - mi_option_init(desc); + mi_option_state_t* state = &_mi_process.options[option]; + if mi_unlikely(state->init < MI_OPTION_DEFAULTED) { // uninit or pending + state = mi_option_state(option); + mi_option_init(option, state); } - return desc->value; + return state->value; } mi_decl_nodiscard long mi_option_get_clamp(mi_option_t option, long min, long max) { @@ -353,15 +343,14 @@ mi_decl_nodiscard size_t mi_option_get_size(mi_option_t option) { void mi_option_set(mi_option_t option, long value) { mi_assert(option >= 0 && option < _mi_option_last); if (option < 0 || option >= _mi_option_last) return; - mi_option_desc_t* desc = &mi_options[option]; - mi_assert(desc->option == option); // index should match the option - desc->value = value; - desc->init = MI_OPTION_INITIALIZED; + mi_option_state_t* const state = &_mi_process.options[option]; + state->value = value; + state->init = MI_OPTION_INITIALIZED; // ensure min/max range; be careful to not recurse. - if (desc->option == mi_option_guarded_min && _mi_option_get_fast(mi_option_guarded_max) < value) { + if (option == mi_option_guarded_min && mi_option_state(mi_option_guarded_max)->value < value) { mi_option_set(mi_option_guarded_max, value); } - else if (desc->option == mi_option_guarded_max && _mi_option_get_fast(mi_option_guarded_min) > value) { + else if (option == mi_option_guarded_max && mi_option_state(mi_option_guarded_min)->value > value) { mi_option_set(mi_option_guarded_min, value); } } @@ -369,9 +358,9 @@ void mi_option_set(mi_option_t option, long value) { void mi_option_set_default(mi_option_t option, long value) { mi_assert(option >= 0 && option < _mi_option_last); if (option < 0 || option >= _mi_option_last) return; - mi_option_desc_t* desc = &mi_options[option]; - if (desc->init != MI_OPTION_INITIALIZED) { - desc->value = value; + mi_option_state_t* const state = mi_option_state(option); + if (state->init != MI_OPTION_INITIALIZED) { + state->value = value; } } @@ -406,28 +395,25 @@ static void mi_cdecl mi_out_stderr(const char* msg, void* arg) { // function we also buffer output that happens earlier. When // an output function is registered it is called immediately with // the output up to that point. -#ifndef MI_MAX_DELAY_OUTPUT -#define MI_MAX_DELAY_OUTPUT ((size_t)(16*1024)) -#endif -static char out_buf[MI_MAX_DELAY_OUTPUT+1]; -static _Atomic(size_t) out_len; +// The buffer is `_mi_process.output_buf` but the lock is static as it is used before process initialization +// and a zero lock is invalid on some platforms. static mi_lock_t out_buf_lock = MI_LOCK_INITIALIZER; static void mi_cdecl mi_out_buf(const char* msg, void* arg) { MI_UNUSED(arg); if (msg==NULL) return; - if (mi_atomic_load_acquire(&out_len)>=MI_MAX_DELAY_OUTPUT) return; + if (mi_atomic_load_acquire(&_mi_process.output_buf_len)>=MI_MAX_DELAY_OUTPUT) return; size_t n = _mi_strlen(msg); if (n==0 || n >= MI_MAX_DELAY_OUTPUT) return; // copy msg into the buffer mi_lock(&out_buf_lock) { - const size_t start = mi_atomic_add_acq_rel(&out_len, n); + const size_t start = mi_atomic_add_acq_rel(&_mi_process.output_buf_len, n); if (start < MI_MAX_DELAY_OUTPUT) { // check bound if (start+n >= MI_MAX_DELAY_OUTPUT) { n = MI_MAX_DELAY_OUTPUT-start-1; } - mi_memcpy(&out_buf[start], msg, n); + mi_memcpy(&_mi_process.output_buf[start], msg, n); } } } @@ -436,13 +422,13 @@ static void mi_out_buf_flush(mi_output_fun* out, bool no_more_buf, void* arg) { if (out==NULL) return; // claim (if `no_more_buf == true`, no more output will be added after this point) mi_lock(&out_buf_lock) { - size_t count = mi_atomic_add_acq_rel(&out_len, (no_more_buf ? MI_MAX_DELAY_OUTPUT : 1)); + size_t count = mi_atomic_add_acq_rel(&_mi_process.output_buf_len, (no_more_buf ? MI_MAX_DELAY_OUTPUT : 1)); // and output the current contents if (count>MI_MAX_DELAY_OUTPUT) count = MI_MAX_DELAY_OUTPUT; - out_buf[count] = 0; - out(out_buf,arg); + _mi_process.output_buf[count] = 0; + out(_mi_process.output_buf,arg); if (!no_more_buf) { - out_buf[count] = '\n'; // if continue with the buffer, insert a newline + _mi_process.output_buf[count] = '\n'; // if continue with the buffer, insert a newline } } } @@ -463,35 +449,30 @@ static void mi_cdecl mi_out_buf_stderr(const char* msg, void* arg) { // The program should only install a single output handler from a single thread // since otherwise the argument and output function may not match. -static _Atomic(void*) mi_out_default; // = // is `mi_output_fun*` (but some platforms don't support atomic function pointers) -static _Atomic(void*) mi_out_arg; // = NULL - +// The output function and argument are stored in `_mi_process.out_default/out_arg`. static mi_output_fun* mi_out_get_default(void** parg) { - mi_output_fun* const out = (mi_output_fun*)mi_atomic_load_ptr_acquire(void,&mi_out_default); - if (parg != NULL) { *parg = mi_atomic_load_ptr_acquire(void,&mi_out_arg); } + mi_output_fun* const out = (mi_output_fun*)mi_atomic_load_ptr_acquire(void,&_mi_process.out_default); + if (parg != NULL) { *parg = mi_atomic_load_ptr_acquire(void,&_mi_process.out_arg); } return (out == NULL ? &mi_out_buf : out); } void mi_register_output(mi_output_fun* out, void* arg) mi_attr_noexcept { - mi_atomic_store_ptr_release(void,&mi_out_default, (void*)(out == NULL ? &mi_out_stderr : out)); // stop using the delayed output buffer - mi_atomic_store_ptr_release(void,&mi_out_arg, arg); + mi_atomic_store_ptr_release(void,&_mi_process.out_default, (void*)(out == NULL ? &mi_out_stderr : out)); // stop using the delayed output buffer + mi_atomic_store_ptr_release(void,&_mi_process.out_arg, arg); if (out!=NULL) { mi_out_buf_flush(out,true,arg); } // output all the delayed output now } // add stderr to the delayed output after the module is loaded static void mi_add_stderr_output(void) { - mi_assert_internal(mi_out_default == NULL); + mi_assert_internal(_mi_process.out_default == NULL); mi_out_buf_flush(&mi_out_stderr, false, NULL); // flush current contents to stderr - mi_atomic_store_ptr_release(void,&mi_out_default,(void*)&mi_out_buf_stderr); // and add stderr to the delayed output - mi_atomic_store_ptr_release(void,&mi_out_arg,NULL); + mi_atomic_store_ptr_release(void,&_mi_process.out_default,(void*)&mi_out_buf_stderr); // and add stderr to the delayed output + mi_atomic_store_ptr_release(void,&_mi_process.out_arg,NULL); } // -------------------------------------------------------- // Messages, all end up calling `_mi_fputs`. // -------------------------------------------------------- -static _Atomic(size_t) error_count; // = 0; // when >= max_error_count stop emitting errors -static _Atomic(size_t) warning_count; // = 0; // when >= max_warning_count stop emitting warnings - static mi_decl_noinline bool mi_recurse_enter_prim(void) { return true; } @@ -549,7 +530,7 @@ void _mi_fprintf( mi_output_fun* out, void* arg, const char* fmt, ... ) { static void mi_vfprintf_thread(mi_output_fun* out, void* arg, const char* prefix, const char* fmt, va_list args) { if (prefix != NULL && _mi_strnlen(prefix,33) <= 32) { // && !_mi_is_main_thread()) { char tprefix[64]; - _mi_snprintf(tprefix, sizeof(tprefix), "%sthread 0x%tx: ", prefix, (uintptr_t)_mi_thread_id()); + _mi_snprintf(tprefix, sizeof(tprefix), "%sthread 0x%tx: ", prefix, (size_t)_mi_thread_id()); mi_vfprintf(out, arg, tprefix, fmt, args); } else { @@ -583,7 +564,8 @@ void _mi_verbose_message(const char* fmt, ...) { static void mi_show_error_message(const char* fmt, va_list args) { if (!mi_option_is_enabled(mi_option_verbose)) { if (!mi_option_is_enabled(mi_option_show_errors)) return; - if (mi_max_error_count >= 0 && (long)mi_atomic_increment_acq_rel(&error_count) > mi_max_error_count) return; + const long max_errors = mi_option_get(mi_option_max_errors); + if (max_errors >= 0 && (long)mi_atomic_increment_acq_rel(&_mi_process.error_count) > max_errors) return; } mi_vfprintf_thread(NULL, NULL, "mimalloc: error: ", fmt, args); } @@ -591,7 +573,8 @@ static void mi_show_error_message(const char* fmt, va_list args) { void _mi_warning_message(const char* fmt, ...) { if (!mi_option_is_enabled(mi_option_verbose)) { if (!mi_option_is_enabled(mi_option_show_errors)) return; - if (mi_max_warning_count >= 0 && (long)mi_atomic_increment_acq_rel(&warning_count) > mi_max_warning_count) return; + const long max_warnings = mi_option_get(mi_option_max_warnings); + if (max_warnings >= 0 && (long)mi_atomic_increment_acq_rel(&_mi_process.warning_count) > max_warnings) return; } va_list args; va_start(args,fmt); @@ -603,6 +586,8 @@ void _mi_warning_message(const char* fmt, ...) { #if MI_DEBUG mi_decl_noreturn mi_decl_cold void _mi_assert_fail(const char* assertion, const char* fname, unsigned line, const char* func ) mi_attr_noexcept { _mi_fprintf(NULL, NULL, "mimalloc: assertion failed: at \"%s\":%u, %s\n assertion: \"%s\"\n", fname, line, (func==NULL?"":func), assertion); + // flush delayed output (if not yet initialized) so the message is not lost + if (mi_atomic_load_ptr_acquire(void,&_mi_process.out_default) == NULL) { mi_out_buf_flush(&mi_out_stderr, true, NULL); } abort(); } #endif @@ -611,9 +596,7 @@ mi_decl_noreturn mi_decl_cold void _mi_assert_fail(const char* assertion, const // Errors // -------------------------------------------------------- -static mi_error_fun* volatile mi_error_handler; // = NULL -static _Atomic(void*) mi_error_arg; // = NULL - +// The error handler and argument are stored in `_mi_process.error_handler/error_arg`. static void mi_error_default(int err) { MI_UNUSED(err); #if (MI_DEBUG>0) @@ -624,7 +607,7 @@ static void mi_error_default(int err) { abort(); } #endif - #if (MI_SECURE>0) + #if MI_SECURE if (err==EFAULT) { // abort on serious errors in secure mode (corrupted meta-data) abort(); } @@ -640,8 +623,8 @@ static void mi_error_default(int err) { } void mi_register_error(mi_error_fun* fun, void* arg) { - mi_error_handler = fun; // can be NULL - mi_atomic_store_ptr_release(void,&mi_error_arg, arg); + mi_atomic_store_ptr_release(void,&_mi_process.error_arg, arg); + mi_atomic_store_ptr_release(void,&_mi_process.error_handler, (void*)fun); // can be NULL } void _mi_error_message(int err, const char* fmt, ...) { @@ -651,8 +634,9 @@ void _mi_error_message(int err, const char* fmt, ...) { mi_show_error_message(fmt, args); va_end(args); // and call the error handler which may abort (or return normally, potentially setting errno) - if (mi_error_handler != NULL) { - mi_error_handler(err, mi_atomic_load_ptr_acquire(void,&mi_error_arg)); + mi_error_fun* const handler = (mi_error_fun*)mi_atomic_load_ptr_acquire(void,&_mi_process.error_handler); + if (handler != NULL) { + handler(err, mi_atomic_load_ptr_acquire(void,&_mi_process.error_arg)); } else { mi_error_default(err); @@ -660,7 +644,7 @@ void _mi_error_message(int err, const char* fmt, ...) { } mi_decl_noinline mi_block_t* _mi_block_next_is_corrupted(const mi_page_t* page, const mi_block_t* block, const mi_block_t* next) { - _mi_error_message(EFAULT, "corrupted free list entry of size %zub at %p: value 0x%zx\n", mi_page_block_size(page), block, (uintptr_t)next); + _mi_error_message(EFAULT, "corrupted free list entry of size %zub at %p: value 0x%zx\n", mi_page_block_size(page), block, (size_t)mi_addr(next)); return NULL; } @@ -668,7 +652,8 @@ mi_decl_noinline mi_block_t* _mi_block_next_is_corrupted(const mi_page_t* page, // Initialize options by checking the environment // -------------------------------------------------------- -static void mi_option_init(mi_option_desc_t* desc) { +static void mi_option_init(mi_option_t option, mi_option_state_t* state) { + const mi_option_desc_t* const desc = mi_option_desc(option); // Read option value from the environment char s[64 + 1]; char buf[64+1]; @@ -691,18 +676,18 @@ static void mi_option_init(mi_option_desc_t* desc) { } buf[len] = 0; if (buf[0] == 0 || _mi_streq(buf,"1") || _mi_streq(buf,"TRUE") || _mi_streq(buf,"YES") || _mi_streq(buf,"ON")) { - desc->value = 1; - desc->init = MI_OPTION_INITIALIZED; + state->value = 1; + state->init = MI_OPTION_INITIALIZED; } else if (_mi_streq(buf,"0") || _mi_streq(buf,"FALSE") || _mi_streq(buf,"NO") || _mi_streq(buf,"OFF")) { - desc->value = 0; - desc->init = MI_OPTION_INITIALIZED; + state->value = 0; + state->init = MI_OPTION_INITIALIZED; } else { const char* end = buf; long value = 0; const bool ok = _mi_strntol(buf, sizeof(buf), &end, &value); - if (ok && (mi_option_has_size_in_kib(desc->option) || mi_option_has_size_in_bytes(desc->option))) { + if (ok && (mi_option_has_size_in_kib(option) || mi_option_has_size_in_bytes(option))) { // the size is interpreted in bytes. size_t size = (value < 0 ? 0 : (size_t)value); bool overflow = false; @@ -717,7 +702,7 @@ static void mi_option_init(mi_option_desc_t* desc) { if (end[0] == 'I' && end[1] == 'B') { end += 2; } // KiB, MiB, GiB, TiB else if (*end == 'B') { end++; } // Kb, Mb, Gb, Tb if (overflow || size > MI_MAX_ALLOC_SIZE) { size = MI_MAX_ALLOC_SIZE; } - if (mi_option_has_size_in_kib(desc->option)) { + if (mi_option_has_size_in_kib(option)) { // This option is interpreted in KiB to prevent overflow of `long` for large allocations // (long is 32-bit on 64-bit windows, which allows for 4TiB max.) size = (size + MI_KiB - 1) / MI_KiB; // no overflow due to check before this @@ -725,27 +710,27 @@ static void mi_option_init(mi_option_desc_t* desc) { value = (size > LONG_MAX ? LONG_MAX : (long)size); } if (ok && *end == 0) { - mi_option_set(desc->option, value); + mi_option_set(option, value); } else { // set `init` first to avoid recursion through _mi_warning_message on mimalloc_verbose. - desc->init = MI_OPTION_DEFAULTED; - if (desc->option == mi_option_verbose && desc->value == 0) { + state->init = MI_OPTION_DEFAULTED; + if (option == mi_option_verbose && state->value == 0) { // if the 'mimalloc_verbose' env var has a bogus value we'd never know // (since the value defaults to 'off') so in that case briefly enable verbose - desc->value = 1; + state->value = 1; _mi_warning_message("environment option mimalloc_%s has an invalid value.\n", desc->name); - desc->value = 0; + state->value = 0; } else { _mi_warning_message("environment option mimalloc_%s has an invalid value.\n", desc->name); } } } - mi_assert_internal(desc->init != MI_OPTION_UNINIT); + mi_assert_internal(state->init >= MI_OPTION_DEFAULTED); } else if (err==ENOENT) { - desc->init = MI_OPTION_DEFAULTED; + state->init = MI_OPTION_DEFAULTED; } - // and on another error, keep unitialized to try again (can happen during preloading if getenv is not available) + // and on another error, keep pending to try again (can happen during preloading if getenv is not available) } diff --git a/system/lib/mimalloc/src/util/random.c b/system/lib/mimalloc/src/util/random.c index 1bfad4bc16863..9e49a00d9dc74 100644 --- a/system/lib/mimalloc/src/util/random.c +++ b/system/lib/mimalloc/src/util/random.c @@ -135,8 +135,9 @@ static bool mi_random_is_initialized(mi_random_ctx_t* ctx) { void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* ctx_new) { mi_assert_internal(ctx != ctx_new); if mi_unlikely(!mi_random_is_initialized(ctx)) { _mi_random_init(ctx); } - const uintptr_t nonce_rnd = _mi_random_next(ctx); - chacha_split(ctx, (uintptr_t)ctx_new ^ nonce_rnd /*nonce*/, ctx_new); + else if mi_unlikely(ctx->weak) { _mi_random_init(ctx); } // try again to get strong randomness (as it may have been unavailable early on) + const size_t nonce_rnd = _mi_random_next(ctx); + chacha_split(ctx, mi_addr(ctx_new) ^ nonce_rnd /*nonce*/, ctx_new); } size_t _mi_random_next(mi_random_ctx_t* ctx) { @@ -160,8 +161,8 @@ To initialize a fresh random context. If we cannot get good randomness, we fall back to weak randomness based on a timer and ASLR. -----------------------------------------------------------------------------*/ -size_t _mi_os_random_weak(size_t extra_seed) { - size_t x = (size_t)&_mi_os_random_weak ^ extra_seed; // ASLR makes the address random +static size_t mi_random_weak(void) { + size_t x = (size_t)(uintptr_t)&mi_random_weak; // ASLR makes the address random x ^= _mi_prim_clock_now(); // and do a few randomization steps size_t max = ((x ^ (x >> 17)) & 0x0F) + 1; @@ -172,15 +173,16 @@ size_t _mi_os_random_weak(size_t extra_seed) { return x; } -static void mi_random_init_ex(mi_random_ctx_t* ctx, bool use_weak) { +void _mi_random_init(mi_random_ctx_t* ctx) { uint8_t key[32]; - if (use_weak || !_mi_prim_random_buf(key, sizeof(key))) { + if (!_mi_prim_random_buf(key, sizeof(key))) { // if we fail to get random data from the OS, we fall back to a // weak random source based on the current time - #if !defined(__wasi__) - if (!use_weak) { _mi_warning_message("unable to use secure randomness\n"); } + #if !defined(__wasi__) && !defined(__EMSCRIPTEN__) + static _Atomic(size_t) warned; // = 0 + if (mi_atomic_exchange_relaxed(&warned, (size_t)1) == 0) { _mi_warning_message("unable to use secure randomness\n"); } #endif - size_t x = _mi_os_random_weak(0); + size_t x = mi_random_weak(); for (size_t i = 0; i < 32; i+=4, x++) { x = mi_random_shuffle(x); key[i] = (uint8_t)(x); @@ -193,18 +195,10 @@ static void mi_random_init_ex(mi_random_ctx_t* ctx, bool use_weak) { else { ctx->weak = false; } - chacha_init(ctx, key, (uintptr_t)ctx /*nonce*/ ); + chacha_init(ctx, key, mi_addr(ctx) /*nonce*/ ); mi_memzero(key, sizeof(key)); } -void _mi_random_init(mi_random_ctx_t* ctx) { - mi_random_init_ex(ctx, false); -} - -void _mi_random_init_weak(mi_random_ctx_t * ctx) { - mi_random_init_ex(ctx, true); -} - void _mi_random_reinit_if_weak(mi_random_ctx_t * ctx) { if (ctx->weak) { _mi_random_init(ctx); diff --git a/system/lib/mimalloc/src/util/stats.c b/system/lib/mimalloc/src/util/stats.c index f2a4549024155..33d9a173b80d2 100644 --- a/system/lib/mimalloc/src/util/stats.c +++ b/system/lib/mimalloc/src/util/stats.c @@ -9,6 +9,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc/prim/prim-tls.h" #include "mimalloc/alloc/page.h" #include "mimalloc/util/stats.h" +#include "mimalloc/heap/process.h" // _mi_process.clock_* #if defined(_MSC_VER) && (_MSC_VER < 1920) #pragma warning(disable:4204) // non-constant aggregate initializer @@ -435,11 +436,9 @@ void _mi_stats_print(const char* name, size_t id, const mi_stats_t* stats, mi_ou } -static mi_msecs_t mi_process_start; // = 0 - // called on process init void _mi_stats_init(void) { - if (mi_process_start == 0) { mi_process_start = _mi_clock_start(); }; + if (_mi_process.clock_process_start == 0) { _mi_process.clock_process_start = _mi_clock_start(); } } static void mi_stats_add_into(mi_stats_t* to, const mi_stats_t* from) { @@ -546,23 +545,21 @@ void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept { // Basic timer for convenience; use milli-seconds to avoid doubles // ---------------------------------------------------------------- -static mi_msecs_t mi_clock_diff; - mi_msecs_t _mi_clock_now(void) { return _mi_prim_clock_now(); } mi_msecs_t _mi_clock_start(void) { - if (mi_clock_diff == 0.0) { - mi_msecs_t t0 = _mi_clock_now(); - mi_clock_diff = _mi_clock_now() - t0; + if (_mi_process.clock_overhead == 0) { + const mi_msecs_t t0 = _mi_clock_now(); + _mi_process.clock_overhead = _mi_clock_now() - t0; } return _mi_clock_now(); } mi_msecs_t _mi_clock_end(mi_msecs_t start) { mi_msecs_t end = _mi_clock_now(); - return (end - start - mi_clock_diff); + return (end - start - _mi_process.clock_overhead); } @@ -574,7 +571,7 @@ mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, s { mi_process_info_t pinfo; _mi_memzero_var(pinfo); - pinfo.elapsed = _mi_clock_end(mi_process_start); + pinfo.elapsed = _mi_clock_end(_mi_process.clock_process_start); { const mi_subproc_t* subproc = _mi_subproc_main(); if (subproc!=NULL) { const int64_t current = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.current)); diff --git a/system/lib/mimalloc/src/util/threadlocal.c b/system/lib/mimalloc/src/util/threadlocal.c index 3a8a8fdc463cc..208fab4e4db45 100644 --- a/system/lib/mimalloc/src/util/threadlocal.c +++ b/system/lib/mimalloc/src/util/threadlocal.c @@ -16,6 +16,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc/util/libc.h" #include "mimalloc/util/options.h" #include "mimalloc/util/threadlocal.h" +#include "mimalloc/heap/process.h" /* ----------------------------------------------------------- Each thread can have (a dynamically expanding) array of @@ -220,30 +221,25 @@ Create and free fresh TLS key's ----------------------------------------------------------- */ #include "../arena/bitmap.h" -static mi_lock_t mi_thread_locals_lock; // we need a lock in order to re-allocate the slot bits -static mi_bitmap_t* mi_thread_locals_free; // reuse an arena bitmap to track which slots were assigned (1=free, 0=in-use) -static mi_memid_t mi_thread_locals_memid; // provenance of mi_thread_locals_free -static size_t mi_thread_locals_version; // version to be able to reuse slots safely - void _mi_thread_locals_init(void) { - mi_lock_init(&mi_thread_locals_lock); + mi_lock_init(&_mi_process.thread_locals_lock); } void _mi_thread_locals_done(void) { - mi_lock(&mi_thread_locals_lock) { - mi_bitmap_t* const slots = mi_thread_locals_free; + mi_lock(&_mi_process.thread_locals_lock) { + mi_bitmap_t* const slots = _mi_process.thread_locals_free; if (slots!=NULL) { - _mi_meta_free(_mi_subproc_main(), slots, mi_thread_locals_memid); + _mi_meta_free(_mi_subproc_main(), slots, _mi_process.thread_locals_memid); } } - mi_lock_done(&mi_thread_locals_lock); + mi_lock_done(&_mi_process.thread_locals_lock); mi_thread_locals_delete(); mi_slot_fast_delete(); } // strange signature but allows us to reuse the arena code for claiming free pages -static bool mi_thread_local_claim_fun(size_t _slice_index, mi_arena_t* _arena, bool* keep_set) { - MI_UNUSED(_slice_index); MI_UNUSED(_arena); +static bool mi_thread_local_claim_fun(size_t _slice_index, void* _arg, bool* keep_set) { + MI_UNUSED(_slice_index); MI_UNUSED(_arg); *keep_set = false; return true; } @@ -252,10 +248,10 @@ static bool mi_thread_local_claim_fun(size_t _slice_index, mi_arena_t* _arena, b // (so if we reuse a slot it will be returning NULL initially when a thread tries to get it) static mi_thread_local_t mi_thread_local_claim(void) { size_t idx = 0; - if (mi_thread_locals_free != NULL && mi_bitmap_try_find_and_claim(mi_thread_locals_free,0,&idx,&mi_thread_local_claim_fun,NULL)) { - mi_thread_locals_version++; - if (mi_thread_locals_version >= MI_TLS_VERSION_MAX) { mi_thread_locals_version = 1; } /* wrap around the version */ - return mi_key_create( idx, mi_thread_locals_version); + if (_mi_process.thread_locals_free != NULL && mi_bitmap_try_find_and_claim(_mi_process.thread_locals_free,0,&idx,&mi_thread_local_claim_fun,NULL)) { + _mi_process.thread_locals_version++; + if (_mi_process.thread_locals_version >= MI_TLS_VERSION_MAX) { _mi_process.thread_locals_version = 1; } /* wrap around the version */ + return mi_key_create( idx, _mi_process.thread_locals_version); } else { return 0; @@ -263,7 +259,7 @@ static mi_thread_local_t mi_thread_local_claim(void) { } static bool mi_thread_local_create_expand(void) { - mi_bitmap_t* const slots = mi_thread_locals_free; + mi_bitmap_t* const slots = _mi_process.thread_locals_free; // 1024 bits at a time const size_t oldcount = (slots==NULL ? 0 : mi_bitmap_max_bits(slots)); const size_t newcount = 1024 + oldcount; @@ -278,12 +274,12 @@ static bool mi_thread_local_create_expand(void) { // copy over the previous bitmap const size_t oldsize = mi_bitmap_size(oldcount,NULL); mi_memcpy_aligned(newslots, slots, oldsize); - _mi_meta_free(_mi_subproc_main(), slots, mi_thread_locals_memid); + _mi_meta_free(_mi_subproc_main(), slots, _mi_process.thread_locals_memid); } mi_bitmap_init(newslots, newcount, true /* pretend already zero'd so we do not zero out the copied old entries */); mi_bitmap_unsafe_setN(newslots, oldcount, newcount - oldcount); /* set the new expanded slots as available */ - mi_thread_locals_free = newslots; - mi_thread_locals_memid = memid; + _mi_process.thread_locals_free = newslots; + _mi_process.thread_locals_memid = memid; return true; } @@ -291,7 +287,7 @@ static bool mi_thread_local_create_expand(void) { // create a fresh key mi_thread_local_t _mi_thread_local_create(void) { mi_thread_local_t key = 0; - mi_lock(&mi_thread_locals_lock) { + mi_lock(&_mi_process.thread_locals_lock) { key = mi_thread_local_claim(); if (key==0) { if (mi_thread_local_create_expand()) { @@ -308,8 +304,8 @@ mi_thread_local_t _mi_thread_local_create(void) { void _mi_thread_local_free(mi_thread_local_t key) { if (key==0) return; const size_t idx = mi_key_index(key); - mi_lock(&mi_thread_locals_lock) { - mi_bitmap_t* const slots = mi_thread_locals_free; + mi_lock(&_mi_process.thread_locals_lock) { + mi_bitmap_t* const slots = _mi_process.thread_locals_free; if (slots!=NULL && idx < mi_bitmap_max_bits(slots)) { mi_bitmap_set(slots,idx); } diff --git a/test/other/test_mimalloc_options.out b/test/other/test_mimalloc_options.out index dccb4c55bcbe9..bd2ebd270e608 100644 --- a/test/other/test_mimalloc_options.out +++ b/test/other/test_mimalloc_options.out @@ -1,4 +1,4 @@ -v3.6.0 +v3.6.1 option 'show_errors': 0 option 'show_stats': 0 option 'verbose': 1 @@ -34,7 +34,7 @@ option 'unsafe_guarded_precise': 0 option 'guarded_sample_rate': 0 option 'guarded_sample_seed': 0 option 'generic_collect': 10000 -option 'page_reclaim_on_free': 1 +option 'page_reclaim_on_free': 0 option 'page_full_retain': 2 option 'page_max_candidates': 4 option 'max_vabits': 0 @@ -51,7 +51,7 @@ option 'profile_alloc_interval': 1073741824 B option 'profile_inuse_interval': 0 B option 'profile_time_interval': 0 option 'profile_sample_rate': 524288 B -option 'stats_merge_threshold': 524288 B +option 'stats_merge_threshold': 4194304 B option 'arena_purge_immediate_size': 2048 KiB free mode : pagemap, page size: 76 mem tracking: none diff --git a/test/test_other.py b/test/test_other.py index 2be6bcc8beac8..a6c5eb2b13fb1 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -7870,10 +7870,10 @@ def test_dlmalloc_modes(self): 'O2': (['-O2'], 137000), 'emmalloc': (['-sMALLOC=emmalloc'], 185000), 'dlmalloc': (['-sMALLOC=dlmalloc'], 191000), - 'mimalloc': (['-sMALLOC=mimalloc'], 254000), + 'mimalloc': (['-sMALLOC=mimalloc'], 258000), 'emmalloc_O2': (['-sMALLOC=emmalloc', '-O2'], 131000), 'dlmalloc_O2': (['-sMALLOC=dlmalloc', '-O2'], 137000), - 'mimalloc_O2': (['-sMALLOC=mimalloc', '-O2'], 198000), + 'mimalloc_O2': (['-sMALLOC=mimalloc', '-O2'], 201000), }) # This test verifies the output code size of the different -sMALLOC= modes. def test_malloc_size(self, args, max_size):