From 01112cb3db7d7b00ba9938da3ba0bd0fa9946874 Mon Sep 17 00:00:00 2001 From: Baruch Even Date: Fri, 31 Jul 2026 00:26:39 +0300 Subject: [PATCH 1/4] druntime: return free pages of surviving GC pools to the OS minimize() can only unmap a pool that is entirely free, so a single live object anywhere in a pool pins every page of it. A long-lived process whose heap grew once and then freed most of it therefore keeps paying for that peak in resident memory however much of each pool has since become free. Add a page-level scavenger for the conservative GC that releases the individual free pages of the pools minimize() had to keep, with madvise(MADV_DONTNEED), and run it as a final phase of minimize(). It is off unless enabled through the new gcopt scavenge option, and bounded by scavengeBudget and scavengeMinFree so an existing minimize() caller cannot inherit an unbounded syscall storm. Heaps that the process has locked into memory are supported. Where a plain madvise would be refused because the range is locked, the sequence becomes munlock/madvise(MADV_DONTNEED)/mlock2(MLOCK_ONFAULT), which drops the page from the resident set while keeping the mlockall(MCL_CURRENT|MCL_FUTURE) guarantee: the page is locked again at the moment it faults back in, so it is never swappable and the allocation site needs to know nothing. That case is detected from the kernel's EINVAL rather than configured. Linux only, and a no-op elsewhere: no other platform offers a way to drop a page out of RSS while keeping it locked. --- changelog/druntime.gc-page-scavenging.dd | 41 + runtime/druntime/src/core/gc/config.d | 18 +- .../core/internal/gc/impl/conservative/gc.d | 17 + .../druntime/src/core/internal/gc/scavenger.d | 971 ++++++++++++++++++ 4 files changed, 1046 insertions(+), 1 deletion(-) create mode 100644 changelog/druntime.gc-page-scavenging.dd create mode 100644 runtime/druntime/src/core/internal/gc/scavenger.d diff --git a/changelog/druntime.gc-page-scavenging.dd b/changelog/druntime.gc-page-scavenging.dd new file mode 100644 index 00000000000..692a5577bad --- /dev/null +++ b/changelog/druntime.gc-page-scavenging.dd @@ -0,0 +1,41 @@ +The conservative GC can now return free pages of surviving pools to the OS + +`GC.minimize()` could only release a memory pool that was entirely free. One live +object anywhere in a pool pinned every page of it, so a long-lived program whose +heap grew once and then freed most of it kept paying for that peak in resident +memory no matter how much of each pool had become free. + +On Linux the conservative GC can now also release the individual free pages of +pools that survive, with `madvise(MADV_DONTNEED)`. It runs as a final phase of +`GC.minimize()` and is off by default; enable it with the `scavenge` GC +configuration option: + +------- +./myprogram --DRT-gcopt=scavenge:1 +------- + +A released page is handed back to the kernel but stays mapped, so the GC needs no +extra bookkeeping to reuse it: the first write after the page is carved out again +simply faults in a fresh zero page. + +Programs that lock their heap into memory are supported. Where a plain `madvise` +would be refused because the range is locked, the sequence becomes +`munlock`/`madvise(MADV_DONTNEED)`/`mlock2(MLOCK_ONFAULT)`, which drops the page +from the resident set while keeping the `mlockall(MCL_CURRENT|MCL_FUTURE)` +guarantee: the page is locked again at the moment it faults back in, so it is +never swappable. This is detected automatically. + +Three further options tune the phase: + +$(UL + $(LI `scavengeMinFree` - leave the heap alone while less than this much of it is + resident-but-free, so that small amounts of free memory do not pay for + syscalls. Defaults to 16 MB.) + $(LI `scavengeBudget` - the most memory one `GC.minimize()` call may release, + bounding how long the phase can run; call `GC.minimize()` again to release + more. Defaults to 256 MB.) + $(LI `scavengeNoHugePages` - apply `MADV_NOHUGEPAGE` to the pools being tracked. + A pool backed by transparent huge pages cannot be released a page at a time, + so turning this off largely defeats scavenging on a host with THP set to + `always`. On by default.) +) diff --git a/runtime/druntime/src/core/gc/config.d b/runtime/druntime/src/core/gc/config.d index 258183fd505..316618a88a3 100644 --- a/runtime/druntime/src/core/gc/config.d +++ b/runtime/druntime/src/core/gc/config.d @@ -27,6 +27,13 @@ struct Config float heapSizeFactor = 2.0; // heap size to used memory ratio string cleanup = "collect"; // select gc cleanup method none|collect|finalize + // Page scavenging (Linux only): hand whole free pool pages back to the OS from minimize(), for the + // common case where a pool is mostly free but too sparsely populated to be unmapped whole. + bool scavenge = false; // enable the scavenge phase of minimize() + @MemVal size_t scavengeBudget = 256 << 20; // most bytes one minimize() may release; caller loops for more + @MemVal size_t scavengeMinFree = 16 << 20; // leave the heap alone below this much resident-free + bool scavengeNoHugePages = true; // MADV_NOHUGEPAGE tracked pools; page-granular release needs it + @nogc nothrow: bool initialize() @@ -52,6 +59,8 @@ struct Config auto _minPoolSize = minPoolSize.bytes2prettyStruct; auto _maxPoolSize = maxPoolSize.bytes2prettyStruct; auto _incPoolSize = incPoolSize.bytes2prettyStruct; + auto _scavengeBudget = scavengeBudget.bytes2prettyStruct; + auto _scavengeMinFree = scavengeMinFree.bytes2prettyStruct; printf(" - select gc implementation (default = conservative) initReserve:N - initial memory to reserve in MB (%lld%c) @@ -62,13 +71,20 @@ struct Config heapSizeFactor:N - targeted heap size to used memory ratio (%g) cleanup:none|collect|finalize - how to treat live objects when terminating (collect) + scavenge:0|1 - release free pool pages to the OS from minimize() (%d) + scavengeBudget:N - most memory one minimize() may release in MB (%lld%c) + scavengeMinFree:N - skip scavenging below this much resident-free memory in MB (%lld%c) + scavengeNoHugePages:0|1 - MADV_NOHUGEPAGE scavenged pools (%d) + Memory-related values can use B, K, M or G suffixes. ".ptr, _initReserve.v, _initReserve.u, _minPoolSize.v, _minPoolSize.u, _maxPoolSize.v, _maxPoolSize.u, _incPoolSize.v, _incPoolSize.u, - cast(long)parallel, heapSizeFactor); + cast(long)parallel, heapSizeFactor, + scavenge, _scavengeBudget.v, _scavengeBudget.u, + _scavengeMinFree.v, _scavengeMinFree.u, scavengeNoHugePages); } string errorName() @nogc nothrow { return "GC"; } diff --git a/runtime/druntime/src/core/internal/gc/impl/conservative/gc.d b/runtime/druntime/src/core/internal/gc/impl/conservative/gc.d index 1ed42f1449d..1eaf1460e85 100644 --- a/runtime/druntime/src/core/internal/gc/impl/conservative/gc.d +++ b/runtime/druntime/src/core/internal/gc/impl/conservative/gc.d @@ -37,6 +37,7 @@ import core.internal.gc.bits; import core.internal.gc.os; import core.gc.config; import core.gc.gcinterface; +import core.internal.gc.scavenger; import core.internal.container.treap; import core.internal.spinlock; @@ -723,6 +724,7 @@ class ConservativeGC : GC lpool.setFreePageOffsets(pagenum + newsz, freesz - newPages); gcx.usedLargePages += newPages; lpool.freepages -= newPages; + scavengerOnPagesCarved(&lpool.base, pagenum + psz, newPages); debug (PRINTF) printFreeInfo(pool); } else @@ -813,6 +815,7 @@ class ConservativeGC : GC lpool.setFreePageOffsets(pagenum + psz + sz, freesz - sz); lpool.freepages -= sz; gcx.usedLargePages += sz; + scavengerOnPagesCarved(&lpool.base, pagenum + psz, sz); return (psz + sz) * PAGESIZE; } } @@ -1907,6 +1910,10 @@ struct Gcx cstdlib.free(pool); } + // Whole-pool unmap above can only take a pool that is 100% free. Pages free inside pools that + // survived are the general case of the same job, so release those too (bounded, see gcopt). + scavengerMinimizePhase(&this); + debug(PRINTF) printf("Done minimizing.\n"); } @@ -2660,6 +2667,7 @@ struct Gcx debug(COLLECT_PRINTF) printf("\tcollecting big %p\n", p); leakDetector.log_free(q, sentinel_size(q, npages * PAGESIZE - SENTINEL_EXTRA)); pool.pagetable[pn..pn+npages] = Bins.B_FREE; + scavengerOnPagesFreed(pool, pn, npages); if (pn < pool.searchStart) pool.searchStart = pn; freedLargePages += npages; pool.freepages += npages; @@ -2790,6 +2798,7 @@ struct Gcx pool.freeAllPageBits(pn); pool.pagetable[pn] = Bins.B_FREE; + scavengerOnPagesFreed(pool, pn, 1); // add to free chain pool.binPageChain[pn] = cast(uint) pool.searchStart; pool.searchStart = pn; @@ -3536,6 +3545,7 @@ struct Pool size_t npages; size_t freepages; // The number of pages not in use. Bins* pagetable; + ubyte* scavengedMap; // one byte/page: scavenger dirty/clean tracking, see core.internal.gc.scavenger bool isLargeObject; @@ -3665,11 +3675,15 @@ struct Pool this.freepages = npages; this.searchStart = 0; this.largestFree = npages; + + scavengerOnPoolCreate(&this); } void Dtor() nothrow { + scavengerOnPoolDestroy(&this); + if (baseAddr) { int result; @@ -4140,6 +4154,7 @@ struct LargeObjectPool bPageOffsets[i + offset] = cast(uint) offset; } freepages -= n; + scavengerOnPagesCarved(&this.base, i, n); return i; } if (p > largest) @@ -4174,6 +4189,7 @@ struct LargeObjectPool } freepages += npages; largestFree = freepages; // invalidate + scavengerOnPagesFreed(&this.base, pagenum, npages); } /** @@ -4436,6 +4452,7 @@ struct SmallObjectPool binPageChain[pn] = Pool.PageRecovered; pagetable[pn] = bin; freepages--; + scavengerOnPagesCarved(&this.base, pn, 1); // Convert page to free list size_t size = binsize[bin]; diff --git a/runtime/druntime/src/core/internal/gc/scavenger.d b/runtime/druntime/src/core/internal/gc/scavenger.d new file mode 100644 index 00000000000..3fdad5fd579 --- /dev/null +++ b/runtime/druntime/src/core/internal/gc/scavenger.d @@ -0,0 +1,971 @@ +/** + * Page-level scavenger for the conservative GC. + * + * Returns whole free (`Bins.B_FREE`) pages of the GC's pools to the OS with + * `madvise(MADV_DONTNEED)`. This covers the case `minimize()` alone cannot: + * unmapping a pool requires it to be 100% free, so a long-lived process whose + * heap grew once and then freed most of it keeps paying for that peak in RSS + * forever, however much of each pool is free. + * + * On a heap the process has mlocked (`mlockall(MCL_CURRENT|MCL_FUTURE)`, or an + * explicit `mlock` over the heap) plain `madvise` is refused, so the sequence + * becomes `munlock -> madvise(MADV_DONTNEED) -> mlock2(MLOCK_ONFAULT)`. That + * preserves the lock guarantee while still dropping the pages: a scavenged page + * stays locked and out of RSS until the GC re-carves it, at which point the + * first touch faults in a zero page that is locked at fault time -- no second + * fault, no swap exposure, and nothing for the allocation site to know about. + * + * This module owns the bookkeeping (per-pool `scavengedMap`, the global + * dirty-free-page counter, arm/disable state); the call sites live in + * `core.internal.gc.impl.conservative.gc` as one-line hooks wherever a page + * transitions to or from `Bins.B_FREE` and wherever a pool is created or + * destroyed. + * + * Scavenging is off unless enabled through `gcopt` (see `core.gc.config`), and + * is a no-op outside Linux: dropping a page out of RSS while keeping an mlock + * guarantee takes `madvise(MADV_DONTNEED)` followed immediately by + * `mlock2(MLOCK_ONFAULT)`, and no other platform offers that pair. + * + * Copyright: D Language Foundation 2026. + * License: $(HTTP www.boost.org/LICENSE_1_0.txt, Boost License 1.0). + * Authors: Baruch Even + */ +module core.internal.gc.scavenger; + +import core.internal.gc.impl.conservative.gc : Pool, Gcx, Bins, PAGESIZE; +import core.gc.config : config; + +/// Status codes reported by `scavengerArm`/`scavengerStatus`. +enum ScavengeStatus : int +{ + OK = 0, + NOT_ARMED = 1, + DISABLED_DEBUG_BUILD = 2, + DISABLED_MLOCK2_ENOSYS = 3, + DISABLED_SYSCALL_FAILED = 4, + DISABLED_MAPS_PRESSURE = 5, // transient; reported for the last pass only + DISABLED_BY_CONFIG = 6, // gcopt scavenge:0 + DISABLED_PAGE_SIZE_MISMATCH = 7, // kernel page size is not the GC's PAGESIZE -- see scavengerArm +} + +// Linux-only mechanism -- see the module documentation. Everywhere else the hooks below are no-ops, so +// gc.d's call sites stay source-compatible on every target for the price of one null pointer per pool. +version (linux) {} else +{ + void scavengerOnPoolCreate(Pool*) nothrow @nogc {} + void scavengerOnPoolDestroy(Pool*) nothrow @nogc {} + void scavengerOnPagesFreed(Pool*, size_t, size_t) nothrow @nogc {} + void scavengerOnPagesCarved(Pool*, size_t, size_t) nothrow @nogc {} + size_t scavengerPass(Gcx*, size_t) nothrow @nogc { return 0; } + void scavengerMinimizePhase(Gcx*) nothrow @nogc {} + size_t scavengerDirtyFreeBytes() nothrow @nogc { return 0; } + + // Reported as never-armed rather than as a distinct "unsupported platform" code: to every caller a + // process that cannot scavenge is indistinguishable from one that never armed. + int scavengerArm(Gcx*, bool) nothrow @nogc { return ScavengeStatus.NOT_ARMED; } + int scavengerStatus() nothrow @nogc { return ScavengeStatus.NOT_ARMED; } +} + +version (linux): + +import core.sys.linux.sys.mman : madvise, MADV_DONTNEED, MADV_NOHUGEPAGE; +import core.sys.posix.sys.mman : mlock, munlock; +import core.stdc.errno : errno, ENOSYS, EINVAL; +import core.sys.posix.unistd : _SC_PAGESIZE; +import core.stdc.string : memset; +static import cstdlib = core.stdc.stdlib; + +// Compiled out entirely under the debug versions that either write patterns +// into freed memory or track it via shadow memory -- MADV_DONTNEED would +// zero it out from under them / desync the shadow state. +debug (MEMSTOMP) + private enum compiledOut = true; +else debug (SENTINEL) + private enum compiledOut = true; +else debug (VALGRIND) + private enum compiledOut = true; +else + private enum compiledOut = false; + +// Below this, syscall and VMA churn isn't worth it. Expressed in bytes and converted with the GC's own +// PAGESIZE rather than assuming 4K, so it stays 64KB whatever the pool page size is. +private enum MIN_RUN_BYTES = 64 * 1024; +private enum MIN_RUN_PAGES = MIN_RUN_BYTES / PAGESIZE; +private enum MLOCK_ONFAULT = 1; +private enum MAP_COUNT_HEADROOM_NUM = 9; // stop a pass above 90% of vm.max_map_count +private enum MAP_COUNT_HEADROOM_DEN = 10; + +// Mutated only under the GC lock: the hooks run inside gc.d with the lock already held, and so does the +// minimize() phase. The status and counter readers are deliberately lock-free -- a stale read of a gauge or +// of a status code is harmless. +private +{ + __gshared size_t g_dirtyFreePages; // pages with B_FREE && !scavenged, across all pools + __gshared bool g_armed; + __gshared bool g_heapMlocked; // process ran mlockall(MCL_CURRENT|MCL_FUTURE) + __gshared bool g_stickyDisabled; + __gshared int g_status = ScavengeStatus.NOT_ARMED; + __gshared size_t g_poolCursor; // rotates which pool a pass starts scanning from + __gshared bool g_lockedDetected; // madvise said EINVAL: the heap is locked, switch modes and retry + __gshared size_t g_kernelPageSize = PAGESIZE; // madvise/mlock2 granularity; see scavengePool + __gshared long g_maxMapCount; // vm.max_map_count, read once: 0 = not yet read, -1 = unreadable + + // mlock2() has no druntime binding (checked core.sys.{posix,linux}.sys.mman), so it is issued as a raw + // syscall and the number has to be known per architecture. Where it is not, only the locked-heap + // sequence is unavailable: a process whose heap is not mlocked never calls mlock2 and scavenges as usual. + version (X86_64) + enum SYS_mlock2 = 325; + else version (AArch64) + enum SYS_mlock2 = 284; + else + enum SYS_mlock2 = -1; + + enum haveMlock2 = SYS_mlock2 >= 0; + + extern (C) long syscall(long number, ...) nothrow @nogc; + + // Declared here with explicit attributes, like syscall above, rather than imported: the posix binding + // does not carry nothrow/@nogc and this is called from an @nogc arm path. + extern (C) long sysconf(int name) nothrow @nogc; + + // Matches the glibc wrapper's -1/errno convention. errno is NOT touched when mlock2 is unavailable on + // this architecture, so callers testing for absence must consult `haveMlock2`, never errno. + int mlock2(const scope void* addr, size_t len, int flags) nothrow @nogc + { + static if (!haveMlock2) + { + return -1; + } + else + { + return cast(int) syscall(cast(long) SYS_mlock2, addr, len, flags); + } + } + + // Clamped decrement: a transient inconsistency that ever tried to + // decrement past 0 would otherwise wrap a size_t to ~2^64 and leave the + // dirty-free gauge reporting garbage forever, instead of a bounded glitch. + void decrementDirtyFreePages(size_t n) nothrow @nogc + { + g_dirtyFreePages = g_dirtyFreePages >= n ? g_dirtyFreePages - n : 0; + } + + // scavengedMap semantics: one byte per page. Nonzero = the page's contents + // were discarded (madvised away, or provably never resident) -- not counted + // in g_dirtyFreePages. Zero = dirty: resident-while-free, or carved/live + // (carved pages are always zero). Only meaningful while the pagetable entry + // is B_FREE. + pragma(inline, true) + bool isMarkedScavenged(const(Pool)* pool, size_t pn) nothrow @nogc + { + return pool.scavengedMap[pn] != 0; + } + + pragma(inline, true) + void markScavenged(Pool* pool, size_t pn) nothrow @nogc + { + pool.scavengedMap[pn] = 1; + } + + pragma(inline, true) + void clearScavengedMark(Pool* pool, size_t pn) nothrow @nogc + { + pool.scavengedMap[pn] = 0; + } + + void allocScavengedMap(Pool* pool) nothrow @nogc + { + if (pool.scavengedMap !is null) + { + return; // already allocated (re-entrant call, or hook invoked twice defensively) + } + pool.scavengedMap = cast(ubyte*) cstdlib.malloc(pool.npages); + // OOM: leave null. The pool becomes untracked -- never counted, never + // scavenged -- which degrades gracefully instead of crashing the GC. + } + + // Best-effort attempt to restore the mlockall(MCL_CURRENT|MCL_FUTURE) + // guarantee after a failed step leaves a range unlocked. Tries the + // ONFAULT variant first (cheap, matches scavenger intent); falls back to + // a plain (prefaulting) mlock() so the range is at least locked again. + void relockBestEffort(void* addr, size_t len) nothrow @nogc + { + if (mlock2(addr, len, MLOCK_ONFAULT) != 0) + { + mlock(addr, len); + } + } + + // Normalizes one pool's VM flags to match the current mlocked state and + // (re)initializes its scavengedMap accordingly. Used both by + // scavengerOnPoolCreate (fresh pools, `freshPool = true`) and + // scavengerArm (existing pools at arm time, `freshPool = false`). + // Returns false iff the scavenger got sticky-disabled while normalizing + // this pool (munlock/mlock2 failure). + // + // CLEAN(1) is only correct for a genuinely fresh, never-mlocked pool -- + // its pages are provably never faulted. Every other case (a fresh + // mlocked pool, whose pages MCL_FUTURE already prefaulted; or an + // existing pool at arm time, whose B_FREE pages likely carry real usage + // history regardless of lock state) treats them as DIRTY(0). This can + // over-count pages that were in fact never faulted (e.g. the never- + // touched tail of a pool at arm time); the over-count is accepted -- + // it self-corrects on the first no-op scavenge of those pages, which + // finds nothing resident to reclaim and marks them CLEAN from there on. + bool normalizeAndTrackPool(Pool* pool, bool freshPool) nothrow @nogc + { + allocScavengedMap(pool); + if (pool.scavengedMap is null) + { + return true; // degrade gracefully: untracked pool, not a failure + } + + auto addr = pool.baseAddr; + auto len = pool.npages * PAGESIZE; + + // Advisory only -- a failure here doesn't compromise correctness, it just means THP interaction + // is whatever it was before. A THP-backed pool cannot be released a page at a time, so turning + // this off (gcopt scavengeNoHugePages:0) largely defeats scavenging on a THP=always host. + if (config.scavengeNoHugePages) + { + cast(void) madvise(addr, len, MADV_NOHUGEPAGE); + } + + bool failed; + if (g_heapMlocked) + { + if (munlock(addr, len) != 0 || mlock2(addr, len, MLOCK_ONFAULT) != 0) + { + relockBestEffort(addr, len); + g_status = ScavengeStatus.DISABLED_SYSCALL_FAILED; + g_stickyDisabled = true; + failed = true; + // Fall through to the same map init as the success path: with + // g_heapMlocked true here, that's always DIRTY(0) -- giving + // up on this pool's lock state doesn't mean giving up on the + // "carved pages are always 0" invariant onPagesFreed's debug + // assert relies on, and once sticky-disabled nothing will + // ever act on this map for scavenging again anyway. + } + } + + immutable clean = freshPool && !g_heapMlocked; + // 1 = marked-scavenged (clean, never faulted); 0 = dirty -- see + // isMarkedScavenged/markScavenged above. + memset(pool.scavengedMap, clean ? 1 : 0, pool.npages); + return !failed; + } + + // The munlock -> madvise(DONTNEED) -> mlock2(ONFAULT) triple for one run + // of already-verified dirty-free pages. Sticky-disables on any failure. + bool scavengeRun(void* addr, size_t len) nothrow @nogc + { + if (g_heapMlocked && munlock(addr, len) != 0) + { + // Nothing touched yet -- no relock needed. + g_status = ScavengeStatus.DISABLED_SYSCALL_FAILED; + g_stickyDisabled = true; + return false; + } + + if (madvise(addr, len, MADV_DONTNEED) != 0) + { + // EINVAL on a range we believe is unlocked is the kernel telling us it is locked after all + // (mlockall, or someone else's mlock). Ask the pass to switch to the locked sequence and + // retry -- and note the lock is pre-existing, so re-locking afterwards restores what was + // there rather than locking memory that wasn't. + if (!g_heapMlocked && errno == EINVAL) + { + g_lockedDetected = true; + return false; + } + + if (g_heapMlocked) + { + relockBestEffort(addr, len); + } + g_status = ScavengeStatus.DISABLED_SYSCALL_FAILED; + g_stickyDisabled = true; + return false; + } + + if (g_heapMlocked && mlock2(addr, len, MLOCK_ONFAULT) != 0) + { + relockBestEffort(addr, len); + g_status = ScavengeStatus.DISABLED_SYSCALL_FAILED; + g_stickyDisabled = true; + return false; + } + + return true; + } + + // Scans one pool's pagetable for maximal dirty-free runs and scavenges + // them (min-run filtered) until `maxBytes` is spent. Returns bytes + // actually scavenged; stops immediately (returning what's scavenged so + // far) if a syscall failure sticky-disables the scavenger mid-pool. + size_t scavengePool(Pool* pool, size_t maxBytes) nothrow @nogc + { + if (pool.scavengedMap is null) + { + return 0; + } + + size_t scavenged; + size_t pn; + while (pn < pool.npages && scavenged < maxBytes) + { + if (pool.pagetable[pn] != Bins.B_FREE || isMarkedScavenged(pool, pn)) + { + ++pn; + continue; + } + + immutable runStart = pn; + while (pn < pool.npages && pool.pagetable[pn] == Bins.B_FREE && !isMarkedScavenged(pool, pn)) + { + ++pn; + } + immutable runLen = pn - runStart; + + // madvise/mlock2 act on whole kernel pages: an unaligned address is refused outright and the + // length is rounded *up*, which would discard live pages past the end of the run. So work on + // the kernel-page-aligned interior of the run and leave its edges dirty -- they cost one page + // each and are picked up whenever the neighbouring run grows to cover them. Where the kernel + // page is the GC's page (every 4K host) this is arithmetically a no-op. + immutable mask = g_kernelPageSize - 1; + auto begin = cast(size_t)(pool.baseAddr + runStart * PAGESIZE); + auto end = begin + runLen * PAGESIZE; + begin = (begin + mask) & ~mask; + end &= ~mask; + if (end <= begin) + { + continue; // the run does not span a whole kernel page; pn is already past it + } + + size_t len = end - begin; + + // Budget clamp in kernel pages, so what is left is still aligned. + if (scavenged + len > maxBytes) + { + len = (maxBytes - scavenged) & ~mask; + } + + // Minimum run is checked *after* trimming: before it, a run can clear the bar and then trim + // away to nothing, which wastes the scan and makes scavengedBytes disagree with the gauge. + if (len < MIN_RUN_BYTES) + { + if (scavenged + MIN_RUN_BYTES > maxBytes) + { + break; // not enough budget left for a worthwhile run + } + continue; + } + + if (!scavengeRun(cast(void*) begin, len)) + { + return scavenged; // sticky-disabled, or lock detected: either way the pass handles it + } + + // Mark only what was actually discarded, in GC pages. + immutable firstPage = (begin - cast(size_t) pool.baseAddr) / PAGESIZE; + immutable pageCount = len / PAGESIZE; + foreach (i; firstPage .. firstPage + pageCount) + { + markScavenged(pool, i); + } + decrementDirtyFreePages(pageCount); + scavenged += len; + } + + return scavenged; + } + + // Only relevant to the locked-heap sequence, and only consulted there (see scavengerPass): + // munlock()/mlock2() on a sub-range of a pool's VMA can transiently split it into as many as three VMAs + // (before/during/after the touched range) until the kernel re-merges them once flags become uniform + // again, and each split consumes an extra vm.max_map_count entry. Close to the ceiling the split itself + // can fail with ENOMEM mid-munlock/mlock2 -- which would sticky-disable the scavenger over what is + // really a transient condition -- and a process at the map limit starts failing unrelated mmap() calls + // too. So refuse to start a pass without headroom rather than find out mid-run. The resulting + // DISABLED_MAPS_PRESSURE status is transient (not sticky) and re-checked every pass. + // + // vm.max_map_count is read once and cached: it is a boot/sysctl value, and /proc/self/maps has to be + // read line by line under the GC lock as it is, so there is no reason to pay for both every pass. An + // unreadable /proc fails open (never blocks scavenging) and is not retried. + bool mapCountUnderPressure() nothrow @nogc + { + if (g_maxMapCount == 0) + { + immutable v = readIntFile("/proc/sys/vm/max_map_count\0".ptr); + g_maxMapCount = v > 0 ? v : -1; + } + if (g_maxMapCount < 0) + { + return false; // unreadable -- fail open rather than block scavenging on a missing /proc + } + + immutable mapCount = countLines("/proc/self/maps\0".ptr); + return mapCount * MAP_COUNT_HEADROOM_DEN > cast(size_t) g_maxMapCount * MAP_COUNT_HEADROOM_NUM; + } + + long readIntFile(const(char)* path) nothrow @nogc + { + import core.sys.posix.fcntl : open, O_RDONLY; + import core.sys.posix.unistd : read, close; + + int fd = open(path, O_RDONLY); + if (fd < 0) + { + return -1; + } + scope (exit) close(fd); + + char[32] buf = void; + auto n = read(fd, buf.ptr, buf.length); + if (n <= 0) + { + return -1; + } + + long val = 0; + foreach (c; buf[0 .. n]) + { + if (c < '0' || c > '9') + { + break; + } + val = val * 10 + (c - '0'); + } + return val; + } + + size_t countLines(const(char)* path) nothrow @nogc + { + import core.sys.posix.fcntl : open, O_RDONLY; + import core.sys.posix.unistd : read, close; + + int fd = open(path, O_RDONLY); + if (fd < 0) + { + return 0; + } + scope (exit) close(fd); + + char[4096] buf = void; + size_t lines; + for (;;) + { + auto n = read(fd, buf.ptr, buf.length); + if (n <= 0) + { + break; + } + foreach (c; buf[0 .. n]) + { + if (c == '\n') + { + ++lines; + } + } + } + return lines; + } +} + +// ============================================================================ +// Public API -- consumed by impl.conservative.gc's hook call sites and +// extern(C) entry points. +// ============================================================================ + +/// Pool creation hook: `Pool.initialize`, after npages/freepages are set. +void scavengerOnPoolCreate(Pool* pool) nothrow @nogc +{ + static if (compiledOut) + { + pool.scavengedMap = null; + } + else + { + // os_mem_map() failed: initialize() zeroes npages/leaves baseAddr + // null in that case. Nothing to map or lock-dance; a 0-length + // munlock/mlock2 call would be pure risk for zero benefit. + if (pool.npages == 0) + { + pool.scavengedMap = null; + return; + } + + if (!g_armed) + { + allocScavengedMap(pool); + if (pool.scavengedMap !is null) + { + // 1 = every page of this new pool starts marked-scavenged (never faulted). + memset(pool.scavengedMap, 1, pool.npages); + } + return; + } + + // A freshly created pool is 100% B_FREE; if the heap is mlocked, + // MCL_FUTURE has already prefaulted it resident (normalizeAndTrackPool + // marks the map DIRTY(0) in that case, CLEAN(1) otherwise). + if (normalizeAndTrackPool(pool, /* freshPool = */ true) + && pool.scavengedMap !is null && g_heapMlocked) + { + g_dirtyFreePages += pool.npages; + } + } +} + +/// Pool destruction hook: `Pool.Dtor`, before anything else in the function +/// (in particular before `pagetable` is freed and before `os_mem_unmap`). +void scavengerOnPoolDestroy(Pool* pool) nothrow @nogc +{ + static if (!compiledOut) + { + if (pool.scavengedMap is null) + { + return; + } + + size_t dirty; + foreach (pn; 0 .. pool.npages) + { + if (pool.pagetable[pn] == Bins.B_FREE && !isMarkedScavenged(pool, pn)) + { + ++dirty; + } + } + decrementDirtyFreePages(dirty); + + cstdlib.free(pool.scavengedMap); + pool.scavengedMap = null; + } +} + +/// Hook for every place `npages` pages starting at `pagenum` transition to +/// `Bins.B_FREE`: sweep's large-object branch, sweep's small-object +/// recoverPage branch, and `LargeObjectPool.freePages` (which alone covers +/// reallocNoSync's shrink path, freeNoSync's large-object path, and +/// runFinalizers -- all three call through it). +/// +/// A carved page's map bit is always 0 (DIRTY) by the carve-hook's own +/// invariant, so freeing never needs to read or write the map -- only bump +/// the counter. +void scavengerOnPagesFreed(Pool* pool, size_t pagenum, size_t npages) nothrow @nogc +{ + static if (!compiledOut) + { + if (pool.scavengedMap is null) + { + return; + } + + debug + { + foreach (pn; pagenum .. pagenum + npages) + { + assert(!isMarkedScavenged(pool, pn), "carved pages must stay DIRTY in the scavenged map"); + } + } + + g_dirtyFreePages += npages; + } +} + +/// Hook for every place `npages` pages starting at `pagenum` are carved out +/// of `Bins.B_FREE`: `SmallObjectPool.allocPage`, `LargeObjectPool.allocPages`, +/// `Gcx.extendNoSync`, and `Gcx.reallocNoSync`'s expand-in-place path. +void scavengerOnPagesCarved(Pool* pool, size_t pagenum, size_t npages) nothrow @nogc +{ + static if (!compiledOut) + { + if (pool.scavengedMap is null) + { + return; + } + + foreach (pn; pagenum .. pagenum + npages) + { + if (isMarkedScavenged(pool, pn)) + { + clearScavengedMark(pool, pn); // was clean-free, no counter change + } + else + { + decrementDirtyFreePages(1); // was dirty-free, now consumed + } + } + } +} + +/// One-time arm: capability-probes mlock2 (if `heapIsMlocked`), normalizes +/// every existing pool, and counts every existing B_FREE page as dirty-free. +/// This over-counts a pool's never-touched tail (pages that are mapped but +/// were never actually faulted in, so they're already effectively clean); +/// the over-count is accepted -- it self-corrects on the first no-op +/// scavenge of those pages. Assumes the caller holds the GC lock. +int scavengerArm(Gcx* gcx, bool heapIsMlocked) nothrow @nogc +{ + static if (compiledOut) + return ScavengeStatus.DISABLED_DEBUG_BUILD; + else + { + if (g_armed) + { + return g_status; // one-time; already armed, report current status + } + + if (!config.scavenge) + { + g_status = ScavengeStatus.DISABLED_BY_CONFIG; + g_stickyDisabled = true; + return g_status; + } + + // madvise() and mlock2() work in kernel pages: they require a page-aligned address and round the + // length *up*. The GC's PAGESIZE is a hardcoded 4096, so where the kernel page is larger -- 64KB + // aarch64 kernels, notably -- a free run is guaranteed neither to start nor to end on a kernel-page + // boundary. An unaligned start fails EINVAL, which scavengeRun would misread as "the heap is locked + // after all"; an unaligned end silently discards live pages past the run. Refuse to arm until runs + // are trimmed to the kernel-page-aligned interior. + // + // Runs are trimmed to this granularity in scavengePool, so it only has to be known, not matched. + // A sysconf failure (-1), or anything below the GC's own page, still disables: the consequence of + // guessing this wrong is discarding live memory, so it fails closed. + immutable kernelPageSize = sysconf(_SC_PAGESIZE); + if (kernelPageSize < PAGESIZE || (kernelPageSize & (kernelPageSize - 1)) != 0) + { + g_status = ScavengeStatus.DISABLED_PAGE_SIZE_MISMATCH; + g_stickyDisabled = true; + return g_status; + } + g_kernelPageSize = cast(size_t) kernelPageSize; + + if (heapIsMlocked && (!haveMlock2 || (mlock2(null, 0, MLOCK_ONFAULT) != 0 && errno == ENOSYS))) + { + g_status = ScavengeStatus.DISABLED_MLOCK2_ENOSYS; + g_stickyDisabled = true; + return g_status; + } + + g_heapMlocked = heapIsMlocked; + + // Authoritative from here: pools created before this arm() call may + // already have driven onPagesFreed/onPagesCarved (their map/counter + // updates are otherwise-harmless bookkeeping against an unarmed, + // never-scavenged state -- see scavengerOnPoolCreate). Discard + // whatever that pre-arm history left in the counter and recompute it + // from the ground truth (pagetable[]) below, once per pool. + g_dirtyFreePages = 0; + + foreach (pool; gcx.pooltable[]) + { + if (!normalizeAndTrackPool(pool, /* freshPool = */ false)) + { + return g_status; // sticky-disabled inside; stop arming further pools + } + + if (pool.scavengedMap !is null) + { + foreach (pn; 0 .. pool.npages) + { + if (pool.pagetable[pn] == Bins.B_FREE) + { + ++g_dirtyFreePages; + } + } + } + } + + g_armed = true; + g_status = ScavengeStatus.OK; + return g_status; + } +} + +/// Scavenges up to `maxBytes` worth of dirty-free pages across all pools, +/// resuming from a rotating pool cursor so consecutive passes don't always +/// restart at pool 0. Assumes the caller holds the GC lock. Returns 0 if +/// unarmed, sticky-disabled, or under map-count pressure this pass. +// Promotes the process to the locked-heap sequence after madvise reported EINVAL, and re-normalizes every +// pool so their lock flags are uniform again -- otherwise later per-run munlock/mlock2 keeps splitting VMAs +// that the kernel would have merged, burning vm.max_map_count entries. +private bool switchToLockedHeap(Gcx* gcx) nothrow @nogc +{ + g_lockedDetected = false; + + if (!haveMlock2 || (mlock2(null, 0, MLOCK_ONFAULT) != 0 && errno == ENOSYS)) + { + g_status = ScavengeStatus.DISABLED_MLOCK2_ENOSYS; + g_stickyDisabled = true; + return false; + } + + g_heapMlocked = true; + + foreach (pool; gcx.pooltable[]) + { + if (!normalizeAndTrackPool(pool, /* freshPool = */ false)) + { + return false; // sticky-disabled inside + } + } + + return true; +} + +size_t scavengerPass(Gcx* gcx, size_t maxBytes) nothrow @nogc +{ + static if (compiledOut) + return 0; + else + { + // Arm on first use so a caller only has to enable the feature, not sequence it. Arming assumes an + // unlocked heap: that way nothing is mlocked that wasn't already, and if the heap turns out to be + // locked the first madvise says so (EINVAL) and switchToLockedHeap() promotes us. + if (!g_armed && scavengerArm(gcx, /* heapIsMlocked = */ false) != ScavengeStatus.OK) + { + return 0; + } + + if (g_stickyDisabled) + { + return 0; + } + + // Only the locked-heap sequence can split VMAs -- munlock/mlock2 change flags on a sub-range, + // while madvise(MADV_DONTNEED) changes none. An unlocked heap therefore cannot approach the + // map-count ceiling, and does not pay to read /proc/self/maps on every pass. + if (g_heapMlocked && mapCountUnderPressure()) + { + g_status = ScavengeStatus.DISABLED_MAPS_PRESSURE; // transient: not sticky, re-checked next pass + return 0; + } + g_status = ScavengeStatus.OK; + + auto pools = gcx.pooltable[]; + if (pools.length == 0) + { + return 0; + } + if (g_poolCursor >= pools.length) + { + g_poolCursor = 0; + } + + size_t scavenged; + // At most two laps: the second only happens if the first discovered the heap is locked, which can + // only be discovered once per process. + foreach (lap; 0 .. 2) + { + foreach (offset; 0 .. pools.length) + { + if (maxBytes - scavenged < MIN_RUN_PAGES * PAGESIZE) + { + break; + } + + auto pool = pools[(g_poolCursor + offset) % pools.length]; + scavenged += scavengePool(pool, maxBytes - scavenged); + + if (g_stickyDisabled || g_lockedDetected) + { + break; + } + } + + if (!g_lockedDetected || !switchToLockedHeap(gcx)) + { + break; + } + } + g_poolCursor = (g_poolCursor + 1) % pools.length; + + return scavenged; + } +} + +/// The scavenge phase of minimize(), called once the whole-pool unmap phase is done: the pools that +/// survived it are mostly-free-but-pinned, and their free pages are what page scavenging exists for. +/// Bounded by gcopt scavengeBudget so an existing minimize() caller cannot inherit an unbounded syscall +/// storm -- a caller with more to release calls minimize() again. +void scavengerMinimizePhase(Gcx* gcx) nothrow @nogc +{ + static if (compiledOut) + { + return; + } + else + { + if (!config.scavenge || g_stickyDisabled) + { + return; + } + + // Arm before consulting the counter: until arming recomputes it from the pagetables it only holds + // whatever the hooks happened to see, which understates a heap that was already free when we + // started. + if (!g_armed && scavengerArm(gcx, /* heapIsMlocked = */ false) != ScavengeStatus.OK) + { + return; + } + + if (g_dirtyFreePages * PAGESIZE <= config.scavengeMinFree) + { + return; + } + + cast(void) scavengerPass(gcx, config.scavengeBudget); + } +} + +/// `g_dirtyFreePages * PAGESIZE`. Lock-free read of a counter mutated only under the GC lock -- fine for a +/// monitoring gauge (the same tolerance as other unguarded __gshared GC stat counters, e.g. +/// `Gcx.mappedPages`). +size_t scavengerDirtyFreeBytes() nothrow @nogc +{ + static if (compiledOut) + return 0; + else + return g_dirtyFreePages * PAGESIZE; +} + +/// Current status/disabled-reason. See `ScavengeStatus` above. +int scavengerStatus() nothrow @nogc +{ + static if (compiledOut) + return ScavengeStatus.DISABLED_DEBUG_BUILD; + else + return g_status; +} + +version (unittest) +{ + // Kept out of the unittest body so the block pointers die with this frame rather than linger in it: + // the collector scans the stack conservatively, and a retained burst would defeat the test. + private void allocateBurst(size_t blockSize, size_t blocks, void*[] pinned) nothrow + { + import core.memory : GC; + + foreach (i; 0 .. blocks) + { + auto p = GC.malloc(blockSize, GC.BlkAttr.NO_SCAN); + if (i % (blocks / pinned.length) == 0) + { + pinned[i / (blocks / pinned.length)] = p; + } + } + } +} + +static if (!compiledOut) +unittest +{ + import core.memory : GC; + + if (scavengerStatus() == ScavengeStatus.DISABLED_DEBUG_BUILD) + { + return; + } + + // The phase only runs when enabled and only above its floor: force both, and put them back after. + immutable savedScavenge = config.scavenge; + immutable savedMinFree = config.scavengeMinFree; + config.scavenge = true; + config.scavengeMinFree = 0; + scope (exit) + { + config.scavenge = savedScavenge; + config.scavengeMinFree = savedMinFree; + } + + // Arms the scavenger and drains whatever was already free, so the counter read below is authoritative + // (before arming it holds only what the hooks happened to see). + GC.minimize(); + + // Pinning one block in eight keeps the pools alive: minimize()'s whole-pool unmap can only take a pool + // that is 100% free, and were it to take them there would be no free page left for the scavenge phase + // to find -- the assertions below would then hold without anything having been scavenged. + enum blockSize = 64 * 1024; // comfortably above MIN_RUN_BYTES once freed + enum blocks = 512; // 32 MB -- big enough that the burst still leaves several MB dirty at + // the final minimize(), whatever earlier collections already drained + __gshared void*[blocks / 8] pinned; + allocateBurst(blockSize, blocks, pinned[]); + GC.collect(); + + immutable dirtyBefore = scavengerDirtyFreeBytes(); + assert(dirtyBefore > 0, "the collected blocks should be counted as dirty-free"); + + GC.minimize(); + assert(scavengerDirtyFreeBytes() < dirtyBefore, "minimize() should have scavenged dirty-free pages"); + + assert(pinned[0] !is null); // keeps the pins observably live past the assertions +} + +// The large-kernel-page path, exercised on a 4K host by overriding the granularity. What was wrong before +// trimming was pure arithmetic -- an unaligned madvise start is refused, and the length is rounded up past +// the end of the run, discarding live pages -- so it is worth testing where the kernel that exposes it is +// not available. 64KB is the aarch64 case that matters (RHEL-family arm64 kernels). +static if (!compiledOut) +unittest +{ + import core.memory : GC; + + if (scavengerStatus() == ScavengeStatus.DISABLED_DEBUG_BUILD) + { + return; + } + + immutable savedScavenge = config.scavenge; + immutable savedMinFree = config.scavengeMinFree; + config.scavenge = true; + config.scavengeMinFree = 0; + scope (exit) + { + config.scavenge = savedScavenge; + config.scavengeMinFree = savedMinFree; + } + + GC.minimize(); // arm, and drain whatever was already free + + enum kernelPage = 64 * 1024; + // Deliberately NOT a multiple of the kernel page: 5 GC pages per block means free runs begin and end at + // arbitrary 4K offsets, which is what gives the trimming something to do. Sizing blocks at 64KB instead + // makes every run naturally aligned and the test passes with the trimming removed. + enum blockSize = 5 * PAGESIZE; + enum blocks = 512; + enum pinOneIn = 16; // runs of 15 blocks = 300KB, so a trimmed interior is 256KB + __gshared void*[blocks / pinOneIn] pinnedLargePage; + foreach (i; 0 .. blocks) + { + auto p = GC.malloc(blockSize, GC.BlkAttr.NO_SCAN); + if (i % pinOneIn == 0) + { + pinnedLargePage[i / pinOneIn] = p; // keeps the pools off minimize()'s whole-pool unmap path + } + } + GC.collect(); + + immutable savedPageSize = g_kernelPageSize; + g_kernelPageSize = kernelPage; + scope (exit) g_kernelPageSize = savedPageSize; + + immutable dirtyBefore = g_dirtyFreePages * PAGESIZE; + assert(dirtyBefore > 0, "the collected blocks should be counted as dirty-free"); + + GC.minimize(); + immutable dirtyAfter = g_dirtyFreePages * PAGESIZE; + + // Measured off the gauge rather than a counter so this test sits with the trimming it exercises. The + // pins keep every pool off minimize()'s whole-pool unmap path, so the only thing that can move the + // gauge is the scavenge phase. + assert(dirtyAfter < dirtyBefore, "trimming to kernel pages must still reclaim"); + immutable reclaimed = dirtyBefore - dirtyAfter; + // The assertion that matters: a reclaim that is not a whole number of kernel pages means the length was + // rounded, which on a real large-page kernel is how live memory past the run gets discarded. + assert(reclaimed % kernelPage == 0, "every reclaim must be a whole number of kernel pages"); + + assert(pinnedLargePage[0] !is null); +} From d314cf78b7e1aa1beb7ffd4ca0233911a78656cf Mon Sep 17 00:00:00 2001 From: Baruch Even Date: Fri, 31 Jul 2026 00:26:54 +0300 Subject: [PATCH 2/4] druntime: phase stats and a continuation hint for the GC scavenge phase The scavenge phase runs inside minimize() and cannot log -- it has no access to the host application's tracing -- so a live system had no way to tell whether it had acted, or which gate stopped it. Expose a ScavengeStats snapshot instead. A struct of named fields rather than a counter-by-index accessor: the caller reads one coherent snapshot instead of a sequence of independent reads, and adding a counter later cannot silently renumber the ones an existing caller already reads. Add a continuation hint alongside it. The phase stops on gcopt scavengeBudget, and nothing calls minimize() again on a quiet process, so record whether it left reclaimable pages behind rather than strand the remainder until the next caller happens along. --- .../druntime/src/core/internal/gc/scavenger.d | 86 ++++++++++++++++++- 1 file changed, 82 insertions(+), 4 deletions(-) diff --git a/runtime/druntime/src/core/internal/gc/scavenger.d b/runtime/druntime/src/core/internal/gc/scavenger.d index 3fdad5fd579..bf42b48d662 100644 --- a/runtime/druntime/src/core/internal/gc/scavenger.d +++ b/runtime/druntime/src/core/internal/gc/scavenger.d @@ -48,6 +48,23 @@ enum ScavengeStatus : int DISABLED_PAGE_SIZE_MISMATCH = 7, // kernel page size is not the GC's PAGESIZE -- see scavengerArm } +/// Why the minimize()-driven phase did or did not act. The phase runs inside minimize() and cannot log, +/// having no access to the host application's tracing, so a live system reads these instead. All monotonic. +struct ScavengeStats +{ + ulong minimizeCalls; // phase entered + ulong minimizePasses; // phase ran a scavenge pass + ulong scavengedBytes; // bytes the phase reclaimed + ulong skippedDisabled; // gcopt scavenge:0 + ulong skippedSticky; // sticky-disabled by an earlier syscall failure + ulong skippedNotArmed; // arming was attempted and did not report OK + ulong skippedBelowMinFree; // dirty-free under gcopt scavengeMinFree + ulong armedFromMinimize; // the phase armed the scavenger itself + ulong lockedPromotions; // madvise said EINVAL: promoted to the locked-heap sequence + ulong continuations; // phase hit its budget and asked to be called back +} + + // Linux-only mechanism -- see the module documentation. Everywhere else the hooks below are no-ops, so // gc.d's call sites stay source-compatible on every target for the price of one null pointer per pool. version (linux) {} else @@ -58,6 +75,9 @@ version (linux) {} else void scavengerOnPagesCarved(Pool*, size_t, size_t) nothrow @nogc {} size_t scavengerPass(Gcx*, size_t) nothrow @nogc { return 0; } void scavengerMinimizePhase(Gcx*) nothrow @nogc {} + ScavengeStats scavengerStats() nothrow @nogc { return ScavengeStats.init; } + int scavengerMinimizeContinuationDue() nothrow @nogc { return 0; } + size_t scavengerMinFree() nothrow @nogc { return 0; } size_t scavengerDirtyFreeBytes() nothrow @nogc { return 0; } // Reported as never-armed rather than as a distinct "unsupported platform" code: to every caller a @@ -107,6 +127,8 @@ private __gshared int g_status = ScavengeStatus.NOT_ARMED; __gshared size_t g_poolCursor; // rotates which pool a pass starts scanning from __gshared bool g_lockedDetected; // madvise said EINVAL: the heap is locked, switch modes and retry + __gshared ScavengeStats g_stats; + __gshared bool g_minimizeContinuationDue; // budget-capped phase left reclaimable pages behind __gshared size_t g_kernelPageSize = PAGESIZE; // madvise/mlock2 granularity; see scavengePool __gshared long g_maxMapCount; // vm.max_map_count, read once: 0 = not yet read, -1 = unreadable @@ -704,6 +726,7 @@ private bool switchToLockedHeap(Gcx* gcx) nothrow @nogc } g_heapMlocked = true; + ++g_stats.lockedPromotions; foreach (pool; gcx.pooltable[]) { @@ -799,28 +822,79 @@ void scavengerMinimizePhase(Gcx* gcx) nothrow @nogc } else { - if (!config.scavenge || g_stickyDisabled) + ++g_stats.minimizeCalls; + + if (!config.scavenge) { + g_minimizeContinuationDue = false; + ++g_stats.skippedDisabled; + return; + } + + if (g_stickyDisabled) + { + g_minimizeContinuationDue = false; + ++g_stats.skippedSticky; return; } // Arm before consulting the counter: until arming recomputes it from the pagetables it only holds // whatever the hooks happened to see, which understates a heap that was already free when we // started. - if (!g_armed && scavengerArm(gcx, /* heapIsMlocked = */ false) != ScavengeStatus.OK) + if (!g_armed) { - return; + ++g_stats.armedFromMinimize; + if (scavengerArm(gcx, /* heapIsMlocked = */ false) != ScavengeStatus.OK) + { + g_minimizeContinuationDue = false; + ++g_stats.skippedNotArmed; + return; + } } if (g_dirtyFreePages * PAGESIZE <= config.scavengeMinFree) { + g_minimizeContinuationDue = false; + ++g_stats.skippedBelowMinFree; return; } - cast(void) scavengerPass(gcx, config.scavengeBudget); + ++g_stats.minimizePasses; + const scavenged = scavengerPass(gcx, config.scavengeBudget); + g_stats.scavengedBytes += scavenged; + + // The budget bounds the scavenge phase only, so a big heap needs more than one visit. Nothing calls + // minimize() again on a quiet process, so ask the application to come back rather than strand the + // remainder. Same rule as the policy path: reclaimed something, and still above the gate. + g_minimizeContinuationDue = scavenged > 0 && (g_dirtyFreePages * PAGESIZE) > config.scavengeMinFree; + if (g_minimizeContinuationDue) + { + ++g_stats.continuations; + } } } +/// Whether the last minimize() scavenge phase stopped on its budget with reclaimable pages left. The +/// caller drives the follow-up; this module has no timer of its own. +int scavengerMinimizeContinuationDue() nothrow @nogc +{ + return g_minimizeContinuationDue ? 1 : 0; +} + +/// gcopt scavengeMinFree, so a follow-up uses the same gate the phase itself applies. +size_t scavengerMinFree() nothrow @nogc +{ + return config.scavengeMinFree; +} + +/// A snapshot of the phase's counters. Returned by value rather than by reference so a caller cannot +/// observe one field advancing while it reads another. +ScavengeStats scavengerStats() nothrow @nogc +{ + return g_stats; +} + + /// `g_dirtyFreePages * PAGESIZE`. Lock-free read of a counter mutated only under the GC lock -- fine for a /// monitoring gauge (the same tolerance as other unguarded __gshared GC stat counters, e.g. /// `Gcx.mappedPages`). @@ -896,10 +970,14 @@ unittest GC.collect(); immutable dirtyBefore = scavengerDirtyFreeBytes(); + immutable scavengedBefore = scavengerStats().scavengedBytes; assert(dirtyBefore > 0, "the collected blocks should be counted as dirty-free"); GC.minimize(); assert(scavengerDirtyFreeBytes() < dirtyBefore, "minimize() should have scavenged dirty-free pages"); + // Name the mechanism, not just the outcome: a pool unmapped whole would also drop the counter. + assert(scavengerStats().scavengedBytes > scavengedBefore, + "the drop must come from the scavenge phase"); assert(pinned[0] !is null); // keeps the pins observably live past the assertions } From bb4c2e715ae2a87265a302d4af783db64dd9c863 Mon Sep 17 00:00:00 2001 From: Baruch Even Date: Fri, 31 Jul 2026 00:28:27 +0300 Subject: [PATCH 3/4] druntime: extern(C) entry points for application-driven GC scavenging A host that wants to pace scavenging itself, rather than leave it to minimize(), needs to trigger a pass, read the dirty-free gauge and read back the phase stats. Expose them as extern(C) wrappers that resolve the live Gcx and take the GC lock, following the shape of ConservativeGC.minimize(). The lock-free readers are @nogc, so a host can call them directly from an @nogc context such as a statistics callback instead of caching their results. Also add the madvise/mlock2 failure injector the sticky-disable path is tested with, since it is only reachable from here. This is not the upstream-native shape -- that would be a GC interface method surfaced through core.memory, the way minimize() is, and would work under any GC rather than silently no-op under the others. Kept as a separate commit so it can be dropped or replaced without touching the mechanism. --- .../core/internal/gc/impl/conservative/gc.d | 79 +++++++++++++++++++ .../druntime/src/core/internal/gc/scavenger.d | 31 +++++++- 2 files changed, 108 insertions(+), 2 deletions(-) diff --git a/runtime/druntime/src/core/internal/gc/impl/conservative/gc.d b/runtime/druntime/src/core/internal/gc/impl/conservative/gc.d index 1eaf1460e85..ecd9d0fa26a 100644 --- a/runtime/druntime/src/core/internal/gc/impl/conservative/gc.d +++ b/runtime/druntime/src/core/internal/gc/impl/conservative/gc.d @@ -5177,3 +5177,82 @@ void undefinedWrite(T)(ref T var, T value) nothrow else var = value; } + +// ============================================================================ +// Page scavenger -- extern(C) entry points for an application that wants to +// drive scavenging itself rather than leave it to minimize(). All the +// bookkeeping lives in core.internal.gc.scavenger; these wrappers only +// resolve the live Gcx instance and take the GC lock, following the shape of +// ConservativeGC.minimize() above (lockNR / scope(failure) unlock / unlock). +// +// Posix-gated because resolving the live Gcx goes through Gcx.instance, which the GC only maintains there +// (it exists for the fork-safety machinery). The mechanism is Linux-only in any case, so on the remaining +// Posix targets these report a scavenger that never armed. +// ============================================================================ + +version (Posix): + +extern (C) size_t gc_scavenger_pass(size_t maxBytes) nothrow +{ + auto gcx = Gcx.instance; + if (gcx is null) + { + return 0; // GC not initialized yet, or not the conservative GC + } + + ConservativeGC.lockNR(); + scope (failure) ConservativeGC.gcLock.unlock(); + auto scavenged = scavengerPass(gcx, maxBytes); + ConservativeGC.gcLock.unlock(); + return scavenged; +} + +extern (C) size_t gc_scavenger_dirty_free_bytes() nothrow @nogc +{ + return scavengerDirtyFreeBytes(); +} + +extern (C) int gc_scavenger_arm(int heapIsMlocked) nothrow +{ + auto gcx = Gcx.instance; + if (gcx is null) + { + return ScavengeStatus.NOT_ARMED; // GC not initialized yet, or not the conservative GC + } + + ConservativeGC.lockNR(); + scope (failure) ConservativeGC.gcLock.unlock(); + auto status = scavengerArm(gcx, heapIsMlocked != 0); + ConservativeGC.gcLock.unlock(); + return status; +} + +extern (C) int gc_scavenger_status() nothrow @nogc +{ + return scavengerStatus(); +} + +// Why the minimize() scavenge phase did or did not act; see core.internal.gc.scavenger.ScavengeStats. +// Returned whole so the caller gets a coherent snapshot, and so adding a counter cannot silently +// renumber the ones a caller already reads. +extern (C) ScavengeStats gc_scavenger_stats() nothrow @nogc +{ + return scavengerStats(); +} + +// Set when the phase stopped on its budget with reclaimable pages left; the application drives the +// follow-up pass, and gc_scavenger_min_free() is the gate the phase itself uses. +extern (C) int gc_scavenger_continuation_due() nothrow @nogc +{ + return scavengerMinimizeContinuationDue(); +} + +extern (C) size_t gc_scavenger_min_free() nothrow @nogc +{ + return scavengerMinFree(); +} + +extern (C) void gc_scavenger_inject_fail(int mode) nothrow @nogc +{ + scavengerInjectFail(mode); +} diff --git a/runtime/druntime/src/core/internal/gc/scavenger.d b/runtime/druntime/src/core/internal/gc/scavenger.d index bf42b48d662..cab8615b136 100644 --- a/runtime/druntime/src/core/internal/gc/scavenger.d +++ b/runtime/druntime/src/core/internal/gc/scavenger.d @@ -79,6 +79,7 @@ version (linux) {} else int scavengerMinimizeContinuationDue() nothrow @nogc { return 0; } size_t scavengerMinFree() nothrow @nogc { return 0; } size_t scavengerDirtyFreeBytes() nothrow @nogc { return 0; } + void scavengerInjectFail(int) nothrow @nogc {} // Reported as never-armed rather than as a distinct "unsupported platform" code: to every caller a // process that cannot scavenge is indistinguishable from one that never armed. @@ -126,6 +127,7 @@ private __gshared bool g_stickyDisabled; __gshared int g_status = ScavengeStatus.NOT_ARMED; __gshared size_t g_poolCursor; // rotates which pool a pass starts scanning from + __gshared int g_injectFailMode; // test hook: 0=off, 1=next madvise(DONTNEED) fails, 2=next mlock2 fails __gshared bool g_lockedDetected; // madvise said EINVAL: the heap is locked, switch modes and retry __gshared ScavengeStats g_stats; __gshared bool g_minimizeContinuationDue; // budget-capped phase left reclaimable pages behind @@ -160,10 +162,27 @@ private } else { + if (g_injectFailMode == 2) + { + g_injectFailMode = 0; + return -1; + } return cast(int) syscall(cast(long) SYS_mlock2, addr, len, flags); } } + // Routes madvise(DONTNEED) through the inject_fail test hook; NOHUGEPAGE + // calls are unaffected (best-effort/advisory, never on the failure path). + int madviseHook(void* addr, size_t len, int advice) nothrow @nogc + { + if (advice == MADV_DONTNEED && g_injectFailMode == 1) + { + g_injectFailMode = 0; + return -1; + } + return madvise(addr, len, advice); + } + // Clamped decrement: a transient inconsistency that ever tried to // decrement past 0 would otherwise wrap a size_t to ~2^64 and leave the // dirty-free gauge reporting garbage forever, instead of a bounded glitch. @@ -250,7 +269,7 @@ private // this off (gcopt scavengeNoHugePages:0) largely defeats scavenging on a THP=always host. if (config.scavengeNoHugePages) { - cast(void) madvise(addr, len, MADV_NOHUGEPAGE); + cast(void) madviseHook(addr, len, MADV_NOHUGEPAGE); } bool failed; @@ -290,7 +309,7 @@ private return false; } - if (madvise(addr, len, MADV_DONTNEED) != 0) + if (madviseHook(addr, len, MADV_DONTNEED) != 0) { // EINVAL on a range we believe is unlocked is the kernel telling us it is locked after all // (mlockall, or someone else's mlock). Ask the pass to switch to the locked sequence and @@ -915,6 +934,14 @@ int scavengerStatus() nothrow @nogc return g_status; } +/// Test hook: force the next madvise(DONTNEED) (mode 1) or mlock2 (mode 2) +/// call to behave as failed, to exercise the sticky-disable path +/// end-to-end. 0 turns injection off. +void scavengerInjectFail(int mode) nothrow @nogc +{ + g_injectFailMode = mode; +} + version (unittest) { // Kept out of the unittest body so the block pointers die with this frame rather than linger in it: From dc711a2c7d0f9f55ec6cc083d76acd4823972c8d Mon Sep 17 00:00:00 2001 From: Baruch Even Date: Fri, 31 Jul 2026 00:28:43 +0300 Subject: [PATCH 4/4] Weka druntime modification: enable GC page scavenging by default Weka's processes lock their heap and depend on the scavenger to return it, so the gcopt option is on unless explicitly disabled rather than opt-in. Not for upstream: turning this on by default also turns MADV_NOHUGEPAGE on for every D program's GC heap, which is a throughput change nobody asked for. --- runtime/druntime/src/core/gc/config.d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/druntime/src/core/gc/config.d b/runtime/druntime/src/core/gc/config.d index 316618a88a3..df17affde2f 100644 --- a/runtime/druntime/src/core/gc/config.d +++ b/runtime/druntime/src/core/gc/config.d @@ -29,7 +29,7 @@ struct Config // Page scavenging (Linux only): hand whole free pool pages back to the OS from minimize(), for the // common case where a pool is mostly free but too sparsely populated to be unmapped whole. - bool scavenge = false; // enable the scavenge phase of minimize() + bool scavenge = true; // enable the scavenge phase of minimize() @MemVal size_t scavengeBudget = 256 << 20; // most bytes one minimize() may release; caller loops for more @MemVal size_t scavengeMinFree = 16 << 20; // leave the heap alone below this much resident-free bool scavengeNoHugePages = true; // MADV_NOHUGEPAGE tracked pools; page-granular release needs it