gc: return free pages of surviving GC pools to the OS

changelog/druntime.gc-page-scavenging.dd

@@ -0,0 +1,41 @@
+The conservative GC can now return free pages of surviving pools to the OS
+
+`GC.minimize()` could only release a memory pool that was entirely free. One live
+object anywhere in a pool pinned every page of it, so a long-lived program whose
+heap grew once and then freed most of it kept paying for that peak in resident
+memory no matter how much of each pool had become free.
+
+On Linux the conservative GC can now also release the individual free pages of
+pools that survive, with `madvise(MADV_DONTNEED)`. It runs as a final phase of
+`GC.minimize()` and is off by default; enable it with the `scavenge` GC
+configuration option:
+
+-------
+./myprogram --DRT-gcopt=scavenge:1
+-------
+
+A released page is handed back to the kernel but stays mapped, so the GC needs no
+extra bookkeeping to reuse it: the first write after the page is carved out again
+simply faults in a fresh zero page.
+
+Programs that lock their heap into memory are supported. Where a plain `madvise`
+would be refused because the range is locked, the sequence becomes
+`munlock`/`madvise(MADV_DONTNEED)`/`mlock2(MLOCK_ONFAULT)`, which drops the page
+from the resident set while keeping the `mlockall(MCL_CURRENT|MCL_FUTURE)`
+guarantee: the page is locked again at the moment it faults back in, so it is
+never swappable. This is detected automatically.
+
+Three further options tune the phase:
+
+$(UL
+  $(LI `scavengeMinFree` - leave the heap alone while less than this much of it is
+    resident-but-free, so that small amounts of free memory do not pay for
+    syscalls. Defaults to 16 MB.)
+  $(LI `scavengeBudget` - the most memory one `GC.minimize()` call may release,
+    bounding how long the phase can run; call `GC.minimize()` again to release
+    more. Defaults to 256 MB.)
+  $(LI `scavengeNoHugePages` - apply `MADV_NOHUGEPAGE` to the pools being tracked.
+    A pool backed by transparent huge pages cannot be released a page at a time,
+    so turning this off largely defeats scavenging on a host with THP set to
+    `always`. On by default.)
+)

runtime/druntime/src/core/gc/config.d

@@ -27,6 +27,13 @@ struct Config
     float heapSizeFactor = 2.0; // heap size to used memory ratio
     string cleanup = "collect"; // select gc cleanup method none|collect|finalize
 
+    // Page scavenging (Linux only): hand whole free pool pages back to the OS from minimize(), for the
+    // common case where a pool is mostly free but too sparsely populated to be unmapped whole.
+    bool scavenge = true;                       // enable the scavenge phase of minimize()
+    @MemVal size_t scavengeBudget = 256 << 20;  // most bytes one minimize() may release; caller loops for more
+    @MemVal size_t scavengeMinFree = 16 << 20;  // leave the heap alone below this much resident-free
+    bool scavengeNoHugePages = true;            // MADV_NOHUGEPAGE tracked pools; page-granular release needs it
+
 @nogc nothrow:
 
     bool initialize()
@@ -52,6 +59,8 @@ struct Config
         auto _minPoolSize = minPoolSize.bytes2prettyStruct;
         auto _maxPoolSize = maxPoolSize.bytes2prettyStruct;
         auto _incPoolSize = incPoolSize.bytes2prettyStruct;
+        auto _scavengeBudget = scavengeBudget.bytes2prettyStruct;
+        auto _scavengeMinFree = scavengeMinFree.bytes2prettyStruct;
         printf(" - select gc implementation (default = conservative)
 
     initReserve:N  - initial memory to reserve in MB (%lld%c)
@@ -62,13 +71,20 @@ struct Config
     heapSizeFactor:N - targeted heap size to used memory ratio (%g)
     cleanup:none|collect|finalize - how to treat live objects when terminating (collect)
 
+    scavenge:0|1   - release free pool pages to the OS from minimize() (%d)
+    scavengeBudget:N - most memory one minimize() may release in MB (%lld%c)
+    scavengeMinFree:N - skip scavenging below this much resident-free memory in MB (%lld%c)
+    scavengeNoHugePages:0|1 - MADV_NOHUGEPAGE scavenged pools (%d)
+
     Memory-related values can use B, K, M or G suffixes.
 ".ptr,
                _initReserve.v, _initReserve.u,
                _minPoolSize.v, _minPoolSize.u,
                _maxPoolSize.v, _maxPoolSize.u,
                _incPoolSize.v, _incPoolSize.u,
-               cast(long)parallel, heapSizeFactor);
+               cast(long)parallel, heapSizeFactor,
+               scavenge, _scavengeBudget.v, _scavengeBudget.u,
+               _scavengeMinFree.v, _scavengeMinFree.u, scavengeNoHugePages);
     }
 
     string errorName() @nogc nothrow { return "GC"; }

runtime/druntime/src/core/internal/gc/impl/conservative/gc.d

@@ -37,6 +37,7 @@ import core.internal.gc.bits;
 import core.internal.gc.os;
 import core.gc.config;
 import core.gc.gcinterface;
+import core.internal.gc.scavenger;
 
 import core.internal.container.treap;
 import core.internal.spinlock;
@@ -723,6 +724,7 @@ class ConservativeGC : GC
                     lpool.setFreePageOffsets(pagenum + newsz, freesz - newPages);
                 gcx.usedLargePages += newPages;
                 lpool.freepages -= newPages;
+                scavengerOnPagesCarved(&lpool.base, pagenum + psz, newPages);
                 debug (PRINTF) printFreeInfo(pool);
             }
             else
@@ -813,6 +815,7 @@ class ConservativeGC : GC
                 lpool.setFreePageOffsets(pagenum + psz + sz, freesz - sz);
             lpool.freepages -= sz;
             gcx.usedLargePages += sz;
+            scavengerOnPagesCarved(&lpool.base, pagenum + psz, sz);
             return (psz + sz) * PAGESIZE;
         }
     }
@@ -1907,6 +1910,10 @@ struct Gcx
             cstdlib.free(pool);
         }
 
+        // Whole-pool unmap above can only take a pool that is 100% free. Pages free inside pools that
+        // survived are the general case of the same job, so release those too (bounded, see gcopt).
+        scavengerMinimizePhase(&this);
+
         debug(PRINTF) printf("Done minimizing.\n");
     }
 
@@ -2660,6 +2667,7 @@ struct Gcx
                         debug(COLLECT_PRINTF) printf("\tcollecting big %p\n", p);
                         leakDetector.log_free(q, sentinel_size(q, npages * PAGESIZE - SENTINEL_EXTRA));
                         pool.pagetable[pn..pn+npages] = Bins.B_FREE;
+                        scavengerOnPagesFreed(pool, pn, npages);
                         if (pn < pool.searchStart) pool.searchStart = pn;
                         freedLargePages += npages;
                         pool.freepages += npages;
@@ -2790,6 +2798,7 @@ struct Gcx
                             pool.freeAllPageBits(pn);
 
                             pool.pagetable[pn] = Bins.B_FREE;
+                            scavengerOnPagesFreed(pool, pn, 1);
                             // add to free chain
                             pool.binPageChain[pn] = cast(uint) pool.searchStart;
                             pool.searchStart = pn;
@@ -3536,6 +3545,7 @@ struct Pool
     size_t npages;
     size_t freepages;     // The number of pages not in use.
     Bins* pagetable;
+    ubyte* scavengedMap; // one byte/page: scavenger dirty/clean tracking, see core.internal.gc.scavenger
 
     bool isLargeObject;
 
@@ -3665,11 +3675,15 @@ struct Pool
         this.freepages = npages;
         this.searchStart = 0;
         this.largestFree = npages;
+
+        scavengerOnPoolCreate(&this);
     }
 
 
     void Dtor() nothrow
     {
+        scavengerOnPoolDestroy(&this);
+
         if (baseAddr)
         {
             int result;
@@ -4140,6 +4154,7 @@ struct LargeObjectPool
                         bPageOffsets[i + offset] = cast(uint) offset;
                 }
                 freepages -= n;
+                scavengerOnPagesCarved(&this.base, i, n);
                 return i;
             }
             if (p > largest)
@@ -4174,6 +4189,7 @@ struct LargeObjectPool
         }
         freepages += npages;
         largestFree = freepages; // invalidate
+        scavengerOnPagesFreed(&this.base, pagenum, npages);
     }
 
     /**
@@ -4436,6 +4452,7 @@ struct SmallObjectPool
         binPageChain[pn] = Pool.PageRecovered;
         pagetable[pn] = bin;
         freepages--;
+        scavengerOnPagesCarved(&this.base, pn, 1);
 
         // Convert page to free list
         size_t size = binsize[bin];
@@ -5160,3 +5177,82 @@ void undefinedWrite(T)(ref T var, T value) nothrow
     else
         var = value;
 }
+
+// ============================================================================
+// Page scavenger -- extern(C) entry points for an application that wants to
+// drive scavenging itself rather than leave it to minimize(). All the
+// bookkeeping lives in core.internal.gc.scavenger; these wrappers only
+// resolve the live Gcx instance and take the GC lock, following the shape of
+// ConservativeGC.minimize() above (lockNR / scope(failure) unlock / unlock).
+//
+// Posix-gated because resolving the live Gcx goes through Gcx.instance, which the GC only maintains there
+// (it exists for the fork-safety machinery). The mechanism is Linux-only in any case, so on the remaining
+// Posix targets these report a scavenger that never armed.
+// ============================================================================
+
+version (Posix):
+
+extern (C) size_t gc_scavenger_pass(size_t maxBytes) nothrow
+{
+    auto gcx = Gcx.instance;
+    if (gcx is null)
+    {
+        return 0; // GC not initialized yet, or not the conservative GC
+    }
+
+    ConservativeGC.lockNR();
+    scope (failure) ConservativeGC.gcLock.unlock();
+    auto scavenged = scavengerPass(gcx, maxBytes);
+    ConservativeGC.gcLock.unlock();
+    return scavenged;
+}
+
+extern (C) size_t gc_scavenger_dirty_free_bytes() nothrow @nogc
+{
+    return scavengerDirtyFreeBytes();
+}
+
+extern (C) int gc_scavenger_arm(int heapIsMlocked) nothrow
+{
+    auto gcx = Gcx.instance;
+    if (gcx is null)
+    {
+        return ScavengeStatus.NOT_ARMED; // GC not initialized yet, or not the conservative GC
+    }
+
+    ConservativeGC.lockNR();
+    scope (failure) ConservativeGC.gcLock.unlock();
+    auto status = scavengerArm(gcx, heapIsMlocked != 0);
+    ConservativeGC.gcLock.unlock();
+    return status;
+}
+
+extern (C) int gc_scavenger_status() nothrow @nogc
+{
+    return scavengerStatus();
+}
+
+// Why the minimize() scavenge phase did or did not act; see core.internal.gc.scavenger.ScavengeStats.
+// Returned whole so the caller gets a coherent snapshot, and so adding a counter cannot silently
+// renumber the ones a caller already reads.
+extern (C) ScavengeStats gc_scavenger_stats() nothrow @nogc
+{
+    return scavengerStats();
+}
+
+// Set when the phase stopped on its budget with reclaimable pages left; the application drives the
+// follow-up pass, and gc_scavenger_min_free() is the gate the phase itself uses.
+extern (C) int gc_scavenger_continuation_due() nothrow @nogc
+{
+    return scavengerMinimizeContinuationDue();
+}
+
+extern (C) size_t gc_scavenger_min_free() nothrow @nogc
+{
+    return scavengerMinFree();
+}
+
+extern (C) void gc_scavenger_inject_fail(int mode) nothrow @nogc
+{
+    scavengerInjectFail(mode);
+}