Adjust some behavior/docs around table/memory growth

crates/wasmtime/src/runtime/limits.rs

@@ -73,13 +73,20 @@ pub trait ResourceLimiter: Send {
         maximum: Option<usize>,
     ) -> Result<bool>;
 
-    /// Notifies the resource limiter that growing a linear memory, permitted by
-    /// the `memory_growing` method, has failed.
+    /// Notifies the resource limiter that growing a linear memory has failed.
+    ///
+    /// This is typically called after `memory_growing` has permitted a growth
+    /// which then failed. It may also be called without a preceding call to
+    /// `memory_growing` when the requested size is invalid for the linear
+    /// memory, for example when it cannot be represented by the memory's type
+    /// (such as a 32-bit memory with a page size of 1 byte growing to 4GiB or
+    /// beyond).
     ///
     /// Note that this method is not called if `memory_growing` returns an
-    /// error.
+    /// error, nor if it returns `Ok(false)`.
     ///
-    /// Reasons for failure include: the growth exceeds the `maximum` passed to
+    /// Reasons for failure include: the requested size is not valid for the
+    /// memory's type, the growth exceeds the `maximum` passed to
     /// `memory_growing`, or the operating system failed to allocate additional
     /// memory. In that case, `error` might be downcastable to a `std::io::Error`.
     ///
@@ -110,13 +117,19 @@ pub trait ResourceLimiter: Send {
         maximum: Option<usize>,
     ) -> Result<bool>;
 
-    /// Notifies the resource limiter that growing a linear memory, permitted by
-    /// the `table_growing` method, has failed.
+    /// Notifies the resource limiter that growing a table has failed.
+    ///
+    /// This is typically called after `table_growing` has permitted a growth
+    /// which then failed. It may also be called without a preceding call to
+    /// `table_growing` when the requested size cannot be represented, for
+    /// example when computing the new size overflows.
     ///
-    /// Note that this method is not called if `table_growing` returns an error.
+    /// Note that this method is not called if `table_growing` returns an
+    /// error, nor if it returns `Ok(false)`.
     ///
-    /// Reasons for failure include: the growth exceeds the `maximum` passed to
-    /// `table_growing`. This could expand in the future.
+    /// Reasons for failure include: the requested size overflows, the growth
+    /// exceeds the `maximum` passed to `table_growing`, or allocating the
+    /// table's additional storage failed. This could expand in the future.
     ///
     /// See the details on the return values for `memory_growing` for what the
     /// return value of this function indicates.

crates/wasmtime/src/runtime/vm/table.rs

@@ -670,24 +670,27 @@ impl Table {
         }
 
         // First resize the storage and then fill with the init value
-        match self {
+        let result = match self {
             Table::Static(StaticTable::Func(StaticFuncTable { data, size, .. })) => {
                 unsafe {
                     debug_assert!(data.as_ref()[*size..new_size].iter().all(|x| x.is_none()));
                 }
                 *size = new_size;
+                Ok(())
             }
             Table::Static(StaticTable::GcRef(StaticGcRefTable { data, size })) => {
                 unsafe {
                     debug_assert!(data.as_ref()[*size..new_size].iter().all(|x| x.is_none()));
                 }
                 *size = new_size;
+                Ok(())
             }
             Table::Static(StaticTable::Cont(StaticContTable { data, size })) => {
                 unsafe {
                     debug_assert!(data.as_ref()[*size..new_size].iter().all(|x| x.is_none()));
                 }
                 *size = new_size;
+                Ok(())
             }
 
             // These calls to `resize` could move the base address of
@@ -698,14 +701,23 @@ impl Table {
             // that delta is non-zero and the new size doesn't exceed the
             // maximum mean we can't get here.
             Table::Dynamic(DynamicTable::Func(DynamicFuncTable { elements, .. })) => {
-                elements.resize_with(new_size, || None)?;
+                elements.resize_with(new_size, || None)
             }
             Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => {
-                elements.resize_with(new_size, || None)?;
+                elements.resize_with(new_size, || None)
             }
             Table::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => {
-                elements.resize_with(new_size, || None)?;
+                elements.resize_with(new_size, || None)
+            }
+        };
+
+        // Failure to grow the table is reported as a -1 value to wasm as
+        // opposed to a trap (e.g. just using `?` on this).
+        if let Err(e) = result {
+            if let Some(limiter) = limiter {
+                limiter.table_grow_failed(e.into())?;
             }
+            return Ok(None);
         }
 
         Ok(Some(old_size))

tests/misc_testsuite/memory64/table-too-big.wast

@@ -13,5 +13,8 @@
   )
 )
 
-(assert_trap (invoke "grow" (i64.const 0x2000_0000_0000_0000))
-  "failed to allocate")
+;; Failing to allocate the table's storage is a failed `table.grow`, not a
+;; trap.
+(assert_return (invoke "grow" (i64.const 0x2000_0000_0000_0000))
+  (i64.const -1))
+(assert_return (invoke "grow" (i64.const 0x1000_0000_0000)) (i64.const -1))

tests/wast.rs

@@ -97,6 +97,12 @@ fn run_wast(test: &WastTest, config: WastConfig) -> wasmtime::Result<()> {
     let test_hogs_memory = test_config.hogs_memory();
     let relaxed_simd = test_config.relaxed_simd();
 
+    // Skip memory-intensive tests on ASAN. Some of these require gracefully
+    // handling OOM but ASAN hard-aborts on OOM.
+    if test_hogs_memory && cfg!(asan) {
+        return Ok(());
+    }
+
     let is_cranelift = match config.compiler {
         Compiler::CraneliftNative | Compiler::CraneliftPulley => true,
         _ => false,