benchmarks: durdir benchmark that evaluates the efficacy and performance of DurDir

benchmarking/README.md

@@ -62,6 +62,53 @@ You can also configure things like the number of users, how quickly those users
 are spawned, the frequency with which requests are made and whether or not tracing is
 enabled.
 
+User classes implemented in boomer rather than Python are selected at deploy
+time — the stack runs one per deployment:
+
+```bash
+./benchmarking/locust/deploy.sh --deploy --user-class durdir
+```
+
+### Headless (automation only)
+
+`runner.py` runs a test without the web UI, writing CSVs, logs and traces to
+`--dest`. The nightly automation submits it as a Job on the test cluster; it is
+not a local entry point. See [automation/README.md](automation/README.md).
+
+```bash
+python3 runner.py -f tests/<user-class>.py -t 1m -u 1 --name <run-name> --dest /tmp/bench
+```
+
+Test-specific flags are appended to the same command; see the sections below.
+
+### DurDir Benchmark
+
+The DurDir benchmark evaluates actor suspend/resume performance, disk persistence overhead,
+and state restoration latency when a durable directory is attached to the actor.
+
+#### DurDir Configuration Knobs
+
+* `--durdir-file-size-bytes`: Size in bytes of the data file (default `8388608` = 8 MiB).
+* `--resume-mode`: Resume trigger mode:
+  * `explicit` (default): Client invokes the `ResumeActor` RPC before sending traffic.
+  * `implicit`: Client sends traffic through the router without an explicit wake RPC, testing traffic-triggered resume.
+* `--durdir-read-mode`: Verification read mode:
+  * `data` (default): Server returns full payload bytes for client-side SHA-256 verification.
+  * `digest`: Server hashes the file and returns size and digest, reducing network transfer.
+* `--durdir-template`: ActorTemplate name:
+  * `glutton-durdir-data` (default): Attaches a durable data directory without memory snapshot restore.
+  * `glutton-durdir-full`: Attaches a durable data directory and performs a full memory snapshot restore.
+
+#### DurDir Reported Metrics
+
+* `DurDirWrite`: Initial truncate-write creating the data file.
+* `DurDirServeInitial`: First read immediately following file creation.
+* `SuspendActor`: Actor suspend latency (snapshot creation + persistence upload).
+* `ResumeActor`: Actor resume latency.
+* `DurDirServeAfterResume`: First read after resume (measures page faults / lazy load overhead on restored volume).
+* `DurDirServeWarm`: Subsequent read within the same active cycle (cached state baseline).
+* `DurDirOverwrite`: In-place file overwrite with checksum verification.
+
 ### Viewing Traces
 You must have enabled otel tracing for your cluster to view traces.
 

benchmarking/automation/tests.yaml

@@ -83,3 +83,117 @@ tests:
       - "0.5"
       - "--max-wait-time"
       - "1.0"
+  - name: durdir_data_baseline
+    description: "DurDir data baseline: 1 concurrent user, 8 MiB file, explicit resume"
+    targetCluster: dev
+    file: /app/tests/durdir.py
+    duration: 1m
+    users: 1
+    workerCount: 1
+    flags:
+      - "--durdir-template"
+      - "glutton-durdir-data"
+      - "--resume-mode"
+      - "explicit"
+      - "--durdir-file-size-bytes"
+      - "8388608"
+      - "--min-wait-time"
+      - "1.0"
+      - "--max-wait-time"
+      - "1.0"
+  - name: durdir_full_baseline
+    description: "DurDir full baseline: 1 concurrent user, 8 MiB file, full memory restore comparison"
+    targetCluster: dev
+    file: /app/tests/durdir.py
+    duration: 1m
+    users: 1
+    workerCount: 1
+    flags:
+      - "--durdir-template"
+      - "glutton-durdir-full"
+      - "--resume-mode"
+      - "explicit"
+      - "--durdir-file-size-bytes"
+      - "8388608"
+      - "--min-wait-time"
+      - "1.0"
+      - "--max-wait-time"
+      - "1.0"
+  - name: durdir_implicit_resume
+    description: "DurDir implicit resume: 1 concurrent user, 8 MiB file, traffic-triggered resume"
+    targetCluster: dev
+    file: /app/tests/durdir.py
+    duration: 1m
+    users: 1
+    workerCount: 1
+    flags:
+      - "--durdir-template"
+      - "glutton-durdir-data"
+      - "--resume-mode"
+      - "implicit"
+      - "--durdir-file-size-bytes"
+      - "8388608"
+      - "--min-wait-time"
+      - "1.0"
+      - "--max-wait-time"
+      - "1.0"
+  - name: durdir_size_5mb
+    description: "DurDir file size sweep (5 MiB): 1 concurrent user, digest read mode"
+    targetCluster: dev
+    file: /app/tests/durdir.py
+    duration: 1m
+    users: 1
+    workerCount: 1
+    flags:
+      - "--durdir-template"
+      - "glutton-durdir-data"
+      - "--resume-mode"
+      - "explicit"
+      - "--durdir-read-mode"
+      - "digest"
+      - "--durdir-file-size-bytes"
+      - "5242880"
+      - "--min-wait-time"
+      - "1.0"
+      - "--max-wait-time"
+      - "1.0"
+  - name: durdir_size_10mb
+    description: "DurDir file size sweep (10 MiB): 1 concurrent user, digest read mode"
+    targetCluster: dev
+    file: /app/tests/durdir.py
+    duration: 1m
+    users: 1
+    workerCount: 1
+    flags:
+      - "--durdir-template"
+      - "glutton-durdir-data"
+      - "--resume-mode"
+      - "explicit"
+      - "--durdir-read-mode"
+      - "digest"
+      - "--durdir-file-size-bytes"
+      - "10485760"
+      - "--min-wait-time"
+      - "1.0"
+      - "--max-wait-time"
+      - "1.0"
+  - name: durdir_size_64mb
+    description: "DurDir file size sweep (64 MiB): 1 concurrent user, digest read mode"
+    targetCluster: dev
+    file: /app/tests/durdir.py
+    duration: 1m
+    users: 1
+    workerCount: 1
+    flags:
+      - "--durdir-template"
+      - "glutton-durdir-data"
+      - "--resume-mode"
+      - "explicit"
+      - "--durdir-read-mode"
+      - "digest"
+      - "--durdir-file-size-bytes"
+      - "67108864"
+      - "--min-wait-time"
+      - "1.0"
+      - "--max-wait-time"
+      - "1.0"

benchmarking/locust/common/boomer_config.py

@@ -17,6 +17,8 @@
 Flag registration lives in the modules that own each flag:
   * --trace-probability             → common.trace.init_tracing
   * --min-wait-time / --max-wait-time → common.wait_time.init_wait_time
+  * --resume-mode                   → common.resume_mode.add_resume_mode_arguments
+  * --durdir-*                      → common.durdir_config.add_durdir_arguments
 
 This module ties them together so boomer-Go workers can pick up the values
 the operator set in the web UI form:
@@ -34,17 +36,19 @@
 import logging
 from collections.abc import Iterable
 
-from locust import events
-from locust.env import Environment
-
-from common.trace import init_tracing
-from common.wait_time import init_wait_time
-
 logger = logging.getLogger(__name__)
 
-# Boomer-tunable flags. CLI form ("--foo-bar") is converted to the
-# attribute / JSON-key form ("foo_bar") by _attr().
-_FLAGS = ("--trace-probability", "--min-wait-time", "--max-wait-time")
+# Boomer-tunable flags and their types. CLI form ("--foo-bar") is converted
+# to the attribute / JSON-key form ("foo_bar") by _attr().
+_FLAGS = {
+    "--trace-probability": float,
+    "--min-wait-time": float,
+    "--max-wait-time": float,
+    "--durdir-file-size-bytes": int,
+    "--resume-mode": str,
+    "--durdir-read-mode": str,
+    "--durdir-template": str,
+}
 
 
 def _attr(flag: str) -> str:
@@ -56,8 +60,8 @@ def build_config_json(argv: Iterable[str]) -> str:
     --config-json flag. Unknown args are ignored; unset flags are omitted so
     boomer falls back to its own defaults."""
     p = argparse.ArgumentParser(add_help=False)
-    for flag in _FLAGS:
-        p.add_argument(flag, type=float)
+    for flag, type_func in _FLAGS.items():
+        p.add_argument(flag, type=type_func)
     parsed, _ = p.parse_known_args(argv)
     cfg = {
         _attr(f): getattr(parsed, _attr(f))
@@ -71,6 +75,14 @@ def init_boomer_config() -> None:
     """Ensure the owning modules have registered the boomer-tunable flags,
     then expose their current values at /boomer-config so boomer-Go workers
     can fetch them at runtime."""
+    from locust import events
+    from locust.env import Environment
+
+    from common.durdir_config import add_durdir_arguments
+    from common.resume_mode import add_resume_mode_arguments
+    from common.trace import init_tracing
+    from common.wait_time import init_wait_time
+
     init_tracing()
     init_wait_time()
 
@@ -82,7 +94,7 @@ def on_init(environment: Environment, **kwargs) -> None:
             return
 
         @environment.web_ui.app.route("/boomer-config")
-        def boomer_config() -> dict[str, float | None]:
+        def boomer_config() -> dict[str, float | int | str | None]:
             opts = environment.parsed_options
             return {_attr(f): getattr(opts, _attr(f), None) for f in _FLAGS}
 

benchmarking/locust/common/durdir_config.py

@@ -0,0 +1,43 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""DurDir benchmark runtime flags."""
+
+from locust import events
+from locust.argument_parser import LocustArgumentParser
+
+
+@events.init_command_line_parser.add_listener
+def add_durdir_arguments(parser: LocustArgumentParser) -> None:
+    group = parser.add_argument_group("DurDir Benchmark")
+    group.add_argument(
+        "--durdir-file-size-bytes",
+        type=int,
+        default=8388608,
+        help="Size of the test file written and read during the DurDir benchmark (default: 8388608 = 8 MiB)",
+    )
+    group.add_argument(
+        "--durdir-read-mode",
+        type=str,
+        default="data",
+        choices=["data", "digest"],
+        help="Read mode for DurDir serves: 'data' (default) returns and client-verifies the full payload; "
+        "'digest' returns only size+sha256 for reduced network overhead",
+    )
+    group.add_argument(
+        "--durdir-template",
+        type=str,
+        default="glutton-durdir-data",
+        help="ActorTemplate name to benchmark (default: glutton-durdir-data)",
+    )

benchmarking/locust/common/glutton_pb2.py

@@ -38,38 +38,44 @@
 
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\rglutton.proto\x12\x07glutton\"T\n\x0fWriteRAMRequest\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x0c\n\x04size\x18\x02 \x01(\x05\x12&\n\nwrite_mode\x18\x03 \x01(\x0e\x32\x12.glutton.WriteMode\"\x12\n\x10WriteRAMResponse\"U\n\x10WriteDiskRequest\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x0c\n\x04size\x18\x02 \x01(\x05\x12&\n\nwrite_mode\x18\x03 \x01(\x0e\x32\x12.glutton.WriteMode\"\x13\n\x11WriteDiskResponse\"\x1e\n\rOpenFDRequest\x12\r\n\x05\x63ount\x18\x01 \x01(\x05\"\x10\n\x0eOpenFDResponse\"\x1e\n\x0bPingRequest\x12\x0f\n\x07message\x18\x01 \x01(\t\"\x1f\n\x0cPingResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\"-\n\rGossipRequest\x12\x1c\n\x05peers\x18\x01 \x03(\x0b\x32\r.glutton.Peer\"\x10\n\x0eGossipResponse\"&\n\x04Peer\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x10\n\x08\x64\x65lay_ms\x18\x02 \x01(\x05*>\n\tWriteMode\x12\x17\n\x13WRITE_MODE_TRUNCATE\x10\x00\x12\x18\n\x14WRITE_MODE_OVERWRITE\x10\x01\x32\xc3\x02\n\x07Glutton\x12\x41\n\x08WriteRAM\x12\x18.glutton.WriteRAMRequest\x1a\x19.glutton.WriteRAMResponse\"\x00\x12\x44\n\tWriteDisk\x12\x19.glutton.WriteDiskRequest\x1a\x1a.glutton.WriteDiskResponse\"\x00\x12;\n\x06OpenFD\x12\x16.glutton.OpenFDRequest\x1a\x17.glutton.OpenFDResponse\"\x00\x12\x35\n\x04Ping\x12\x14.glutton.PingRequest\x1a\x15.glutton.PingResponse\"\x00\x12;\n\x06Gossip\x12\x16.glutton.GossipRequest\x1a\x17.glutton.GossipResponse\"\x00\x42=Z;github.com/agent-substrate/substrate/internal/proto/gluttonb\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\rglutton.proto\x12\x07glutton\"T\n\x0fWriteRAMRequest\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x0c\n\x04size\x18\x02 \x01(\x05\x12&\n\nwrite_mode\x18\x03 \x01(\x0e\x32\x12.glutton.WriteMode\"\x12\n\x10WriteRAMResponse\"U\n\x10WriteDiskRequest\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x0c\n\x04size\x18\x02 \x01(\x05\x12&\n\nwrite_mode\x18\x03 \x01(\x0e\x32\x12.glutton.WriteMode\"1\n\x11WriteDiskResponse\x12\x0c\n\x04size\x18\x01 \x01(\x03\x12\x0e\n\x06sha256\x18\x02 \x01(\x0c\"D\n\x0fReadDiskRequest\x12\x0b\n\x03key\x18\x01 \x01(\t\x12$\n\tread_mode\x18\x02 \x01(\x0e\x32\x11.glutton.ReadMode\">\n\x10ReadDiskResponse\x12\x0c\n\x04size\x18\x01 \x01(\x03\x12\x0e\n\x06sha256\x18\x02 \x01(\x0c\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\"\x1e\n\rOpenFDRequest\x12\r\n\x05\x63ount\x18\x01 \x01(\x05\"\x10\n\x0eOpenFDResponse\"\x1e\n\x0bPingRequest\x12\x0f\n\x07message\x18\x01 \x01(\t\"\x1f\n\x0cPingResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\"-\n\rGossipRequest\x12\x1c\n\x05peers\x18\x01 \x03(\x0b\x32\r.glutton.Peer\"\x10\n\x0eGossipResponse\"&\n\x04Peer\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x10\n\x08\x64\x65lay_ms\x18\x02 \x01(\x05*>\n\tWriteMode\x12\x17\n\x13WRITE_MODE_TRUNCATE\x10\x00\x12\x18\n\x14WRITE_MODE_OVERWRITE\x10\x01*9\n\x08ReadMode\x12\x12\n\x0eREAD_MODE_DATA\x10\x00\x12\x19\n\x15READ_MODE_DIGEST_ONLY\x10\x01\x32\x86\x03\n\x07Glutton\x12\x41\n\x08WriteRAM\x12\x18.glutton.WriteRAMRequest\x1a\x19.glutton.WriteRAMResponse\"\x00\x12\x44\n\tWriteDisk\x12\x19.glutton.WriteDiskRequest\x1a\x1a.glutton.WriteDiskResponse\"\x00\x12\x41\n\x08ReadDisk\x12\x18.glutton.ReadDiskRequest\x1a\x19.glutton.ReadDiskResponse\"\x00\x12;\n\x06OpenFD\x12\x16.glutton.OpenFDRequest\x1a\x17.glutton.OpenFDResponse\"\x00\x12\x35\n\x04Ping\x12\x14.glutton.PingRequest\x1a\x15.glutton.PingResponse\"\x00\x12;\n\x06Gossip\x12\x16.glutton.GossipRequest\x1a\x17.glutton.GossipResponse\"\x00\x42=Z;github.com/agent-substrate/substrate/internal/proto/gluttonb\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
 _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'glutton_pb2', _globals)
 if not _descriptor._USE_C_DESCRIPTORS:
   _globals['DESCRIPTOR']._loaded_options = None
   _globals['DESCRIPTOR']._serialized_options = b'Z;github.com/agent-substrate/substrate/internal/proto/glutton'
-  _globals['_WRITEMODE']._serialized_start=460
-  _globals['_WRITEMODE']._serialized_end=522
+  _globals['_WRITEMODE']._serialized_start=624
+  _globals['_WRITEMODE']._serialized_end=686
+  _globals['_READMODE']._serialized_start=688
+  _globals['_READMODE']._serialized_end=745
   _globals['_WRITERAMREQUEST']._serialized_start=26
   _globals['_WRITERAMREQUEST']._serialized_end=110
   _globals['_WRITERAMRESPONSE']._serialized_start=112
   _globals['_WRITERAMRESPONSE']._serialized_end=130
   _globals['_WRITEDISKREQUEST']._serialized_start=132
   _globals['_WRITEDISKREQUEST']._serialized_end=217
   _globals['_WRITEDISKRESPONSE']._serialized_start=219
-  _globals['_WRITEDISKRESPONSE']._serialized_end=238
-  _globals['_OPENFDREQUEST']._serialized_start=240
-  _globals['_OPENFDREQUEST']._serialized_end=270
-  _globals['_OPENFDRESPONSE']._serialized_start=272
-  _globals['_OPENFDRESPONSE']._serialized_end=288
-  _globals['_PINGREQUEST']._serialized_start=290
-  _globals['_PINGREQUEST']._serialized_end=320
-  _globals['_PINGRESPONSE']._serialized_start=322
-  _globals['_PINGRESPONSE']._serialized_end=353
-  _globals['_GOSSIPREQUEST']._serialized_start=355
-  _globals['_GOSSIPREQUEST']._serialized_end=400
-  _globals['_GOSSIPRESPONSE']._serialized_start=402
-  _globals['_GOSSIPRESPONSE']._serialized_end=418
-  _globals['_PEER']._serialized_start=420
-  _globals['_PEER']._serialized_end=458
-  _globals['_GLUTTON']._serialized_start=525
-  _globals['_GLUTTON']._serialized_end=848
+  _globals['_WRITEDISKRESPONSE']._serialized_end=268
+  _globals['_READDISKREQUEST']._serialized_start=270
+  _globals['_READDISKREQUEST']._serialized_end=338
+  _globals['_READDISKRESPONSE']._serialized_start=340
+  _globals['_READDISKRESPONSE']._serialized_end=402
+  _globals['_OPENFDREQUEST']._serialized_start=404
+  _globals['_OPENFDREQUEST']._serialized_end=434
+  _globals['_OPENFDRESPONSE']._serialized_start=436
+  _globals['_OPENFDRESPONSE']._serialized_end=452
+  _globals['_PINGREQUEST']._serialized_start=454
+  _globals['_PINGREQUEST']._serialized_end=484
+  _globals['_PINGRESPONSE']._serialized_start=486
+  _globals['_PINGRESPONSE']._serialized_end=517
+  _globals['_GOSSIPREQUEST']._serialized_start=519
+  _globals['_GOSSIPREQUEST']._serialized_end=564
+  _globals['_GOSSIPRESPONSE']._serialized_start=566
+  _globals['_GOSSIPRESPONSE']._serialized_end=582
+  _globals['_PEER']._serialized_start=584
+  _globals['_PEER']._serialized_end=622
+  _globals['_GLUTTON']._serialized_start=748
+  _globals['_GLUTTON']._serialized_end=1138
 # @@protoc_insertion_point(module_scope)