benchmark: keep service ports out of the ephemeral pool, and stop the cache failing silently

frameworks/aspnet-minimal_nginx/compose.production-stack.yml

@@ -49,6 +49,17 @@ services:
       - ${DATA_DIR}/redis-seed.txt:/seed.txt:ro
       - ${DATA_DIR}/redis-entrypoint.sh:/entrypoint.sh:ro
     entrypoint: ["sh", "/entrypoint.sh"]
+    # A cache that cannot bind is a silent downgrade, not a failure: with the
+    # short `depends_on: - cache` form the server starts anyway and quietly
+    # talks to whatever else holds 6379, so the run publishes numbers measured
+    # against a Redis this profile never configured. Fail the stack instead.
+    healthcheck:
+      test: ["CMD", "redis-cli", "-p", "6379", "ping"]
+      interval: 30s
+      timeout: 2s
+      retries: 3
+      start_period: 30s
+      start_interval: 1s
 
   authsvc:
     build: ../_shared/authsvc
@@ -85,4 +96,5 @@ services:
     volumes:
       - ${DATA_DIR}/dataset.json:/data/dataset.json:ro
     depends_on:
-      - cache
+      cache:
+        condition: service_healthy

frameworks/fulmine/compose.production-stack.yml

@@ -51,6 +51,17 @@ services:
       - ${DATA_DIR}/redis-seed.txt:/seed.txt:ro
       - ${DATA_DIR}/redis-entrypoint.sh:/entrypoint.sh:ro
     entrypoint: ["sh", "/entrypoint.sh"]
+    # A cache that cannot bind is a silent downgrade, not a failure: with the
+    # short `depends_on: - cache` form the server starts anyway and quietly
+    # talks to whatever else holds 6379, so the run publishes numbers measured
+    # against a Redis this profile never configured. Fail the stack instead.
+    healthcheck:
+      test: ["CMD", "redis-cli", "-p", "6379", "ping"]
+      interval: 30s
+      timeout: 2s
+      retries: 3
+      start_period: 30s
+      start_interval: 1s
 
   authsvc:
     build: ../_shared/authsvc
@@ -98,4 +109,5 @@ services:
     volumes:
       - ${DATA_DIR}/dataset.json:/data/dataset.json:ro
     depends_on:
-      - cache
+      cache:
+        condition: service_healthy

frameworks/sark-production/compose.production-stack.yml

@@ -24,6 +24,17 @@ services:
     cpuset: "${GATEWAY_CACHE_CPUS:-16-19,80-83}"
     security_opt:
       - seccomp:unconfined
+    # A cache that cannot bind is a silent downgrade, not a failure: with the
+    # short `depends_on: - cache` form the server starts anyway and quietly
+    # talks to whatever else holds 6379, so the run publishes numbers measured
+    # against a Redis this profile never configured. Fail the stack instead.
+    healthcheck:
+      test: ["CMD", "redis-cli", "-p", "6379", "ping"]
+      interval: 30s
+      timeout: 2s
+      retries: 3
+      start_period: 30s
+      start_interval: 1s
 
   authsvc:
     build: ../_shared/authsvc
@@ -56,4 +67,5 @@ services:
       - SARK_HTTPARENA_PER_IP_CAP=0
       - SARK_HTTPARENA_H1_ONLY=1
     depends_on:
-      - cache
+      cache:
+        condition: service_healthy

scripts/lib/gateway.sh

@@ -21,6 +21,7 @@ GATEWAY_ACTIVE_PROFILE=""
 GATEWAY_ACTIVE_FRAMEWORK=""
 GATEWAY_CONTAINERS=""
 GATEWAY_CONTAINER_COUNT=0
+GATEWAY_STOPPED_REDIS=false
 
 _gateway_env() {
     # All compose invocations need the same env vars for interpolation.
@@ -96,6 +97,28 @@ _gateway_dump_logs() {
              | awk -F'|' -v p="$project" '$2 == p { print $1 " " $3 }')
 }
 
+# The harness Redis sidecar and a stack's own `cache` both want the host's
+# 6379, and redis_start runs once for the whole run whenever the entry
+# subscribes to crud — so for any entry subscribed to both crud and
+# production-stack the two collide on every run. It was invisible because the
+# server depended on `cache` with the short form, which only waits for the
+# container to start: the cache died, the server carried on against the
+# harness Redis, and the profile published numbers measured against a cache it
+# never configured. The compose files now wait for a healthy cache, which turns
+# that into a failure; this gives the port up so it can succeed instead.
+_gateway_yield_redis() {
+    local compose_file="$1" project="$2"
+    GATEWAY_STOPPED_REDIS=false
+    command -v redis_stop >/dev/null 2>&1 || return 0
+    [ -n "${REDIS_CONTAINER:-}" ] || return 0
+    docker ps --format '{{.Names}}' 2>/dev/null | grep -qx "$REDIS_CONTAINER" || return 0
+    _gateway_env docker compose -f "$compose_file" -p "$project" config --services 2>/dev/null \
+        | grep -qx "cache" || return 0
+    info "stopping the harness redis sidecar: this stack ships its own cache on the same port"
+    redis_stop
+    GATEWAY_STOPPED_REDIS=true
+}
+
 gateway_up() {
     local framework="$1"
     local profile="${2:-gateway-64}"
@@ -110,6 +133,7 @@ gateway_up() {
     _gateway_env docker compose -f "$compose_file" -p "$GATEWAY_PROJECT" \
         down --remove-orphans 2>/dev/null || true
     _gateway_clear_stale "$GATEWAY_PROJECT"
+    _gateway_yield_redis "$compose_file" "$GATEWAY_PROJECT"
 
     info "starting gateway compose stack: $framework ($profile)"
     # --build forces compose to rebuild from source if any file in the
@@ -137,6 +161,16 @@ gateway_down() {
     # Tear down whatever gateway stack is currently active. Callers can
     # pass (framework, profile) explicitly, but the normal cleanup path
     # (EXIT trap, post-run teardown) relies on the state gateway_up stored.
+    # Before the early returns: if this stack took the harness Redis's port,
+    # give it back even when there is no active stack left to tear down. The
+    # subshell keeps redis_start's `fail` from exiting the caller, since this
+    # also runs from the cleanup trap.
+    if [ "$GATEWAY_STOPPED_REDIS" = true ]; then
+        GATEWAY_STOPPED_REDIS=false
+        info "restarting the harness redis sidecar"
+        ( redis_start ) || warn "redis sidecar did not come back up"
+    fi
+
     local framework="${1:-$GATEWAY_ACTIVE_FRAMEWORK}"
     local profile="${2:-$GATEWAY_ACTIVE_PROFILE}"
     [ -n "$framework" ] || return 0

scripts/lib/system.sh

@@ -43,6 +43,23 @@ system_tune() {
     sudo sysctl -w net.ipv4.tcp_max_tw_buckets=131072        >/dev/null 2>&1 || true
     sudo sysctl -w net.ipv4.tcp_tw_reuse=1                   >/dev/null 2>&1 || true
 
+    # Widening the range above put every port a server binds inside the pool the
+    # kernel draws outbound connections from, so a load generator can be sitting
+    # on one when a later profile tries to listen on it. That is what killed
+    # production-stack at the end of a full run: authsvc panicked with
+    # "bind 0.0.0.0:9090: Address in use", and because the edge waits on it the
+    # whole stack aborted. Running the same profile first, with no load behind
+    # it, passed. tcp_tw_reuse does not help - it governs outbound reuse, not a
+    # listening bind.
+    #
+    # ip_local_reserved_ports takes these out of ephemeral allocation while
+    # leaving them bindable, which is exactly the case it exists for. Every
+    # fixed port the harness or a stack listens on belongs here: 8080/8081/8082/
+    # 8443 from common.sh, 9090 for the production-stack auth sidecar, 6379 and
+    # 5432 for the Redis and Postgres sidecars.
+    sudo sysctl -w net.ipv4.ip_local_reserved_ports='5432,6379,8080,8081,8082,8443,9090' \
+        >/dev/null 2>&1 || warn "ip_local_reserved_ports"
+
     info "setting UDP buffer sizes for QUIC"
     sudo sysctl -w net.core.rmem_max=7500000 >/dev/null 2>&1 || true
     sudo sysctl -w net.core.wmem_max=7500000 >/dev/null 2>&1 || true

site/content/docs/hardware/_index.md

@@ -135,7 +135,8 @@ This split is empirically tuned - see the CHANGELOG entry for 2026-04-16 for the
 - `net.ipv4.tcp_max_syn_backlog` → 65535
 - `net.core.netdev_max_backlog` → 65535
 - `net.ipv4.ip_local_port_range` → `1024 65535` (avoid ephemeral port exhaustion under `-r` reconnect storms)
-- `net.ipv4.tcp_tw_reuse` → 1
+- `net.ipv4.ip_local_reserved_ports` → `5432,6379,8080,8081,8082,8443,9090` — every port a server or sidecar listens on. The range above is wide enough to include them, so without this the kernel can hand one to an outbound load-generator socket and the next profile to bind it fails. That is not theoretical: `production-stack` died at the end of a full run with `bind 0.0.0.0:9090: Address in use` while passing when run first. Reserving keeps them bindable and out of ephemeral allocation.
+- `net.ipv4.tcp_tw_reuse` → 1 (outbound reuse only — it does not help a listening bind)
 - `net.ipv4.tcp_max_tw_buckets` → 131072
 - `net.core.rmem_max` / `wmem_max` → 7.5 MB (UDP buffer for QUIC)
 - Loopback MTU → 1500 (realistic Ethernet; the default 65536 hides kernel segmentation cost)

site/content/docs/running-locally/scripts/benchmark.md

@@ -31,6 +31,7 @@ The driver itself is small (~320 lines of orchestration) - all of the real work
 4. **Host tuning** (`scripts/lib/system.sh`):
    - CPU governor → `performance` via `cpupower` (falls back to writing `/sys/devices/system/cpu/cpu*/cpufreq/scaling_governor`).
    - `net.core.somaxconn=65535`, `tcp_max_syn_backlog=65535`, `netdev_max_backlog=65535`, `rmem_max=wmem_max=7500000` (QUIC).
+   - `ip_local_port_range=1024 65535`, and `ip_local_reserved_ports=5432,6379,8080,8081,8082,8443,9090` so the widened range cannot hand a service's own port to an outbound socket. See [Kernel tuning](../../../hardware/#kernel-tuning-applied-per-run).
    - `ip link set lo mtu 1500` - realistic Ethernet MTU, not the kernel's default 65536.
    - `systemctl restart docker` - guarantees every subsequent container starts from a fresh daemon state.
    - `echo 3 > /proc/sys/vm/drop_caches`.

site/content/docs/test-profiles/gateway/production-stack/implementation.md

@@ -4,7 +4,7 @@ seo_title: "Production Stack Benchmark — Implementation Guide"
 description: "Endpoint contract, request and response shapes, and the anti-cheat constraints a framework must satisfy for the four-service Production Stack deployment."
 weight: 1
 ---
-{{< type-rules standard="Must ship exactly four services: edge (reverse proxy), cache (Redis), authsvc (the shared JWT verifier from frameworks/_shared/authsvc/, built as-is), and server (the framework). No custom auth implementations - JWT verification must happen at the edge via auth_request / forward_auth using the shared authsvc. The framework must implement cache-aside on /api/items reads (check cache → miss → query DB → populate cache with ≤1 second TTL) and cache invalidation on /api/items writes (clear cache after DB update). How the caching is implemented is the framework's choice - any combination of in-process cache, Redis, or framework-specific cache abstraction is allowed." tuned="Same four-service shape as production. May tune proxy configuration, connection pools, CPU split, and cache TTLs. May NOT replace authsvc with a custom implementation or skip JWT verification." engine="No specific rules. May replace any service with a custom implementation. Ranked separately from frameworks." >}}
+{{< type-rules standard="Must ship exactly four services: edge (reverse proxy), cache (Redis), authsvc (the shared JWT verifier from frameworks/_shared/authsvc/, built as-is), and server (the framework). No custom auth implementations - JWT verification must happen at the edge via auth_request / forward_auth using the shared authsvc. The framework must implement cache-aside on /api/items reads (check cache → miss → query DB → populate cache with ≤1 second TTL) and cache invalidation on /api/items writes (clear cache after DB update). How the caching is implemented is the framework's choice - any combination of in-process cache, Redis, or framework-specific cache abstraction is allowed. The server must wait on a healthy cache - depends_on with condition: service_healthy, not the short list form - so a cache that cannot start fails the stack instead of leaving the run measured against whatever else holds 6379." tuned="Same four-service shape as production. May tune proxy configuration, connection pools, CPU split, and cache TTLs. May NOT replace authsvc with a custom implementation or skip JWT verification. The server must wait on a healthy cache - depends_on with condition: service_healthy, not the short list form - so a cache that cannot start fails the stack instead of leaving the run measured against whatever else holds 6379." engine="No specific rules. May replace any service with a custom implementation. Ranked separately from frameworks." >}}
 
 ## Overview
 
@@ -164,6 +164,46 @@ Every production-stack entry ships `compose.production-stack.yml` with exactly f
 
 Same as [Gateway-64](../gateway-h2/implementation/#required-compose-settings): `network_mode: host`, pinned `cpuset`, `seccomp:unconfined`, and memlock/nofile ulimits on all four services.
 
+Two more are specific to this profile, and both exist so the stack fails loudly instead of quietly measuring something else:
+
+- **`server` must wait for a healthy `cache`.** Use the map form with a condition, not the short list form:
+
+  ```yaml
+  cache:
+    healthcheck:
+      test: ["CMD", "redis-cli", "-p", "6379", "ping"]
+      interval: 30s
+      timeout: 2s
+      retries: 3
+      start_period: 30s
+      start_interval: 1s
+
+  server:
+    depends_on:
+      cache:
+        condition: service_healthy
+  ```
+
+  `depends_on: - cache` waits only for the container to *start*. A cache that dies on boot — because something else already holds 6379, say — leaves the server running happily against whatever Redis does hold it, and the profile publishes numbers measured against a cache it never configured. Entries ran this way for weeks without anyone noticing.
+
+- **`edge` must wait for a healthy `authsvc`** (`condition: service_healthy`). Without it a run can measure 5xx from the edge and never reach the server at all.
+
+### Ports the stack owns
+
+All four services run on `network_mode: host`, so for the duration of the profile these are host ports, not container ports:
+
+| Port | Service |
+|---|---|
+| 8443 | edge (h2 over TLS) |
+| 9090 | authsvc |
+| 8080 | server |
+| 6379 | cache |
+
+Two things the benchmark driver does so they are actually free:
+
+- It **stops its own Redis sidecar** before bringing the stack up, and restarts it afterwards. That sidecar is started once per run whenever the entry also subscribes to [CRUD](../../h1/isolated/crud/implementation/) and holds 6379 for the whole run, so an entry subscribed to both would otherwise collide with itself on every run.
+- It keeps all four **out of the ephemeral port range**. `ip_local_port_range` is widened to `1024 65535` for connection-churning profiles, which would otherwise let a load generator be holding one of these when the stack starts. See [Kernel tuning](../../../hardware/#kernel-tuning-applied-per-run).
+
 ### Environment variables
 
 | Variable | Set on | Value |